diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -265,6 +265,37 @@ def LLVM_AssumeOp : LLVM_ZeroResultIntrOp<"assume", []>, Arguments<(ins I1:$cond)>; +// +// Expect intrinsics. +// + +def LLVM_ExpectOp + : LLVM_OneResultIntrOp<"expect", [], [0], + [Pure, SameOperandsAndResultType]> { + let arguments = (ins AnySignlessInteger:$val, + AnySignlessInteger:$expected); + let assemblyFormat = "$val `,` $expected attr-dict `:` type($val)"; +} + +def LLVM_ExpectWithProbabilityOp + : LLVM_OneResultIntrOp<"expect.with.probability", [], [0], + [Pure, AllTypesMatch<["val", "expected", "res"]>]> { + let arguments = (ins AnySignlessInteger:$val, + AnySignlessInteger:$expected, + F64Attr:$prob); + string llvmBuilder = [{ + createIntrinsicCall( + builder, llvm::Intrinsic::expect_with_probability, + {$val, $expected, llvm::ConstantFP::get(builder.getDoubleTy(), $prob)}, + {$_resultType}); + }]; + string mlirBuilder = [{ + $res = $_builder.create( + $_location, $val, $expected, $_float_attr($prob)); + }]; + let assemblyFormat = "$val `,` $expected `,` $prob attr-dict `:` type($val)"; +} + // // Coroutine intrinsics. // @@ -727,8 +758,8 @@ def LLVM_UBSanTrap : LLVM_ZeroResultIntrOp<"ubsantrap"> { let arguments = (ins I8Attr:$failureKind); string llvmBuilder = [{ - builder.CreateIntrinsic( - llvm::Intrinsic::ubsantrap, {}, {builder.getInt8($failureKind)}); + createIntrinsicCall( + builder, llvm::Intrinsic::ubsantrap, {builder.getInt8($failureKind)}); }]; string mlirBuilder = [{ $_op = diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -159,6 +159,7 @@ // - $_op - substituted by a reference to store the newly created MLIR // operation (only for MLIR operations that return no result); // - $_int_attr - substituted by a call to an integer attribute matcher; + // - $_float_attr - substituted by a call to a float attribute matcher; // - $_var_attr - substituted by a call to a variable attribute matcher; // - $_resultType - substituted with the MLIR result type; // - $_location - substituted with the MLIR location; diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -136,6 +136,9 @@ /// Converts `value` to an integer attribute. Asserts if the matching fails. IntegerAttr matchIntegerAttr(llvm::Value *value); + /// Converts `value` to a float attribute. Asserts if the matching fails. + FloatAttr matchFloatAttr(llvm::Value *value); + /// Converts `value` to a local variable attribute. Asserts if the matching /// fails. DILocalVariableAttr matchLocalVariableAttr(llvm::Value *value); diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -24,6 +24,7 @@ using namespace mlir; using namespace mlir::LLVM; +using mlir::LLVM::detail::createIntrinsicCall; using mlir::LLVM::detail::getLLVMConstant; #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc" diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -1163,11 +1163,21 @@ FailureOr converted = convertValue(value); bool success = succeeded(converted) && matchPattern(*converted, m_Constant(&integerAttr)); - assert(success && "expected a constant value"); + assert(success && "expected a constant integer value"); (void)success; return integerAttr; } +FloatAttr ModuleImport::matchFloatAttr(llvm::Value *value) { + FloatAttr floatAttr; + FailureOr converted = convertValue(value); + bool success = + succeeded(converted) && matchPattern(*converted, m_Constant(&floatAttr)); + assert(success && "expected a constant float value"); + (void)success; + return floatAttr; +} + DILocalVariableAttr ModuleImport::matchLocalVariableAttr(llvm::Value *value) { auto *nodeAsVal = cast(value); auto *node = cast(nodeAsVal->getMetadata()); diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -467,6 +467,24 @@ ret void } +; CHECK-LABEL: @expect +; CHECK-SAME: %[[VAL:[a-zA-Z0-9]+]] +define void @expect(i32 %0) { + ; CHECK: %[[EXP:.+]] = llvm.mlir.constant(42 : i32) : i32 + ; CHECK: llvm.intr.expect %[[VAL]], %[[EXP]] : i32 + %2 = call i32 @llvm.expect.i32(i32 %0, i32 42) + ret void +} + +; CHECK-LABEL: @expect_with_probability +; CHECK-SAME: %[[VAL:[a-zA-Z0-9]+]] +define void @expect_with_probability(i16 %0) { + ; CHECK: %[[EXP:.+]] = llvm.mlir.constant(42 : i16) : i16 + ; CHECK: llvm.intr.expect.with.probability %[[VAL]], %[[EXP]], 5.000000e-01 : i16 + %2 = call i16 @llvm.expect.with.probability.i16(i16 %0, i16 42, double 0.5) + ret void +} + ; CHECK-LABEL: llvm.func @coro_id define void @coro_id(i32 %0, ptr %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (i32, !llvm.ptr, !llvm.ptr, !llvm.ptr) -> !llvm.token @@ -774,6 +792,8 @@ declare { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>) declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.expect.i32(i32, i32) +declare i16 @llvm.expect.with.probability.i16(i16, i16, double immarg) declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) declare ptr @llvm.coro.begin(token, ptr writeonly) declare i64 @llvm.coro.size.i64() diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -466,6 +466,22 @@ llvm.return } +// CHECK-LABEL: @expect +llvm.func @expect(%arg0: i32) { + %0 = llvm.mlir.constant(42 : i32) : i32 + // CHECK: call i32 @llvm.expect.i32(i32 %{{.*}}, i32 42) + %1 = llvm.intr.expect %arg0, %0 : i32 + llvm.return +} + +// CHECK-LABEL: @expect_with_probability +llvm.func @expect_with_probability(%arg0: i16) { + %0 = llvm.mlir.constant(42 : i16) : i16 + // CHECK: call i16 @llvm.expect.with.probability.i16(i16 %{{.*}}, i16 42, double 5.000000e-01) + %1 = llvm.intr.expect.with.probability %arg0, %0, 5.000000e-01 : i16 + llvm.return +} + // CHECK-LABEL: @coro_id llvm.func @coro_id(%arg0: i32, %arg1: !llvm.ptr) { // CHECK: call token @llvm.coro.id @@ -814,6 +830,8 @@ // CHECK-DAG: declare { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32>, <8 x i32>) // CHECK-DAG: declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) // CHECK-DAG: declare { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.expect.i32(i32, i32) +// CHECK-DAG: declare i16 @llvm.expect.with.probability.i16(i16, i16, double immarg) // CHECK-DAG: declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) // CHECK-DAG: declare ptr @llvm.coro.begin(token, ptr writeonly) // CHECK-DAG: declare i64 @llvm.coro.size.i64() diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp --- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp +++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp @@ -266,6 +266,8 @@ bs << "moduleImport.mapNoResultOp(inst)"; } else if (name == "_int_attr") { bs << "moduleImport.matchIntegerAttr"; + } else if (name == "_float_attr") { + bs << "moduleImport.matchFloatAttr"; } else if (name == "_var_attr") { bs << "moduleImport.matchLocalVariableAttr"; } else if (name == "_resultType") {