diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt --- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt @@ -37,7 +37,8 @@ set(LLVM_TARGET_DEFINITIONS LLVMIntrinsicOps.td) mlir_tablegen(LLVMIntrinsicConversions.inc -gen-llvmir-conversions) -mlir_tablegen(LLVMIntrinsicToLLVMIROpPairs.inc -gen-llvmintrinsic-to-llvmirop-pairs) +mlir_tablegen(LLVMIntrinsicFromLLVMIRConversions.inc -gen-intr-from-llvmir-conversions) +mlir_tablegen(LLVMConvertibleLLVMIRIntrinsics.inc -gen-convertible-llvmir-intrinsics) add_public_tablegen_target(MLIRLLVMIntrinsicConversionsIncGen) add_mlir_dialect(NVVMOps nvvm) @@ -55,4 +56,3 @@ set(LLVM_TARGET_DEFINITIONS ROCDLOps.td) mlir_tablegen(ROCDLConversions.inc -gen-llvmir-conversions) add_public_tablegen_target(MLIRROCDLConversionsIncGen) - diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -118,6 +118,14 @@ moduleTranslation.lookupValue(op.getPtr())}); }]; + // Custom builder to convert the size argument to an attribute. + string mlirBuilder = [{ + OperationState state($_location, "llvm.}] # opName # [{"); + state.addAttribute("size", lookupIntegerAttr(inst->getArgOperand(0))); + state.addOperands(lookupValue(inst->getArgOperand(1))); + b.create(state); + }]; + let assemblyFormat = "$size `,` $ptr attr-dict `:` type($ptr)"; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -302,7 +302,9 @@ // to be accessed via the LLVMStructType, instead of directly via the result. // "opName" contains the name of the operation to be associated with the // intrinsic and "enumName" contains the name of the intrinsic as appears in -// `llvm::Intrinsic` enum; one usually wants these to be related. +// `llvm::Intrinsic` enum; one usually wants these to be related. Additionally, +// the base class also defines the "mlirBuilder" field to support the inverse +// translation starting from an LLVM IR intrinsic. class LLVM_IntrOpBase overloadedResults, list overloadedOperands, list traits, int numResults, @@ -312,7 +314,7 @@ string resultPattern = !if(!gt(numResults, 1), LLVM_IntrPatterns.structResult, LLVM_IntrPatterns.result); - string llvmEnumName = enumName; + string llvmEnumName = enumName; let llvmBuilder = [{ llvm::Module *module = builder.GetInsertBlock()->getModule(); llvm::Function *fn = llvm::Intrinsic::getDeclaration( @@ -332,6 +334,22 @@ "moduleTranslation.setAliasScopeMetadata(op, inst);", "(void) inst;") # !if(!gt(numResults, 0), "$res = inst;", ""); + + // A pattern for build an MLIR LLVM dialect operation starting from the + // matching LLVM IR intrinsic. The pattern can use the following special + // variable names: + // - $_resultType - LLVM IR intrinsic result type translated to MLIR; + // - $_location - MLIR location of the intrinsic to build. + // Additionally, `$$` can be used to produce the dollar character. + string mlirBuilder = [{ + OperationState state($_location, "llvm.}] # opName # [{"); + SmallVector operands(inst->args()); + state.addOperands(lookupValues(operands)); + }] # !if(!gt(numResults, 0), + "state.addTypes($_resultType);", "") # [{ + Operation *op = b.create(state); + }] # !if(!gt(numResults, 0), + "instMap[inst] = op->getResult(0);", "(void) op;"); } // Base class for LLVM intrinsic operations, should not be used directly. Places diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp --- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp +++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp @@ -18,6 +18,7 @@ #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/MLIRContext.h" +#include "mlir/IR/Matchers.h" #include "mlir/Interfaces/DataLayoutInterfaces.h" #include "mlir/Target/LLVMIR/TypeFromLLVM.h" #include "mlir/Tools/mlir-translate/Translation.h" @@ -31,6 +32,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Type.h" #include "llvm/IRReader/IRReader.h" @@ -42,6 +44,15 @@ #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsFromLLVM.inc" +/// Returns true if the LLVM IR intrinsic is convertible to an MLIR LLVM dialect +/// intrinsic, or false if no counterpart exists. +static bool isConvertibleIntrinsic(llvm::Intrinsic::ID id) { + static const DenseSet convertibleIntrinsics = { +#include "mlir/Dialect/LLVMIR/LLVMConvertibleLLVMIRIntrinsics.inc" + }; + return convertibleIntrinsics.contains(id); +} + // Utility to print an LLVM value as a string for passing to emitError(). // FIXME: Diagnostic should be able to natively handle types that have // operator << (raw_ostream&) defined. @@ -173,6 +184,10 @@ /// visited. SmallVector lookupValues(ArrayRef values); + /// Calls `lookupValue` and returns the remapped version of `value` converted + /// to an integer attribute. Asserts if the conversion fails. + IntegerAttr lookupIntegerAttr(llvm::Value *value); + /// Translate the debug location to a FileLineColLoc, if `loc` is non-null. /// Otherwise, return UnknownLoc. Location translateLoc(llvm::DILocation *loc); @@ -180,6 +195,10 @@ /// Converts the type from LLVM to MLIR LLVM dialect. Type convertType(llvm::Type *type); + /// Converts an LLVM intrinsic to an MLIR LLVM dialect operation if an MLIR + /// counterpart exists. Otherwise, returns failure. + LogicalResult convertIntrinsic(llvm::CallInst *inst); + /// Imports `f` into the current module. LogicalResult processFunction(llvm::Function *f); @@ -263,6 +282,21 @@ return typeTranslator.translateType(type); } +LogicalResult Importer::convertIntrinsic(llvm::CallInst *inst) { + llvm::Function *callee = inst->getCalledFunction(); + if (!callee || !callee->isIntrinsic()) + return failure(); + + // Check if the intrinsic is convertible to an MLIR dialect counterpart. + llvm::Intrinsic::ID id = callee->getIntrinsicID(); + if (!isConvertibleIntrinsic(id)) + return failure(); + +#include "mlir/Dialect/LLVMIR/LLVMIntrinsicFromLLVMIRConversions.inc" + + return failure(); +} + // We only need integers, floats, doubles, and vectors and tensors thereof for // attributes. Scalar and vector types are converted to the standard // equivalents. Array types are converted to ranked tensors; nested array types @@ -568,6 +602,14 @@ return remapped; } +IntegerAttr Importer::lookupIntegerAttr(llvm::Value *value) { + IntegerAttr integerAttr; + bool success = matchPattern(lookupValue(value), m_Constant(&integerAttr)); + assert(success && "expected a constant value"); + (void)success; + return integerAttr; +} + /// Return the MLIR OperationName for the given LLVM opcode. static StringRef lookupOperationNameFromOpcode(unsigned opcode) { // Maps from LLVM opcode to MLIR OperationName. This is deliberately ordered @@ -649,15 +691,6 @@ return opcMap.lookup(opcode); } -/// Return the MLIR OperationName for the given LLVM intrinsic ID. -static StringRef lookupOperationNameFromIntrinsicID(unsigned id) { - // Maps from LLVM intrinsic ID to MLIR OperationName. - static const DenseMap intrMap = { -#include "mlir/Dialect/LLVMIR/LLVMIntrinsicToLLVMIROpPairs.inc" - }; - return intrMap.lookup(id); -} - static ICmpPredicate getICmpPredicate(llvm::CmpInst::Predicate p) { switch (p) { default: @@ -959,6 +992,11 @@ } case llvm::Instruction::Call: { llvm::CallInst *ci = cast(inst); + + // For all intrinsics, try to generate to the corresponding op. + if (succeeded(convertIntrinsic(ci))) + return success(); + SmallVector args(ci->args()); SmallVector ops = lookupValues(args); SmallVector tys; @@ -968,20 +1006,6 @@ } Operation *op; if (llvm::Function *callee = ci->getCalledFunction()) { - // For all intrinsics, try to generate to the corresponding op. - if (callee->isIntrinsic()) { - auto id = callee->getIntrinsicID(); - StringRef opName = lookupOperationNameFromIntrinsicID(id); - if (!opName.empty()) { - OperationState state(loc, opName); - state.addOperands(ops); - state.addTypes(tys); - Operation *op = b.create(state); - if (!inst->getType()->isVoidTy()) - instMap[inst] = op->getResult(0); - return success(); - } - } op = b.create( loc, tys, SymbolRefAttr::get(b.getContext(), callee->getName()), ops); } else { @@ -1171,12 +1195,8 @@ auto functionType = convertType(f->getFunctionType()).dyn_cast(); - if (f->isIntrinsic()) { - StringRef opName = lookupOperationNameFromIntrinsicID(f->getIntrinsicID()); - // Skip the intrinsic decleration if we could found a corresponding op. - if (!opName.empty()) - return success(); - } + if (f->isIntrinsic() && isConvertibleIntrinsic(f->getIntrinsicID())) + return success(); bool dsoLocal = f->hasLocalLinkage(); CConv cconv = convertCConvFromLLVM(f->getCallingConv()); diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -234,13 +234,13 @@ define void @vector_reductions(float %0, <8 x float> %1, <8 x i32> %2) { ; CHECK: "llvm.intr.vector.reduce.add"(%{{.*}}) : (vector<8xi32>) -> i32 %4 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2) - ; CHECK: "llvm.intr.vector.reduce.and"(%{{.*}}) : (vector<8xi32>) -> i32 + ; CHECK: "llvm.intr.vector.reduce.and"(%{{.*}}) : (vector<8xi32>) -> i32 %5 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %2) ; CHECK: "llvm.intr.vector.reduce.fmax"(%{{.*}}) : (vector<8xf32>) -> f32 %6 = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> %1) ; CHECK: "llvm.intr.vector.reduce.fmin"(%{{.*}}) : (vector<8xf32>) -> f32 %7 = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> %1) - ; CHECK: "llvm.intr.vector.reduce.mul"(%{{.*}}) : (vector<8xi32>) -> i32 + ; CHECK: "llvm.intr.vector.reduce.mul"(%{{.*}}) : (vector<8xi32>) -> i32 %8 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> %2) ; CHECK: "llvm.intr.vector.reduce.or"(%{{.*}}) : (vector<8xi32>) -> i32 %9 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %2) @@ -447,7 +447,7 @@ ; CHECK-LABEL: llvm.func @coro_end define void @coro_end(i8* %0, i1 %1) { - ; CHECK: llvm.intr.coro.end + ; CHECK: llvm.intr.coro.end call i1 @llvm.coro.end(i8* %0, i1 %1) ret void } @@ -489,11 +489,20 @@ ret void } +; CHECK-LABEL: llvm.func @lifetime +define void @lifetime(i8* %0) { + ; CHECK: llvm.intr.lifetime.start 16, %{{.*}} : !llvm.ptr + call void @llvm.lifetime.start.p0i8(i64 16, i8* %0) + ; CHECK: llvm.intr.lifetime.end 32, %{{.*}} : !llvm.ptr + call void @llvm.lifetime.end.p0i8(i64 32, i8* %0) + ret void +} + ; CHECK-LABEL: llvm.func @vector_predication_intrinsics define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x float> %2, <8 x float> %3, <8 x i64> %4, <8 x double> %5, <8 x i32*> %6, i32 %7, float %8, i32* %9, float* %10, <8 x i1> %11, i32 %12) { ; CHECK: "llvm.intr.vp.add"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> %14 = call <8 x i32> @llvm.vp.add.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.sub"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> + ; CHECK: "llvm.intr.vp.sub"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> %15 = call <8 x i32> @llvm.vp.sub.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.mul"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> %16 = call <8 x i32> @llvm.vp.mul.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i1> %11, i32 %12) @@ -585,7 +594,7 @@ %57 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double> %5, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.ptrtoint"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.vec<8 x ptr>, vector<8xi1>, i32) -> vector<8xi64> %58 = call <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0i32(<8 x i32*> %6, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.inttoptr"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> !llvm.vec<8 x ptr> + ; CHECK: "llvm.intr.vp.inttoptr"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> !llvm.vec<8 x ptr> %59 = call <8 x i32*> @llvm.vp.inttoptr.v8p0i32.v8i64(<8 x i64> %4, <8 x i1> %11, i32 %12) ret void } @@ -662,7 +671,7 @@ declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, float* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) declare <7 x i1> @llvm.get.active.lane.mask.v7i1.i64(i64, i64) declare <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>*, i32 immarg, <7 x i1>, <7 x float>) -declare void @llvm.masked.store.v7f32.p0v7f32(<7 x float>, <7 x float>*, i32 immarg, <7 x i1>) +declare void @llvm.masked.store.v7f32.p0v7f32(<7 x float>, <7 x float>*, i32 immarg, <7 x i1>) declare <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*>, i32 immarg, <7 x i1>, <7 x float>) declare void @llvm.masked.scatter.v7f32.v7p0f32(<7 x float>, <7 x float*>, i32 immarg, <7 x i1>) declare <7 x float> @llvm.masked.expandload.v7f32(float*, <7 x i1>, <7 x float>) @@ -748,3 +757,5 @@ declare <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0i32(<8 x i32*>, <8 x i1>, i32) declare <8 x i32*> @llvm.vp.inttoptr.v8p0i32.v8i64(<8 x i64>, <8 x i1>, i32) +declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) +declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp --- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp +++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp @@ -174,6 +174,67 @@ return false; } +// Emit an intrinsic identifier driven check and a call to the builder of the +// MLIR LLVM dialect intrinsic operation to build for the given LLVM IR +// intrinsic identifier. Return true on success. +static bool emitOneIntrBuilder(const Record &record, raw_ostream &os) { + auto op = tblgen::Operator(record); + + if (!record.getValue("mlirBuilder")) + return emitError("no 'mlirBuilder' field for op " + op.getOperationName()); + + // Return early if there is no builder specified. + auto builderStrRef = record.getValueAsString("mlirBuilder"); + if (builderStrRef.empty()) + return true; + + // Progressively create the builder string by replacing $-variables with + // remapped operands, the result type, or the translated debug location. Keep + // only the not-yet-traversed part of the builder pattern to avoid + // re-traversing the string multiple times. + std::string builder; + llvm::raw_string_ostream bs(builder); + while (auto loc = findNextVariable(builderStrRef)) { + auto name = loc.in(builderStrRef).drop_front(); + // First, insert the non-matched part as is. + bs << builderStrRef.substr(0, loc.pos); + // Then, rewrite the name based on its kind. + if (name == "_resultType") { + bs << "convertType(inst->getType())"; + } else if (name == "_location") { + bs << "translateLoc(inst->getDebugLoc())"; + } else if (name == "$") { + bs << '$'; + } else { + return emitError(name + " is neither an argument nor a result of " + + op.getOperationName()); + } + // Finally, only keep the untraversed part of the string. + builderStrRef = builderStrRef.substr(loc.pos + loc.length); + } + + // Output the check and the builder string. + os << "if (id == llvm::Intrinsic::" << record.getValueAsString("llvmEnumName") + << ") {\n"; + os << bs.str() << builderStrRef << "\n"; + os << " return success();\n"; + os << "}\n"; + + return true; +} + +// Emit all intrinsic builders. Returns false on success because of the +// generator registration requirements. +static bool emitIntrBuilders(const RecordKeeper &recordKeeper, + raw_ostream &os) { + for (const auto *def : + recordKeeper.getAllDerivedDefinitions("LLVM_IntrOpBase")) { + if (!emitOneIntrBuilder(*def, os)) + return true; + } + return false; +} + namespace { // Wrapper class around a Tablegen definition of an LLVM enum attribute case. class LLVMEnumAttrCase : public tblgen::EnumAttrCase { @@ -377,16 +438,18 @@ return false; } -static void emitIntrOpPair(const Record &record, raw_ostream &os) { +static void emitOneIntrinsic(const Record &record, raw_ostream &os) { auto op = tblgen::Operator(record); - os << "{llvm::Intrinsic::" << record.getValueAsString("llvmEnumName") << ", " - << op.getQualCppClassName() << "::getOperationName()},\n"; + os << "llvm::Intrinsic::" << record.getValueAsString("llvmEnumName") << ",\n"; } -static bool emitIntrOpPairs(const RecordKeeper &recordKeeper, raw_ostream &os) { +// Emit the list of LLVM IR intrinsics identifiers that are convertible to a +// matching MLIR LLVM dialect intrinsic operation. +static bool emitConvertibleIntrinsics(const RecordKeeper &recordKeeper, + raw_ostream &os) { for (const auto *def : recordKeeper.getAllDerivedDefinitions("LLVM_IntrOpBase")) - emitIntrOpPair(*def, os); + emitOneIntrinsic(*def, os); return false; } @@ -395,6 +458,11 @@ genLLVMIRConversions("gen-llvmir-conversions", "Generate LLVM IR conversions", emitBuilders); +static mlir::GenRegistration + genIntrFromLLVMIRConversions("gen-intr-from-llvmir-conversions", + "Generate intrinsic conversions from LLVM IR", + emitIntrBuilders); + static mlir::GenRegistration genEnumToLLVMConversion("gen-enum-to-llvmir-conversions", "Generate conversions of EnumAttrs to LLVM IR", @@ -405,7 +473,7 @@ "Generate conversions of EnumAttrs from LLVM IR", emitEnumConversionDefs); -static mlir::GenRegistration - genLLVMIntrinsicToOpPairs("gen-llvmintrinsic-to-llvmirop-pairs", - "Generate LLVM intrinsic to LLVMIR op pairs", - emitIntrOpPairs); +static mlir::GenRegistration genConvertibleLLVMIRIntrinsics( + "gen-convertible-llvmir-intrinsics", + "Generate list of convertible LLVM IR intrinsics", + emitConvertibleIntrinsics);