diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -49,38 +49,147 @@ } //===----------------------------------------------------------------------===// -// LoopOptionsAttr +// Loop Attributes //===----------------------------------------------------------------------===// -def LoopOptionsAttr : LLVM_Attr<"LoopOptions", "loopopts"> { +def LoopVectorizeAttr : LLVM_Attr<"LoopVectorize", "loop_vectorize"> { let description = [{ - This attributes encapsulates "loop options". It is means to decorate - branches that are "latches" (loop backedges) and maps to the `!llvm.loop` - metadatas: https://llvm.org/docs/LangRef.html#llvm-loop - It store the options as a pair in a sorted array and expose - APIs to retrieve the value for each option with a stronger type (bool for - example). + This attribute defines vectorization specific loop annotations that map to + the "!llvm.loop.vectorize" metadata. }]; let parameters = (ins - ArrayRefParameter<"std::pair", "">:$options + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$predicateEnable, + OptionalParameter<"BoolAttr">:$scalableEnable, + OptionalParameter<"IntegerAttr">:$width, + OptionalParameter<"LoopAnnotationAttr">:$followupVectorized, + OptionalParameter<"LoopAnnotationAttr">:$followupEpilogue, + OptionalParameter<"LoopAnnotationAttr">:$followupAll ); - let extraClassDeclaration = [{ - using OptionValuePair = std::pair; - using OptionsArray = ArrayRef>; - std::optional disableUnroll(); - std::optional disableLICM(); - std::optional interleaveCount(); + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopInterleaveAttr : LLVM_Attr<"LoopInterleave", "loop_interleave"> { + let description = [{ + This attribute defines interleaving specific loop annotations that map to + the "!llvm.loop.interleave" metadata. }]; - let builders = [ - /// Build the LoopOptions Attribute from a sorted array of individual options. - AttrBuilder<(ins "ArrayRef>":$sortedOptions)>, - AttrBuilder<(ins "LoopOptionsAttrBuilder &":$optionBuilders)> - ]; - let hasCustomAssemblyFormat = 1; - let skipDefaultBuilders = 1; + let parameters = (ins + "IntegerAttr":$count + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollAttr : LLVM_Attr<"LoopUnroll", "loop_unroll"> { + let description = [{ + This attribute defines unrolling specific loop annotations that map to + the "!llvm.loop.unroll" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"BoolAttr">:$runtimeDisable, + OptionalParameter<"BoolAttr">:$full, + OptionalParameter<"LoopAnnotationAttr">:$followup, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainder + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollAndJamAttr : LLVM_Attr<"LoopUnrollAndJam", "loop_unroll_and_jam"> { + let description = [{ + This attribute defines "unroll and jam" specific loop annotations that map to + the "!llvm.loop.unroll_and_jam" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"LoopAnnotationAttr">:$followupOuter, + OptionalParameter<"LoopAnnotationAttr">:$followupInner, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainderOuter, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainderInner, + OptionalParameter<"LoopAnnotationAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopLICMAttr : LLVM_Attr<"LoopLICM", "loop_licm"> { + let description = [{ + This attribute encapsulates loop invariant code motion (licm) specific loop + annotations. The fields correspond to the "!llvm.licm.disable" and the + "!llvm.loop.licm_versioning.disable" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$versioningDisable + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopDistributeAttr : LLVM_Attr<"LoopDistribute", "loop_distribute"> { + let description = [{ + This attribute defines distribution specific loop annotations that map to + the "!llvm.loop.distribute" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"LoopAnnotationAttr">:$followupCoincident, + OptionalParameter<"LoopAnnotationAttr">:$followupSequential, + OptionalParameter<"LoopAnnotationAttr">:$followupFallback, + OptionalParameter<"LoopAnnotationAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopPipelineAttr : LLVM_Attr<"LoopPipeline", "loop_pipeline"> { + let description = [{ + This attribute defines pipelining specific loop annotations that map to + the "!llvm.loop.pipeline" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$initiationinterval + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> { + let description = [{ + This attributes encapsulates "loop metadata". It is meant to decorate + branches that are "latches" (loop backedges) and maps to the `!llvm.loop` + metadatas: https://llvm.org/docs/LangRef.html#llvm-loop + It stores annotations in attribute parameters and groups related options in + nested attributes to provide structured access. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disableNonforced, + OptionalParameter<"LoopVectorizeAttr">:$vectorize, + OptionalParameter<"LoopInterleaveAttr">:$interleave, + OptionalParameter<"LoopUnrollAttr">:$unroll, + OptionalParameter<"LoopUnrollAndJamAttr">:$unrollAndJam, + OptionalParameter<"LoopLICMAttr">:$licm, + OptionalParameter<"LoopDistributeAttr">:$distribute, + OptionalParameter<"LoopPipelineAttr">:$pipeline, + OptionalParameter<"BoolAttr">:$mustProgress, + OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses + ); + + let assemblyFormat = "`<` struct(params) `>`"; } //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h @@ -22,7 +22,6 @@ namespace mlir { namespace LLVM { -class LoopOptionsAttrBuilder; /// This class represents the base attribute for all debug info attributes. class DINodeAttr : public Attribute { @@ -74,54 +73,4 @@ #define GET_ATTRDEF_CLASSES #include "mlir/Dialect/LLVMIR/LLVMOpsAttrDefs.h.inc" -namespace mlir { -namespace LLVM { - -/// Builder class for LoopOptionsAttr. This helper class allows to progressively -/// build a LoopOptionsAttr one option at a time, and pay the price of attribute -/// creation once all the options are in place. -class LoopOptionsAttrBuilder { -public: - /// Construct a empty builder. - LoopOptionsAttrBuilder() = default; - - /// Construct a builder with an initial list of options from an existing - /// LoopOptionsAttr. - LoopOptionsAttrBuilder(LoopOptionsAttr attr); - - /// Set the `disable_licm` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableLICM(std::optional value); - - /// Set the `interleave_count` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setInterleaveCount(std::optional count); - - /// Set the `disable_unroll` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableUnroll(std::optional value); - - /// Set the `disable_pipeline` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisablePipeline(std::optional value); - - /// Set the `pipeline_initiation_interval` option to the provided value. - /// If no value is provided the option is deleted. - LoopOptionsAttrBuilder & - setPipelineInitiationInterval(std::optional count); - - /// Returns true if any option has been set. - bool empty() { return options.empty(); } - -private: - template - LoopOptionsAttrBuilder &setOption(LoopOptionCase tag, std::optional value); - - friend class LoopOptionsAttr; - SmallVector options; -}; - -} // namespace LLVM -} // namespace mlir - #endif // MLIR_DIALECT_LLVMIR_LLVMATTRS_H_ diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -489,25 +489,6 @@ "::mlir::LLVM::LinkageAttr::get($_builder.getContext(), $0)"; } -//===----------------------------------------------------------------------===// -// LoopOptions -//===----------------------------------------------------------------------===// - -def LOptDisableUnroll : I32EnumAttrCase<"disable_unroll", 1>; -def LOptDisableLICM : I32EnumAttrCase<"disable_licm", 2>; -def LOptInterleaveCount : I32EnumAttrCase<"interleave_count", 3>; -def LOptDisablePipeline : I32EnumAttrCase<"disable_pipeline", 4>; -def LOptPipelineInitiationInterval : I32EnumAttrCase<"pipeline_initiation_interval", 5>; - -def LoopOptionCase : I32EnumAttr< - "LoopOptionCase", - "LLVM loop option", - [LOptDisableUnroll, LOptDisableLICM, LOptInterleaveCount, - LOptDisablePipeline, LOptPipelineInitiationInterval - ]> { - let cppNamespace = "::mlir::LLVM"; -} - //===----------------------------------------------------------------------===// // UnnamedAddr //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -39,8 +39,6 @@ static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } static StringRef getAliasScopesAttrName() { return "alias_scopes"; } static StringRef getLoopAttrName() { return "llvm.loop"; } - static StringRef getParallelAccessAttrName() { return "parallel_access"; } - static StringRef getLoopOptionsAttrName() { return "options"; } static StringRef getAccessGroupsAttrName() { return "access_groups"; } static StringRef getStructAttrsAttrName() { return "llvm.struct_attrs"; } static StringRef getTBAAAttrName() { return "llvm.tbaa"; } diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -40,6 +40,7 @@ namespace detail { class DebugTranslation; +class LoopAnnotationTranslation; } // namespace detail class DINodeAttr; @@ -129,19 +130,6 @@ llvm::MDNode *getAliasScope(Operation &opInst, SymbolRefAttr aliasScopeRef) const; - /// Returns the LLVM metadata corresponding to a llvm loop's codegen - /// options attribute. - llvm::MDNode *lookupLoopOptionsMetadata(Attribute options) const { - return loopOptionsMetadataMapping.lookup(options); - } - - void mapLoopOptionsMetadata(Attribute options, llvm::MDNode *metadata) { - auto result = loopOptionsMetadataMapping.try_emplace(options, metadata); - (void)result; - assert(result.second && - "attempting to map loop options that was already mapped"); - } - // Sets LLVM metadata for memory operations that are in a parallel loop. void setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst); @@ -152,6 +140,10 @@ /// TBAA attributes. void setTBAAMetadata(Operation *op, llvm::Instruction *inst); + /// Sets LLVM loop metadata for branch operations that have a loop annotation + /// attribute. + void setLoopMetadata(Operation *op, llvm::Instruction *inst); + /// Converts the type from MLIR LLVM dialect to LLVM. llvm::Type *convertType(Type type); @@ -315,6 +307,9 @@ /// A converter for translating debug information. std::unique_ptr debugTranslation; + /// A converter for translating loop annotations. + std::unique_ptr loopAnnotationTranslation; + /// Builder for LLVM IR generation of OpenMP constructs. std::unique_ptr ompBuilder; @@ -343,11 +338,6 @@ /// identified via their branches) and contained memory accesses. DenseMap accessGroupMetadataMapping; - /// Mapping from an attribute describing loop codegen options to its LLVM - /// metadata. The metadata is attached to Latch block branches with this - /// attribute. - DenseMap loopOptionsMetadataMapping; - /// Mapping from an alias scope metadata operation to its LLVM metadata. /// This map is populated on module entry. DenseMap aliasScopeMetadataMapping; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp @@ -92,176 +92,6 @@ return success(); } -//===----------------------------------------------------------------------===// -// LoopOptionsAttrBuilder -//===----------------------------------------------------------------------===// - -LoopOptionsAttrBuilder::LoopOptionsAttrBuilder(LoopOptionsAttr attr) - : options(attr.getOptions().begin(), attr.getOptions().end()) {} - -template -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setOption(LoopOptionCase tag, std::optional value) { - auto option = llvm::find_if( - options, [tag](auto option) { return option.first == tag; }); - if (option != options.end()) { - if (value) - option->second = *value; - else - options.erase(option); - } else { - options.push_back(LoopOptionsAttr::OptionValuePair(tag, *value)); - } - return *this; -} - -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableLICM(std::optional value) { - return setOption(LoopOptionCase::disable_licm, value); -} - -/// Set the `interleave_count` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setInterleaveCount(std::optional count) { - return setOption(LoopOptionCase::interleave_count, count); -} - -/// Set the `disable_unroll` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableUnroll(std::optional value) { - return setOption(LoopOptionCase::disable_unroll, value); -} - -/// Set the `disable_pipeline` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisablePipeline(std::optional value) { - return setOption(LoopOptionCase::disable_pipeline, value); -} - -/// Set the `pipeline_initiation_interval` option to the provided value. -/// If no value is provided the option is deleted. -LoopOptionsAttrBuilder &LoopOptionsAttrBuilder::setPipelineInitiationInterval( - std::optional count) { - return setOption(LoopOptionCase::pipeline_initiation_interval, count); -} - -//===----------------------------------------------------------------------===// -// LoopOptionsAttr -//===----------------------------------------------------------------------===// - -template -static std::optional -getOption(ArrayRef> options, - LoopOptionCase option) { - auto it = - lower_bound(options, option, [](auto optionPair, LoopOptionCase option) { - return optionPair.first < option; - }); - if (it == options.end()) - return {}; - return static_cast(it->second); -} - -std::optional LoopOptionsAttr::disableUnroll() { - return getOption(getOptions(), LoopOptionCase::disable_unroll); -} - -std::optional LoopOptionsAttr::disableLICM() { - return getOption(getOptions(), LoopOptionCase::disable_licm); -} - -std::optional LoopOptionsAttr::interleaveCount() { - return getOption(getOptions(), LoopOptionCase::interleave_count); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get( - MLIRContext *context, - ArrayRef> sortedOptions) { - assert(llvm::is_sorted(sortedOptions, llvm::less_first()) && - "LoopOptionsAttr ctor expects a sorted options array"); - return Base::get(context, sortedOptions); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get(MLIRContext *context, - LoopOptionsAttrBuilder &optionBuilders) { - llvm::sort(optionBuilders.options, llvm::less_first()); - return Base::get(context, optionBuilders.options); -} - -void LoopOptionsAttr::print(AsmPrinter &printer) const { - printer << "<"; - llvm::interleaveComma(getOptions(), printer, [&](auto option) { - printer << stringifyEnum(option.first) << " = "; - switch (option.first) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - printer << (option.second ? "true" : "false"); - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - printer << option.second; - break; - } - }); - printer << ">"; -} - -Attribute LoopOptionsAttr::parse(AsmParser &parser, Type type) { - if (failed(parser.parseLess())) - return {}; - - SmallVector> options; - llvm::SmallDenseSet seenOptions; - auto parseLoopOptions = [&]() -> ParseResult { - StringRef optionName; - if (parser.parseKeyword(&optionName)) - return failure(); - - auto option = symbolizeLoopOptionCase(optionName); - if (!option) - return parser.emitError(parser.getNameLoc(), "unknown loop option: ") - << optionName; - if (!seenOptions.insert(*option).second) - return parser.emitError(parser.getNameLoc(), "loop option present twice"); - if (failed(parser.parseEqual())) - return failure(); - - int64_t value; - switch (*option) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - if (succeeded(parser.parseOptionalKeyword("true"))) - value = 1; - else if (succeeded(parser.parseOptionalKeyword("false"))) - value = 0; - else { - return parser.emitError(parser.getNameLoc(), - "expected boolean value 'true' or 'false'"); - } - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - if (failed(parser.parseInteger(value))) - return parser.emitError(parser.getNameLoc(), "expected integer value"); - break; - } - options.push_back(std::make_pair(*option, value)); - return success(); - }; - if (parser.parseCommaSeparatedList(parseLoopOptions) || parser.parseGreater()) - return {}; - - llvm::sort(options, llvm::less_first()); - return get(parser.getContext(), options); -} - //===----------------------------------------------------------------------===// // MemoryEffectsAttr //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2713,7 +2713,10 @@ .Case([&](auto attr) { + DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr, + LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr, + LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr, + LoopPipelineAttr>([&](auto attr) { os << decltype(attr)::getMnemonic(); return AliasResult::OverridableAlias; }) @@ -2991,45 +2994,27 @@ // If the `llvm.loop` attribute is present, enforce the following structure, // which the module translation can assume. if (attr.getName() == LLVMDialect::getLoopAttrName()) { - auto loopAttr = attr.getValue().dyn_cast(); + auto loopAttr = attr.getValue().dyn_cast(); if (!loopAttr) return op->emitOpError() << "expected '" << LLVMDialect::getLoopAttrName() - << "' to be a dictionary attribute"; - std::optional parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - auto accessGroups = parallelAccessGroup->getValue().dyn_cast(); - if (!accessGroups) - return op->emitOpError() - << "expected '" << LLVMDialect::getParallelAccessAttrName() - << "' to be an array attribute"; - for (Attribute attr : accessGroups) { - auto accessGroupRef = attr.dyn_cast(); - if (!accessGroupRef) - return op->emitOpError() - << "expected '" << attr << "' to be a symbol reference"; - StringAttr metadataName = accessGroupRef.getRootReference(); - auto metadataOp = - SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << attr << "' to reference a metadata op"; - StringAttr accessGroupName = accessGroupRef.getLeafReference(); - Operation *accessGroupOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); - if (!accessGroupOp) - return op->emitOpError() - << "expected '" << attr << "' to reference an access_group op"; - } + << "' to be a loop annotation attribute"; + ArrayRef parallelAccesses = loopAttr.getParallelAccesses(); + if (parallelAccesses.empty()) + return success(); + for (SymbolRefAttr accessGroupRef : parallelAccesses) { + StringAttr metadataName = accessGroupRef.getRootReference(); + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference a metadata op"; + StringAttr accessGroupName = accessGroupRef.getLeafReference(); + Operation *accessGroupOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); + if (!accessGroupOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference an access_group op"; } - - std::optional loopOptions = - loopAttr.getNamed(LLVMDialect::getLoopOptionsAttrName()); - if (loopOptions && !loopOptions->getValue().isa()) - return op->emitOpError() - << "expected '" << LLVMDialect::getLoopOptionsAttrName() - << "' to be a `loopopts` attribute"; } if (attr.getName() == LLVMDialect::getStructAttrsAttrName()) { diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt --- a/mlir/lib/Target/LLVMIR/CMakeLists.txt +++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt @@ -5,6 +5,7 @@ ConvertToLLVMIR.cpp DebugTranslation.cpp DebugImporter.cpp + LoopAnnotationTranslation.cpp ModuleTranslation.cpp ModuleImport.cpp TypeToLLVM.cpp @@ -14,6 +15,7 @@ add_mlir_translation_library(MLIRTargetLLVMIRExport DebugTranslation.cpp + LoopAnnotationTranslation.cpp ModuleTranslation.cpp TypeToLLVM.cpp diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -167,90 +167,6 @@ return ret; } -/// Returns an LLVM metadata node corresponding to a loop option. This metadata -/// is attached to an llvm.loop node. -static llvm::MDNode *getLoopOptionMetadata(llvm::LLVMContext &ctx, - LoopOptionCase option, - int64_t value) { - StringRef name; - llvm::Constant *cstValue = nullptr; - switch (option) { - case LoopOptionCase::disable_licm: - name = "llvm.licm.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::disable_unroll: - name = "llvm.loop.unroll.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::interleave_count: - name = "llvm.loop.interleave.count"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; - case LoopOptionCase::disable_pipeline: - name = "llvm.loop.pipeline.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::pipeline_initiation_interval: - name = "llvm.loop.pipeline.initiationinterval"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; - } - return llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), - llvm::ConstantAsMetadata::get(cstValue)}); -} - -static void setLoopMetadata(Operation &opInst, llvm::Instruction &llvmInst, - llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation) { - if (Attribute attr = opInst.getAttr(LLVMDialect::getLoopAttrName())) { - llvm::Module *module = builder.GetInsertBlock()->getModule(); - llvm::MDNode *loopMD = moduleTranslation.lookupLoopOptionsMetadata(attr); - if (!loopMD) { - llvm::LLVMContext &ctx = module->getContext(); - - SmallVector loopOptions; - // Reserve operand 0 for loop id self reference. - auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); - loopOptions.push_back(dummy.get()); - - auto loopAttr = attr.cast(); - auto parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - SmallVector parallelAccess; - parallelAccess.push_back( - llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); - for (SymbolRefAttr accessGroupRef : parallelAccessGroup->getValue() - .cast() - .getAsRange()) - parallelAccess.push_back( - moduleTranslation.getAccessGroup(opInst, accessGroupRef)); - loopOptions.push_back(llvm::MDNode::get(ctx, parallelAccess)); - } - - if (auto loopOptionsAttr = loopAttr.getAs( - LLVMDialect::getLoopOptionsAttrName())) { - for (auto option : loopOptionsAttr.getOptions()) - loopOptions.push_back( - getLoopOptionMetadata(ctx, option.first, option.second)); - } - - // Create loop options and set the first operand to itself. - loopMD = llvm::MDNode::get(ctx, loopOptions); - loopMD->replaceOperandWith(0, loopMD); - - // Store a map from this Attribute to the LLVM metadata in case we - // encounter it again. - moduleTranslation.mapLoopOptionsMetadata(attr, loopMD); - } - - llvmInst.setMetadata(module->getMDKindID("llvm.loop"), loopMD); - } -} - /// Convert the value of a DenseI64ArrayAttr to a vector of unsigned indices. static SmallVector extractPosition(ArrayRef indices) { SmallVector position; @@ -505,7 +421,7 @@ llvm::BranchInst *branch = builder.CreateBr(moduleTranslation.lookupBlock(brOp.getSuccessor())); moduleTranslation.mapBranch(&opInst, branch); - setLoopMetadata(opInst, *branch, builder, moduleTranslation); + moduleTranslation.setLoopMetadata(&opInst, branch); return success(); } if (auto condbrOp = dyn_cast(opInst)) { @@ -516,7 +432,7 @@ moduleTranslation.lookupBlock(condbrOp.getSuccessor(0)), moduleTranslation.lookupBlock(condbrOp.getSuccessor(1)), branchWeights); moduleTranslation.mapBranch(&opInst, branch); - setLoopMetadata(opInst, *branch, builder, moduleTranslation); + moduleTranslation.setLoopMetadata(&opInst, branch); return success(); } if (auto switchOp = dyn_cast(opInst)) { diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h new file mode 100644 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h @@ -0,0 +1,57 @@ +//===- LoopAnnotationTranslation.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the translation between an MLIR loop annotations and +// the corresponding LLVMIR metadata representation. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ +#define MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Target/LLVMIR/ModuleTranslation.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// A helper class that converts a LoopAnnotationAttr into a corresponding +/// llvm::MDNode. +class LoopAnnotationTranslation { +public: + LoopAnnotationTranslation(LLVM::ModuleTranslation &moduleTranslation) + : moduleTranslation(moduleTranslation) {} + + llvm::MDNode *translate(LoopAnnotationAttr attr, Operation *op); + +private: + /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. + llvm::MDNode *lookupLoopMetadata(Attribute options) const { + return loopMetadataMapping.lookup(options); + } + + void mapLoopMetadata(Attribute options, llvm::MDNode *metadata) { + auto result = loopMetadataMapping.try_emplace(options, metadata); + (void)result; + assert(result.second && + "attempting to map loop options that was already mapped"); + } + + /// Mapping from an attribute describing loop metadata to its LLVM metadata. + /// The metadata is attached to Latch block branches with this attribute. + DenseMap loopMetadataMapping; + + LLVM::ModuleTranslation &moduleTranslation; +}; + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#endif // MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp new file mode 100644 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -0,0 +1,221 @@ +#include "LoopAnnotationTranslation.h" + +using namespace mlir; +using namespace mlir::LLVM; +using namespace mlir::LLVM::detail; + +namespace { +/// Helper class that keeps the state of one attribute to metadata conversion. +struct LoopAnnotationConversion { + LoopAnnotationConversion(LoopAnnotationAttr attr, + ModuleTranslation &moduleTranslation, Operation *op, + LoopAnnotationTranslation &loopAnnotationTranslation) + : attr(attr), moduleTranslation(moduleTranslation), op(op), + loopAnnotationTranslation(loopAnnotationTranslation), + ctx(moduleTranslation.getLLVMContext()) {} + + /// Converts this struct's loop annotation into a corresponding LLVMIR + /// metadata representation. + llvm::MDNode *convert(); + + /// Conversion functions for different payload attribute kinds. + void addUnitNode(StringRef name); + void addUnitNode(StringRef name, BoolAttr attr); + void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false); + void convertI32Node(StringRef name, IntegerAttr attr); + void convertFollowupNode(StringRef name, LoopAnnotationAttr attr); + + /// Conversion functions for each for each loop annotation sub-attribute. + void convertLoopOptions(LoopVectorizeAttr options); + void convertLoopOptions(LoopInterleaveAttr options); + void convertLoopOptions(LoopUnrollAttr options); + void convertLoopOptions(LoopUnrollAndJamAttr options); + void convertLoopOptions(LoopLICMAttr options); + void convertLoopOptions(LoopDistributeAttr options); + void convertLoopOptions(LoopPipelineAttr options); + + LoopAnnotationAttr attr; + ModuleTranslation &moduleTranslation; + Operation *op; + LoopAnnotationTranslation &loopAnnotationTranslation; + llvm::LLVMContext &ctx; + llvm::SmallVector metadataNodes; +}; +} // namespace + +void LoopAnnotationConversion::addUnitNode(StringRef name) { + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name)})); +} + +void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) { + if (attr && attr.getValue()) + addUnitNode(name); +} + +void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr, + bool negated) { + if (!attr) + return; + bool val = negated ^ attr.getValue(); + llvm::Constant *cstValue = llvm::ConstantInt::getBool(ctx, val); + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + +void LoopAnnotationConversion::convertI32Node(StringRef name, + IntegerAttr attr) { + if (!attr) + return; + uint32_t val = attr.getInt(); + llvm::Constant *cstValue = llvm::ConstantInt::get( + llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false); + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + +void LoopAnnotationConversion::convertFollowupNode(StringRef name, + LoopAnnotationAttr attr) { + if (!attr) + return; + + llvm::MDNode *node = loopAnnotationTranslation.translate(attr, op); + + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), node})); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) { + convertBoolNode("llvm.loop.vectorize.enable", options.getDisable(), true); + convertBoolNode("llvm.loop.vectorize.predicate.enable", + options.getPredicateEnable()); + convertBoolNode("llvm.loop.vectorize.scalable.enable", + options.getScalableEnable()); + convertI32Node("llvm.loop.vectorize.width", options.getWidth()); + convertFollowupNode("llvm.loop.vectorize.followup_vectorized", + options.getFollowupVectorized()); + convertFollowupNode("llvm.loop.vectorize.followup_epilogue", + options.getFollowupEpilogue()); + convertFollowupNode("llvm.loop.vectorize.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopInterleaveAttr options) { + convertI32Node("llvm.loop.interleave.count", options.getCount()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) { + if (auto disable = options.getDisable()) + addUnitNode(disable.getValue() ? "llvm.loop.unroll.disable" + : "llvm.loop.unroll.enable"); + convertI32Node("llvm.loop.unroll.count", options.getCount()); + convertBoolNode("llvm.loop.unroll.runtime.disable", + options.getRuntimeDisable()); + addUnitNode("llvm.loop.unroll.full", options.getFull()); + convertFollowupNode("llvm.loop.unroll.followup", options.getFollowup()); + convertFollowupNode("llvm.loop.unroll.followup_remainder", + options.getFollowupRemainder()); +} + +void LoopAnnotationConversion::convertLoopOptions( + LoopUnrollAndJamAttr options) { + if (auto disable = options.getDisable()) + addUnitNode(disable.getValue() ? "llvm.loop.unroll_and_jam.disable" + : "llvm.loop.unroll_and_jam.enable"); + convertI32Node("llvm.loop.unroll_and_jam.count", options.getCount()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_outer", + options.getFollowupOuter()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_inner", + options.getFollowupInner()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer", + options.getFollowupRemainderOuter()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner", + options.getFollowupRemainderInner()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopLICMAttr options) { + addUnitNode("llvm.licm.disable", options.getDisable()); + addUnitNode("llvm.loop.licm_versioning.disable", + options.getVersioningDisable()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopDistributeAttr options) { + convertBoolNode("llvm.loop.distribute.enable", options.getDisable(), true); + convertFollowupNode("llvm.loop.distribute.followup_coincident", + options.getFollowupCoincident()); + convertFollowupNode("llvm.loop.distribute.followup_sequential", + options.getFollowupSequential()); + convertFollowupNode("llvm.loop.distribute.followup_fallback", + options.getFollowupFallback()); + convertFollowupNode("llvm.loop.distribute.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) { + convertBoolNode("llvm.loop.pipeline.disable", options.getDisable()); + convertI32Node("llvm.loop.pipeline.initiationinterval", + options.getInitiationinterval()); +} + +llvm::MDNode *LoopAnnotationConversion::convert() { + + // Reserve operand 0 for loop id self reference. + auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); + metadataNodes.push_back(dummy.get()); + + addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced()); + addUnitNode("llvm.loop.mustprogress", attr.getMustProgress()); + + if (auto options = attr.getVectorize()) + convertLoopOptions(options); + if (auto options = attr.getInterleave()) + convertLoopOptions(options); + if (auto options = attr.getUnroll()) + convertLoopOptions(options); + if (auto options = attr.getUnrollAndJam()) + convertLoopOptions(options); + if (auto options = attr.getLicm()) + convertLoopOptions(options); + if (auto options = attr.getDistribute()) + convertLoopOptions(options); + if (auto options = attr.getPipeline()) + convertLoopOptions(options); + + ArrayRef parallelAccessGroups = attr.getParallelAccesses(); + if (!parallelAccessGroups.empty()) { + SmallVector parallelAccess; + parallelAccess.push_back( + llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); + for (SymbolRefAttr accessGroupRef : parallelAccessGroups) + parallelAccess.push_back( + moduleTranslation.getAccessGroup(*op, accessGroupRef)); + metadataNodes.push_back(llvm::MDNode::get(ctx, parallelAccess)); + } + + // Create loop options and set the first operand to itself. + llvm::MDNode *loopMD = llvm::MDNode::get(ctx, metadataNodes); + loopMD->replaceOperandWith(0, loopMD); + + return loopMD; +} + +llvm::MDNode *LoopAnnotationTranslation::translate(LoopAnnotationAttr attr, + Operation *op) { + if (!attr) + return nullptr; + + llvm::MDNode *loopMD = lookupLoopMetadata(attr); + if (loopMD) + return loopMD; + + loopMD = + LoopAnnotationConversion(attr, moduleTranslation, op, *this).convert(); + // Store a map from this Attribute to the LLVM metadata in case we + // encounter it again. + mapLoopMetadata(attr, loopMD); + return loopMD; +} diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -15,6 +15,7 @@ #include "AttrKindDetail.h" #include "DebugTranslation.h" +#include "LoopAnnotationTranslation.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h" @@ -411,6 +412,8 @@ : mlirModule(module), llvmModule(std::move(llvmModule)), debugTranslation( std::make_unique(module, *this->llvmModule)), + loopAnnotationTranslation( + std::make_unique(*this)), typeTranslator(this->llvmModule->getContext()), iface(module->getContext()) { assert(satisfiesLLVMModule(mlirModule) && @@ -1215,6 +1218,16 @@ return success(); } +void ModuleTranslation::setLoopMetadata(Operation *op, + llvm::Instruction *inst) { + auto attr = + op->getAttrOfType(LLVMDialect::getLoopAttrName()); + if (!attr) + return; + llvm::MDNode *loopMD = loopAnnotationTranslation->translate(attr, op); + inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD); +} + llvm::Type *ModuleTranslation::convertType(Type type) { return typeTranslator.translateType(type); } diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -842,7 +842,7 @@ module { llvm.func @loopOptions() { - // expected-error@below {{expected 'llvm.loop' to be a dictionary attribute}} + // expected-error@below {{expected 'llvm.loop' to be a loop annotation attribute}} llvm.br ^bb4 {llvm.loop = "test"} ^bb4: llvm.return @@ -851,32 +851,10 @@ // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'parallel_access' to be an array attribute}} - llvm.br ^bb4 {llvm.loop = {parallel_access = "loop"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{expected '"loop"' to be a symbol reference}} - llvm.br ^bb4 {llvm.loop = {parallel_access = ["loop"]}} - ^bb4: - llvm.return - } -} - -// ----- - module { llvm.func @loopOptions() { // expected-error@below {{expected '@func1' to reference a metadata op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@func1]}} + llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation} ^bb4: llvm.return } @@ -890,7 +868,7 @@ module { llvm.func @loopOptions() { // expected-error@below {{expected '@metadata' to reference an access_group op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@metadata]}} + llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation} ^bb4: llvm.return } @@ -900,49 +878,6 @@ // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'options' to be a `loopopts` attribute}} - llvm.br ^bb4 {llvm.loop = {options = "name"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{unknown loop option: name}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{loop option present twice}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { - // expected-error@below {{attribute 'access_groups' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "access_groups" = "test" } : !llvm.ptr - llvm.return - } -} - -// ----- - module { llvm.func @accessGroups(%arg0 : !llvm.ptr) { // expected-error@below {{expected '@func1' to specify a fully qualified reference}} diff --git a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir new file mode 100644 --- /dev/null +++ b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir @@ -0,0 +1,70 @@ +// RUN: mlir-opt %s | mlir-opt | FileCheck %s + +// CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +#followup = #llvm.loop_annotation + +// CHECK-DAG: #[[VECTORIZE:.*]] = #llvm.loop_vectorize +#vectorize = #llvm.loop_vectorize< + disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32, + followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[INTERLEAVE:.*]] = #llvm.loop_interleave +#interleave = #llvm.loop_interleave + +// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll +#unroll = #llvm.loop_unroll< + disable = true, count = 32 : i32, runtimeDisable = true, full = false, + followup = #followup, followupRemainder = #followup +> + +// CHECK-DAG: #[[UNROLL_AND_JAM:.*]] = #llvm.loop_unroll_and_jam +#unrollAndJam = #llvm.loop_unroll_and_jam< + disable = false, count = 16 : i32, followupOuter = #followup, followupInner = #followup, + followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[LICM:.*]] = #llvm.loop_licm +#licm = #llvm.loop_licm + +// CHECK-DAG: #[[DISTRIBUTE:.*]] = #llvm.loop_distribute +#distribute = #llvm.loop_distribute< + disable = true, followupCoincident = #followup, followupSequential = #followup, + followupFallback = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[PIPELINE:.*]] = #llvm.loop_pipeline +#pipeline = #llvm.loop_pipeline + +// CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation< +// CHECK-DAG: disableNonforced = false +// CHECK-DAG: mustProgress = true +// CHECK-DAG: unroll = #[[UNROLL]] +// CHECK-DAG: unrollAndJam = #[[UNROLL_AND_JAM]] +// CHECK-DAG: licm = #[[LICM]] +// CHECK-DAG: distribute = #[[DISTRIBUTE]] +// CHECK-DAG: pipeline = #[[PIPELINE]] +// CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2> +#loopMD = #llvm.loop_annotation + +// CHECK: llvm.func @loop_annotation +llvm.func @loop_annotation() { + // CHECK: llvm.br ^bb1 {llvm.loop = #[[LOOP_ANNOT]] + llvm.br ^bb1 {llvm.loop = #loopMD} +^bb1: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -497,22 +497,6 @@ return } -module { - // CHECK-LABEL: @loopOptions - llvm.func @loopOptions() { - // CHECK: llvm.br - // CHECK-SAME: llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - llvm.br ^bb1 {llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - ^bb1: - llvm.return - } - // CHECK: llvm.metadata @metadata attributes {test_attribute} { - llvm.metadata @metadata attributes {test_attribute} { - // CHECK: llvm.access_group @group1 - llvm.access_group @group1 - } -} - // CHECK-LABEL: llvm.func @vararg_func llvm.func @vararg_func(%arg0: i32, ...) { // CHECK: %{{.*}} = llvm.mlir.constant(1 : i32) : i32 diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1923,44 +1923,6 @@ // ----- -module { - llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { - %0 = llvm.mlir.constant(0 : i32) : i32 - %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) - llvm.br ^bb3(%0 : i32) - ^bb3(%1: i32): - %2 = llvm.icmp "slt" %1, %arg1 : i32 - // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb4: - %3 = llvm.add %1, %arg2 : i32 - // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] - %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr - // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] - llvm.br ^bb3(%3 : i32) {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb5: - llvm.return - } - - llvm.metadata @metadata { - llvm.access_group @group1 - llvm.access_group @group2 - } -} - -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[PA_NODE:[0-9]+]], ![[UNROLL_DISABLE_NODE:[0-9]+]], ![[LICM_DISABLE_NODE:[0-9]+]], ![[INTERLEAVE_NODE:[0-9]+]], ![[PIPELINE_DISABLE_NODE:[0-9]+]], ![[II_NODE:[0-9]+]]} -// CHECK: ![[PA_NODE]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} -// CHECK: ![[GROUP_NODE1]] = distinct !{} -// CHECK: ![[GROUP_NODE2]] = distinct !{} -// CHECK: ![[UNROLL_DISABLE_NODE]] = !{!"llvm.loop.unroll.disable", i1 true} -// CHECK: ![[LICM_DISABLE_NODE]] = !{!"llvm.licm.disable", i1 true} -// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 1} -// CHECK: ![[PIPELINE_DISABLE_NODE]] = !{!"llvm.loop.pipeline.disable", i1 true} -// CHECK: ![[II_NODE]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} -// CHECK: ![[ACCESS_GROUPS_NODE]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]} - -// ----- - module { llvm.func @aliasScope(%arg1 : !llvm.ptr, %arg2 : !llvm.ptr, %arg3 : !llvm.ptr) { %0 = llvm.mlir.constant(0 : i32) : i32 diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir new file mode 100644 --- /dev/null +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -0,0 +1,239 @@ +// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s + +// CHECK-LABEL: @disableNonForced +llvm.func @disableNonForced() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} + +// ----- + +// CHECK-LABEL: @mustprogress +llvm.func @mustprogress() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.mustprogress"} + +// ----- + + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @vectorizeOptions +llvm.func @vectorizeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.predicate.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.scalable.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.width", i32 16} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_vectorized", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_epilogue", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @interleaveOptions +llvm.func @interleaveOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[INTERLEAVE_NODE:[0-9]+]]} +// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 32} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @unrollOptions +llvm.func @unrollOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.count", i32 64} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.runtime.disable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_remainder", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @unrollOptions2 +llvm.func @unrollOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unroll.enable"} +// CHECK-DAG: ![[VEC_NODE2:[0-9]+]] = !{!"llvm.loop.unroll.full"} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @unrollAndJamOptions +llvm.func @unrollAndJamOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.enable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.count", i32 8} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @licmOptions +llvm.func @licmOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.licm_versioning.disable"} + +// ----- + +// CHECK-LABEL: @licmOptions2 +llvm.func @licmOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.licm.disable"} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @distributeOptions +llvm.func @distributeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_coincident", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_sequential", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_fallback", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @pipelineOptions +llvm.func @pipelineOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 false} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 1} + +// ----- + +// CHECK-LABEL: @loopOptions +llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { + %0 = llvm.mlir.constant(0 : i32) : i32 + %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) + llvm.br ^bb3(%0 : i32) + ^bb3(%1: i32): + %2 = llvm.icmp "slt" %1, %arg1 : i32 + // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = #llvm.loop_annotation< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + ^bb4: + %3 = llvm.add %1, %arg2 : i32 + // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] + %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr + // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] + llvm.br ^bb3(%3 : i32) {llvm.loop = #llvm.loop_annotation< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + + ^bb5: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[PA_NODE:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} +// CHECK-DAG: ![[GROUP_NODE1:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[GROUP_NODE2:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[UNROLL_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: ![[LICM_DISABLE_NODE:[0-9]+]] = !{!"llvm.licm.disable"} +// CHECK-DAG: ![[INTERLEAVE_NODE:[0-9]+]] = !{!"llvm.loop.interleave.count", i32 1} +// CHECK-DAG: ![[PIPELINE_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 true} +// CHECK-DAG: ![[II_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} +// CHECK-DAG: ![[ACCESS_GROUPS_NODE:[0-9]+]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]}