diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -49,38 +49,147 @@ } //===----------------------------------------------------------------------===// -// LoopOptionsAttr +// LoopMetadataAttr //===----------------------------------------------------------------------===// -def LoopOptionsAttr : LLVM_Attr<"LoopOptions", "loopopts"> { +def LoopVectorizeOptionsAttr : LLVM_Attr<"LoopVectorizeOptions", "vectorize"> { let description = [{ - This attributes encapsulates "loop options". It is means to decorate - branches that are "latches" (loop backedges) and maps to the `!llvm.loop` - metadatas: https://llvm.org/docs/LangRef.html#llvm-loop - It store the options as a pair in a sorted array and expose - APIs to retrieve the value for each option with a stronger type (bool for - example). + This attribute encapsulates vectorization specific loop options. The fields + represent the "!llvm.loop.vectorize" options. }]; let parameters = (ins - ArrayRefParameter<"std::pair", "">:$options + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$predicateEnable, + OptionalParameter<"BoolAttr">:$scalableEnable, + OptionalParameter<"IntegerAttr">:$width, + OptionalParameter<"LoopMetadataAttr">:$followupVectorized, + OptionalParameter<"LoopMetadataAttr">:$followupEpilogue, + OptionalParameter<"LoopMetadataAttr">:$followupAll ); - let extraClassDeclaration = [{ - using OptionValuePair = std::pair; - using OptionsArray = ArrayRef>; - std::optional disableUnroll(); - std::optional disableLICM(); - std::optional interleaveCount(); + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopInterleaveOptionsAttr : LLVM_Attr<"LoopInterleaveOptions", "interleave"> { + let description = [{ + This attribute encapsulates interleave specific loop options. The fields + represent the "!llvm.loop.interleave" options. }]; - let builders = [ - /// Build the LoopOptions Attribute from a sorted array of individual options. - AttrBuilder<(ins "ArrayRef>":$sortedOptions)>, - AttrBuilder<(ins "LoopOptionsAttrBuilder &":$optionBuilders)> - ]; - let hasCustomAssemblyFormat = 1; - let skipDefaultBuilders = 1; + let parameters = (ins + "IntegerAttr":$count + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollOptionsAttr : LLVM_Attr<"LoopUnrollOptions", "unroll"> { + let description = [{ + This attribute encapsulates unroll specific loop options. The fields + represent the "!llvm.loop.unroll" options. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"BoolAttr">:$runtimeDisable, + OptionalParameter<"BoolAttr">:$full, + OptionalParameter<"LoopMetadataAttr">:$followup, + OptionalParameter<"LoopMetadataAttr">:$followupRemainder + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollAndJamOptionsAttr : LLVM_Attr<"LoopUnrollAndJamOptions", "unroll_and_jam"> { + let description = [{ + This attribute encapsulates unroll and jam specific loop options. The fields + represent the "!llvm.loop.unroll_and_jam" options. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"LoopMetadataAttr">:$followupOuter, + OptionalParameter<"LoopMetadataAttr">:$followupInner, + OptionalParameter<"LoopMetadataAttr">:$followupRemainderOuter, + OptionalParameter<"LoopMetadataAttr">:$followupRemainderInner, + OptionalParameter<"LoopMetadataAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopLICMOptionsAttr : LLVM_Attr<"LoopLICMOptions", "licm"> { + let description = [{ + This attribute encapsulates loop invariant code motion (licm) specific loop options. + The fields correspond to the "!llvm.licm.disable" and the + "!llvm.loop.licm_versioning.disable" options. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$versioningDisable + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopDistributeOptionsAttr : LLVM_Attr<"LoopDistributeOptions", "distribute"> { + let description = [{ + This attribute encapsulates distribute specific loop options. The fields + represent the "!llvm.loop.distribute" options. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"LoopMetadataAttr">:$followupCoincident, + OptionalParameter<"LoopMetadataAttr">:$followupSequential, + OptionalParameter<"LoopMetadataAttr">:$followupFallback, + OptionalParameter<"LoopMetadataAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopPipelineOptionsAttr : LLVM_Attr<"LoopPipelineOptions", "pipeline"> { + let description = [{ + This attribute encapsulates pipeline specific loop options. The fields + represent the "!llvm.loop.pipeline" options. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$initiationinterval + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopMetadataAttr : LLVM_Attr<"LoopMetadata", "loop_md"> { + let description = [{ + This attributes encapsulates "loop metadata". It is meant to decorate + branches that are "latches" (loop backedges) and maps to the `!llvm.loop` + metadatas: https://llvm.org/docs/LangRef.html#llvm-loop + It stores options in attribute parameters and groups related options in + nested attributes to provide structured access. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disableNonforced, + OptionalParameter<"LoopVectorizeOptionsAttr">:$vectorize, + OptionalParameter<"LoopInterleaveOptionsAttr">:$interleave, + OptionalParameter<"LoopUnrollOptionsAttr">:$unroll, + OptionalParameter<"LoopUnrollAndJamOptionsAttr">:$unrollAndJam, + OptionalParameter<"LoopLICMOptionsAttr">:$licm, + OptionalParameter<"LoopDistributeOptionsAttr">:$distribute, + OptionalParameter<"LoopPipelineOptionsAttr">:$pipeline, + OptionalParameter<"BoolAttr">:$mustProgress, + OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses + ); + + let assemblyFormat = "`<` struct(params) `>`"; } //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h @@ -22,7 +22,6 @@ namespace mlir { namespace LLVM { -class LoopOptionsAttrBuilder; /// This class represents the base attribute for all debug info attributes. class DINodeAttr : public Attribute { @@ -74,54 +73,4 @@ #define GET_ATTRDEF_CLASSES #include "mlir/Dialect/LLVMIR/LLVMOpsAttrDefs.h.inc" -namespace mlir { -namespace LLVM { - -/// Builder class for LoopOptionsAttr. This helper class allows to progressively -/// build a LoopOptionsAttr one option at a time, and pay the price of attribute -/// creation once all the options are in place. -class LoopOptionsAttrBuilder { -public: - /// Construct a empty builder. - LoopOptionsAttrBuilder() = default; - - /// Construct a builder with an initial list of options from an existing - /// LoopOptionsAttr. - LoopOptionsAttrBuilder(LoopOptionsAttr attr); - - /// Set the `disable_licm` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableLICM(std::optional value); - - /// Set the `interleave_count` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setInterleaveCount(std::optional count); - - /// Set the `disable_unroll` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableUnroll(std::optional value); - - /// Set the `disable_pipeline` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisablePipeline(std::optional value); - - /// Set the `pipeline_initiation_interval` option to the provided value. - /// If no value is provided the option is deleted. - LoopOptionsAttrBuilder & - setPipelineInitiationInterval(std::optional count); - - /// Returns true if any option has been set. - bool empty() { return options.empty(); } - -private: - template - LoopOptionsAttrBuilder &setOption(LoopOptionCase tag, std::optional value); - - friend class LoopOptionsAttr; - SmallVector options; -}; - -} // namespace LLVM -} // namespace mlir - #endif // MLIR_DIALECT_LLVMIR_LLVMATTRS_H_ diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -489,25 +489,6 @@ "::mlir::LLVM::LinkageAttr::get($_builder.getContext(), $0)"; } -//===----------------------------------------------------------------------===// -// LoopOptions -//===----------------------------------------------------------------------===// - -def LOptDisableUnroll : I32EnumAttrCase<"disable_unroll", 1>; -def LOptDisableLICM : I32EnumAttrCase<"disable_licm", 2>; -def LOptInterleaveCount : I32EnumAttrCase<"interleave_count", 3>; -def LOptDisablePipeline : I32EnumAttrCase<"disable_pipeline", 4>; -def LOptPipelineInitiationInterval : I32EnumAttrCase<"pipeline_initiation_interval", 5>; - -def LoopOptionCase : I32EnumAttr< - "LoopOptionCase", - "LLVM loop option", - [LOptDisableUnroll, LOptDisableLICM, LOptInterleaveCount, - LOptDisablePipeline, LOptPipelineInitiationInterval - ]> { - let cppNamespace = "::mlir::LLVM"; -} - //===----------------------------------------------------------------------===// // UnnamedAddr //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -39,8 +39,6 @@ static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } static StringRef getAliasScopesAttrName() { return "alias_scopes"; } static StringRef getLoopAttrName() { return "llvm.loop"; } - static StringRef getParallelAccessAttrName() { return "parallel_access"; } - static StringRef getLoopOptionsAttrName() { return "options"; } static StringRef getAccessGroupsAttrName() { return "access_groups"; } static StringRef getStructAttrsAttrName() { return "llvm.struct_attrs"; } static StringRef getTBAAAttrName() { return "llvm.tbaa"; } diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -129,14 +129,13 @@ llvm::MDNode *getAliasScope(Operation &opInst, SymbolRefAttr aliasScopeRef) const; - /// Returns the LLVM metadata corresponding to a llvm loop's codegen - /// options attribute. - llvm::MDNode *lookupLoopOptionsMetadata(Attribute options) const { - return loopOptionsMetadataMapping.lookup(options); + /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. + llvm::MDNode *lookupLoopMetadata(Attribute options) const { + return loopMetadataMapping.lookup(options); } - void mapLoopOptionsMetadata(Attribute options, llvm::MDNode *metadata) { - auto result = loopOptionsMetadataMapping.try_emplace(options, metadata); + void mapLoopMetadata(Attribute options, llvm::MDNode *metadata) { + auto result = loopMetadataMapping.try_emplace(options, metadata); (void)result; assert(result.second && "attempting to map loop options that was already mapped"); @@ -343,10 +342,9 @@ /// identified via their branches) and contained memory accesses. DenseMap accessGroupMetadataMapping; - /// Mapping from an attribute describing loop codegen options to its LLVM - /// metadata. The metadata is attached to Latch block branches with this - /// attribute. - DenseMap loopOptionsMetadataMapping; + /// Mapping from an attribute describing loop metadata to its LLVM metadata. + /// The metadata is attached to Latch block branches with this attribute. + DenseMap loopMetadataMapping; /// Mapping from an alias scope metadata operation to its LLVM metadata. /// This map is populated on module entry. diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp @@ -92,176 +92,6 @@ return success(); } -//===----------------------------------------------------------------------===// -// LoopOptionsAttrBuilder -//===----------------------------------------------------------------------===// - -LoopOptionsAttrBuilder::LoopOptionsAttrBuilder(LoopOptionsAttr attr) - : options(attr.getOptions().begin(), attr.getOptions().end()) {} - -template -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setOption(LoopOptionCase tag, std::optional value) { - auto option = llvm::find_if( - options, [tag](auto option) { return option.first == tag; }); - if (option != options.end()) { - if (value) - option->second = *value; - else - options.erase(option); - } else { - options.push_back(LoopOptionsAttr::OptionValuePair(tag, *value)); - } - return *this; -} - -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableLICM(std::optional value) { - return setOption(LoopOptionCase::disable_licm, value); -} - -/// Set the `interleave_count` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setInterleaveCount(std::optional count) { - return setOption(LoopOptionCase::interleave_count, count); -} - -/// Set the `disable_unroll` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableUnroll(std::optional value) { - return setOption(LoopOptionCase::disable_unroll, value); -} - -/// Set the `disable_pipeline` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisablePipeline(std::optional value) { - return setOption(LoopOptionCase::disable_pipeline, value); -} - -/// Set the `pipeline_initiation_interval` option to the provided value. -/// If no value is provided the option is deleted. -LoopOptionsAttrBuilder &LoopOptionsAttrBuilder::setPipelineInitiationInterval( - std::optional count) { - return setOption(LoopOptionCase::pipeline_initiation_interval, count); -} - -//===----------------------------------------------------------------------===// -// LoopOptionsAttr -//===----------------------------------------------------------------------===// - -template -static std::optional -getOption(ArrayRef> options, - LoopOptionCase option) { - auto it = - lower_bound(options, option, [](auto optionPair, LoopOptionCase option) { - return optionPair.first < option; - }); - if (it == options.end()) - return {}; - return static_cast(it->second); -} - -std::optional LoopOptionsAttr::disableUnroll() { - return getOption(getOptions(), LoopOptionCase::disable_unroll); -} - -std::optional LoopOptionsAttr::disableLICM() { - return getOption(getOptions(), LoopOptionCase::disable_licm); -} - -std::optional LoopOptionsAttr::interleaveCount() { - return getOption(getOptions(), LoopOptionCase::interleave_count); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get( - MLIRContext *context, - ArrayRef> sortedOptions) { - assert(llvm::is_sorted(sortedOptions, llvm::less_first()) && - "LoopOptionsAttr ctor expects a sorted options array"); - return Base::get(context, sortedOptions); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get(MLIRContext *context, - LoopOptionsAttrBuilder &optionBuilders) { - llvm::sort(optionBuilders.options, llvm::less_first()); - return Base::get(context, optionBuilders.options); -} - -void LoopOptionsAttr::print(AsmPrinter &printer) const { - printer << "<"; - llvm::interleaveComma(getOptions(), printer, [&](auto option) { - printer << stringifyEnum(option.first) << " = "; - switch (option.first) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - printer << (option.second ? "true" : "false"); - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - printer << option.second; - break; - } - }); - printer << ">"; -} - -Attribute LoopOptionsAttr::parse(AsmParser &parser, Type type) { - if (failed(parser.parseLess())) - return {}; - - SmallVector> options; - llvm::SmallDenseSet seenOptions; - auto parseLoopOptions = [&]() -> ParseResult { - StringRef optionName; - if (parser.parseKeyword(&optionName)) - return failure(); - - auto option = symbolizeLoopOptionCase(optionName); - if (!option) - return parser.emitError(parser.getNameLoc(), "unknown loop option: ") - << optionName; - if (!seenOptions.insert(*option).second) - return parser.emitError(parser.getNameLoc(), "loop option present twice"); - if (failed(parser.parseEqual())) - return failure(); - - int64_t value; - switch (*option) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - if (succeeded(parser.parseOptionalKeyword("true"))) - value = 1; - else if (succeeded(parser.parseOptionalKeyword("false"))) - value = 0; - else { - return parser.emitError(parser.getNameLoc(), - "expected boolean value 'true' or 'false'"); - } - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - if (failed(parser.parseInteger(value))) - return parser.emitError(parser.getNameLoc(), "expected integer value"); - break; - } - options.push_back(std::make_pair(*option, value)); - return success(); - }; - if (parser.parseCommaSeparatedList(parseLoopOptions) || parser.parseGreater()) - return {}; - - llvm::sort(options, llvm::less_first()); - return get(parser.getContext(), options); -} - //===----------------------------------------------------------------------===// // MemoryEffectsAttr //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2713,7 +2713,11 @@ .Case([&](auto attr) { + DISubprogramAttr, DISubroutineTypeAttr, LoopMetadataAttr, + LoopVectorizeOptionsAttr, LoopInterleaveOptionsAttr, + LoopUnrollOptionsAttr, LoopUnrollAndJamOptionsAttr, + LoopLICMOptionsAttr, LoopDistributeOptionsAttr, + LoopPipelineOptionsAttr>([&](auto attr) { os << decltype(attr)::getMnemonic(); return AliasResult::OverridableAlias; }) @@ -2991,45 +2995,27 @@ // If the `llvm.loop` attribute is present, enforce the following structure, // which the module translation can assume. if (attr.getName() == LLVMDialect::getLoopAttrName()) { - auto loopAttr = attr.getValue().dyn_cast(); + auto loopAttr = attr.getValue().dyn_cast(); if (!loopAttr) return op->emitOpError() << "expected '" << LLVMDialect::getLoopAttrName() - << "' to be a dictionary attribute"; - std::optional parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - auto accessGroups = parallelAccessGroup->getValue().dyn_cast(); - if (!accessGroups) - return op->emitOpError() - << "expected '" << LLVMDialect::getParallelAccessAttrName() - << "' to be an array attribute"; - for (Attribute attr : accessGroups) { - auto accessGroupRef = attr.dyn_cast(); - if (!accessGroupRef) - return op->emitOpError() - << "expected '" << attr << "' to be a symbol reference"; - StringAttr metadataName = accessGroupRef.getRootReference(); - auto metadataOp = - SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << attr << "' to reference a metadata op"; - StringAttr accessGroupName = accessGroupRef.getLeafReference(); - Operation *accessGroupOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); - if (!accessGroupOp) - return op->emitOpError() - << "expected '" << attr << "' to reference an access_group op"; - } + << "' to be a loop metadata attribute"; + ArrayRef parallelAccesses = loopAttr.getParallelAccesses(); + if (parallelAccesses.empty()) + return success(); + for (SymbolRefAttr accessGroupRef : parallelAccesses) { + StringAttr metadataName = accessGroupRef.getRootReference(); + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference a metadata op"; + StringAttr accessGroupName = accessGroupRef.getLeafReference(); + Operation *accessGroupOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); + if (!accessGroupOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference an access_group op"; } - - std::optional loopOptions = - loopAttr.getNamed(LLVMDialect::getLoopOptionsAttrName()); - if (loopOptions && !loopOptions->getValue().isa()) - return op->emitOpError() - << "expected '" << LLVMDialect::getLoopOptionsAttrName() - << "' to be a `loopopts` attribute"; } if (attr.getName() == LLVMDialect::getStructAttrsAttrName()) { diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -167,88 +167,246 @@ return ret; } -/// Returns an LLVM metadata node corresponding to a loop option. This metadata -/// is attached to an llvm.loop node. -static llvm::MDNode *getLoopOptionMetadata(llvm::LLVMContext &ctx, - LoopOptionCase option, - int64_t value) { - StringRef name; - llvm::Constant *cstValue = nullptr; - switch (option) { - case LoopOptionCase::disable_licm: - name = "llvm.licm.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::disable_unroll: - name = "llvm.loop.unroll.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::interleave_count: - name = "llvm.loop.interleave.count"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; - case LoopOptionCase::disable_pipeline: - name = "llvm.loop.pipeline.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::pipeline_initiation_interval: - name = "llvm.loop.pipeline.initiationinterval"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; +namespace { +struct LoopMDConverter { + LoopMDConverter(LoopMetadataAttr attr, llvm::LLVMContext &ctx, + LLVM::ModuleTranslation &moduleTranslation, Operation &opInst) + : attr(attr), ctx(ctx), moduleTranslation(moduleTranslation), + opInst(opInst) {} + llvm::MDNode *convert(); + +private: + void createAndAddUnitNode(StringRef name); + void createAndAddBoolNode(StringRef name, bool val); + void createAndAddI32Node(StringRef name, uint32_t val); + void createAndAddNestedNode(StringRef name, llvm::MDNode *node); + + llvm::MDNode *convertFollowup(LoopMetadataAttr loopMD); + void convertLoopOptions(LoopVectorizeOptionsAttr options); + void convertLoopOptions(LoopInterleaveOptionsAttr options); + void convertLoopOptions(LoopUnrollOptionsAttr options); + void convertLoopOptions(LoopUnrollAndJamOptionsAttr options); + void convertLoopOptions(LoopLICMOptionsAttr options); + void convertLoopOptions(LoopDistributeOptionsAttr options); + void convertLoopOptions(LoopPipelineOptionsAttr options); + void convertPassthrough(DictionaryAttr passthrough); + + LoopMetadataAttr attr; + llvm::LLVMContext &ctx; + LLVM::ModuleTranslation &moduleTranslation; + Operation &opInst; + llvm::SmallVector mdNodes; +}; +} // namespace + +void LoopMDConverter::createAndAddUnitNode(StringRef name) { + mdNodes.push_back(llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name)})); +} + +void LoopMDConverter::createAndAddBoolNode(StringRef name, bool val) { + llvm::Constant *cstValue = llvm::ConstantInt::getBool(ctx, val); + mdNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + +void LoopMDConverter::createAndAddI32Node(StringRef name, uint32_t val) { + llvm::Constant *cstValue = llvm::ConstantInt::get( + llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false); + mdNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} +void LoopMDConverter::createAndAddNestedNode(StringRef name, + llvm::MDNode *node) { + mdNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), node})); +} + +llvm::MDNode *LoopMDConverter::convertFollowup(LoopMetadataAttr loopMD) { + return LoopMDConverter(loopMD, ctx, moduleTranslation, opInst).convert(); +} + +void LoopMDConverter::convertLoopOptions(LoopVectorizeOptionsAttr options) { + if (auto disable = options.getDisable()) + createAndAddBoolNode("llvm.loop.vectorize.enable", !disable.getValue()); + if (auto predicateEnable = options.getPredicateEnable()) + createAndAddBoolNode("llvm.loop.vectorize.predicate.enable", + predicateEnable.getValue()); + if (auto scalableEnable = options.getScalableEnable()) + createAndAddBoolNode("llvm.loop.vectorize.scalable.enable", + scalableEnable.getValue()); + if (auto width = options.getWidth()) + createAndAddI32Node("llvm.loop.vectorize.width", width.getInt()); + if (auto followupVec = options.getFollowupVectorized()) + createAndAddNestedNode("llvm.loop.vectorize.followup_vectorized", + convertFollowup(followupVec)); + if (auto followupEpi = options.getFollowupEpilogue()) + createAndAddNestedNode("llvm.loop.vectorize.followup_epilogue", + convertFollowup(followupEpi)); + if (auto followupAll = options.getFollowupAll()) + createAndAddNestedNode("llvm.loop.vectorize.followup_all", + convertFollowup(followupAll)); +} + +void LoopMDConverter::convertLoopOptions(LoopInterleaveOptionsAttr options) { + createAndAddI32Node("llvm.loop.interleave.count", + options.getCount().getInt()); +} + +void LoopMDConverter::convertLoopOptions(LoopUnrollOptionsAttr options) { + if (auto disable = options.getDisable()) { + StringRef name; + if (disable.getValue()) + name = "llvm.loop.unroll.disable"; + else + name = "llvm.loop.unroll.enable"; + createAndAddUnitNode(name); } - return llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), - llvm::ConstantAsMetadata::get(cstValue)}); + if (auto count = options.getCount()) + createAndAddI32Node("llvm.loop.unroll.count", count.getInt()); + if (auto runtimeDisable = options.getRuntimeDisable()) + createAndAddBoolNode("llvm.loop.unroll.runtime.disable", + runtimeDisable.getValue()); + if (auto full = options.getFull()) + if (full.getValue()) + createAndAddUnitNode("llvm.loop.unroll.full"); + if (auto followup = options.getFollowup()) + createAndAddNestedNode("llvm.loop.unroll.followup", + convertFollowup(followup)); + if (auto followupRem = options.getFollowupRemainder()) + createAndAddNestedNode("llvm.loop.unroll.followup_remainder", + convertFollowup(followupRem)); } -static void setLoopMetadata(Operation &opInst, llvm::Instruction &llvmInst, - llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation) { - if (Attribute attr = opInst.getAttr(LLVMDialect::getLoopAttrName())) { - llvm::Module *module = builder.GetInsertBlock()->getModule(); - llvm::MDNode *loopMD = moduleTranslation.lookupLoopOptionsMetadata(attr); - if (!loopMD) { - llvm::LLVMContext &ctx = module->getContext(); - - SmallVector loopOptions; - // Reserve operand 0 for loop id self reference. - auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); - loopOptions.push_back(dummy.get()); - - auto loopAttr = attr.cast(); - auto parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - SmallVector parallelAccess; - parallelAccess.push_back( - llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); - for (SymbolRefAttr accessGroupRef : parallelAccessGroup->getValue() - .cast() - .getAsRange()) - parallelAccess.push_back( - moduleTranslation.getAccessGroup(opInst, accessGroupRef)); - loopOptions.push_back(llvm::MDNode::get(ctx, parallelAccess)); - } +void LoopMDConverter::convertLoopOptions(LoopUnrollAndJamOptionsAttr options) { + if (auto disable = options.getDisable()) { + StringRef name; + if (disable.getValue()) + name = "llvm.loop.unroll_and_jam.disable"; + else + name = "llvm.loop.unroll_and_jam.enable"; + createAndAddUnitNode(name); + } + if (auto count = options.getCount()) + createAndAddI32Node("llvm.loop.unroll_and_jam.count", count.getInt()); + if (auto followupOut = options.getFollowupOuter()) + createAndAddNestedNode("llvm.loop.unroll_and_jam.followup_outer", + convertFollowup(followupOut)); + if (auto followupIn = options.getFollowupInner()) + createAndAddNestedNode("llvm.loop.unroll_and_jam.followup_inner", + convertFollowup(followupIn)); + if (auto followupRemOut = options.getFollowupRemainderOuter()) + createAndAddNestedNode("llvm.loop.unroll_and_jam.followup_remainder_outer", + convertFollowup(followupRemOut)); + if (auto followupRemIn = options.getFollowupRemainderInner()) + createAndAddNestedNode("llvm.loop.unroll_and_jam.followup_remainder_inner", + convertFollowup(followupRemIn)); + if (auto followupAll = options.getFollowupAll()) + createAndAddNestedNode("llvm.loop.unroll_and_jam.followup_all", + convertFollowup(followupAll)); +} - if (auto loopOptionsAttr = loopAttr.getAs( - LLVMDialect::getLoopOptionsAttrName())) { - for (auto option : loopOptionsAttr.getOptions()) - loopOptions.push_back( - getLoopOptionMetadata(ctx, option.first, option.second)); - } +void LoopMDConverter::convertLoopOptions(LoopLICMOptionsAttr options) { + if (auto disable = options.getDisable()) + if (disable.getValue()) + createAndAddUnitNode("llvm.licm.disable"); - // Create loop options and set the first operand to itself. - loopMD = llvm::MDNode::get(ctx, loopOptions); - loopMD->replaceOperandWith(0, loopMD); + if (auto versioningDisable = options.getVersioningDisable()) + if (versioningDisable.getValue()) + createAndAddUnitNode("llvm.loop.licm_versioning.disable"); +} - // Store a map from this Attribute to the LLVM metadata in case we - // encounter it again. - moduleTranslation.mapLoopOptionsMetadata(attr, loopMD); - } +void LoopMDConverter::convertLoopOptions(LoopDistributeOptionsAttr options) { + if (auto disable = options.getDisable()) + createAndAddBoolNode("llvm.loop.distribute.enable", !disable.getValue()); + if (auto followupCoi = options.getFollowupCoincident()) + createAndAddNestedNode("llvm.loop.distribute.followup_coincident", + convertFollowup(followupCoi)); + if (auto followupSeq = options.getFollowupSequential()) + createAndAddNestedNode("llvm.loop.distribute.followup_sequential", + convertFollowup(followupSeq)); + if (auto followupFb = options.getFollowupFallback()) + createAndAddNestedNode("llvm.loop.distribute.followup_fallback", + convertFollowup(followupFb)); + if (auto followupAll = options.getFollowupAll()) + createAndAddNestedNode("llvm.loop.distribute.followup_all", + convertFollowup(followupAll)); +} + +void LoopMDConverter::convertLoopOptions(LoopPipelineOptionsAttr options) { + if (auto disable = options.getDisable()) + createAndAddBoolNode("llvm.loop.pipeline.disable", disable.getValue()); + if (auto initiationinterval = options.getInitiationinterval()) + createAndAddI32Node("llvm.loop.pipeline.initiationinterval", + initiationinterval.getInt()); +} - llvmInst.setMetadata(module->getMDKindID("llvm.loop"), loopMD); +llvm::MDNode *LoopMDConverter::convert() { + llvm::MDNode *loopMD = moduleTranslation.lookupLoopMetadata(attr); + if (loopMD) + return loopMD; + + // Reserve operand 0 for loop id self reference. + auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); + mdNodes.push_back(dummy.get()); + + if (auto disableNonforced = attr.getDisableNonforced()) + if (disableNonforced.getValue()) + createAndAddUnitNode("llvm.loop.disable_nonforced"); + if (auto mustProgress = attr.getMustProgress()) + if (mustProgress.getValue()) + createAndAddUnitNode("llvm.loop.mustprogress"); + + if (auto options = attr.getVectorize()) + convertLoopOptions(options); + if (auto options = attr.getInterleave()) + convertLoopOptions(options); + if (auto options = attr.getUnroll()) + convertLoopOptions(options); + if (auto options = attr.getUnrollAndJam()) + convertLoopOptions(options); + if (auto options = attr.getLicm()) + convertLoopOptions(options); + if (auto options = attr.getDistribute()) + convertLoopOptions(options); + if (auto options = attr.getPipeline()) + convertLoopOptions(options); + + ArrayRef parallelAccessGroups = attr.getParallelAccesses(); + if (!parallelAccessGroups.empty()) { + SmallVector parallelAccess; + parallelAccess.push_back( + llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); + for (SymbolRefAttr accessGroupRef : parallelAccessGroups) + parallelAccess.push_back( + moduleTranslation.getAccessGroup(opInst, accessGroupRef)); + mdNodes.push_back(llvm::MDNode::get(ctx, parallelAccess)); } + + // Create loop options and set the first operand to itself. + loopMD = llvm::MDNode::get(ctx, mdNodes); + loopMD->replaceOperandWith(0, loopMD); + + // Store a map from this Attribute to the LLVM metadata in case we + // encounter it again. + moduleTranslation.mapLoopMetadata(attr, loopMD); + return loopMD; +} + +static void setLoopMetadata(Operation &opInst, llvm::Instruction &llvmInst, + llvm::IRBuilderBase &builder, + LLVM::ModuleTranslation &moduleTranslation) { + llvm::Module *module = builder.GetInsertBlock()->getModule(); + llvm::LLVMContext &ctx = module->getContext(); + auto attr = + opInst.getAttrOfType(LLVMDialect::getLoopAttrName()); + if (!attr) + return; + LoopMDConverter loopMDConverter(attr, ctx, moduleTranslation, opInst); + llvm::MDNode *loopMD = + LoopMDConverter(attr, ctx, moduleTranslation, opInst).convert(); + llvmInst.setMetadata(llvm::LLVMContext::MD_loop, loopMD); } /// Convert the value of a DenseI64ArrayAttr to a vector of unsigned indices. diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -842,7 +842,7 @@ module { llvm.func @loopOptions() { - // expected-error@below {{expected 'llvm.loop' to be a dictionary attribute}} + // expected-error@below {{expected 'llvm.loop' to be a loop metadata attribute}} llvm.br ^bb4 {llvm.loop = "test"} ^bb4: llvm.return @@ -851,32 +851,10 @@ // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'parallel_access' to be an array attribute}} - llvm.br ^bb4 {llvm.loop = {parallel_access = "loop"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{expected '"loop"' to be a symbol reference}} - llvm.br ^bb4 {llvm.loop = {parallel_access = ["loop"]}} - ^bb4: - llvm.return - } -} - -// ----- - module { llvm.func @loopOptions() { // expected-error@below {{expected '@func1' to reference a metadata op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@func1]}} + llvm.br ^bb4 {llvm.loop = #llvm.loop_md} ^bb4: llvm.return } @@ -890,7 +868,7 @@ module { llvm.func @loopOptions() { // expected-error@below {{expected '@metadata' to reference an access_group op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@metadata]}} + llvm.br ^bb4 {llvm.loop = #llvm.loop_md} ^bb4: llvm.return } @@ -900,49 +878,6 @@ // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'options' to be a `loopopts` attribute}} - llvm.br ^bb4 {llvm.loop = {options = "name"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{unknown loop option: name}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{loop option present twice}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { - // expected-error@below {{attribute 'access_groups' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "access_groups" = "test" } : !llvm.ptr - llvm.return - } -} - -// ----- - module { llvm.func @accessGroups(%arg0 : !llvm.ptr) { // expected-error@below {{expected '@func1' to specify a fully qualified reference}} diff --git a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir new file mode 100644 --- /dev/null +++ b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir @@ -0,0 +1,70 @@ +// RUN: mlir-opt %s | mlir-opt | FileCheck %s + +// CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_md +#followup = #llvm.loop_md + +// CHECK-DAG: #[[VECTORIZE:.*]] = #llvm.vectorize +#vectorize = #llvm.vectorize< + disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32, + followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[INTERLEAVE:.*]] = #llvm.interleave +#interleave = #llvm.interleave + +// CHECK-DAG: #[[UNROLL:.*]] = #llvm.unroll +#unroll = #llvm.unroll< + disable = true, count = 32 : i32, runtimeDisable = true, full = false, + followup = #followup, followupRemainder = #followup +> + +// CHECK-DAG: #[[UNROLL_AND_JAM:.*]] = #llvm.unroll_and_jam +#unrollAndJam = #llvm.unroll_and_jam< + disable = false, count = 16 : i32, followupOuter = #followup, followupInner = #followup, + followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[LICM:.*]] = #llvm.licm +#licm = #llvm.licm + +// CHECK-DAG: #[[DISTRIBUTE:.*]] = #llvm.distribute +#distribute = #llvm.distribute< + disable = true, followupCoincident = #followup, followupSequential = #followup, + followupFallback = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[PIPELINE:.*]] = #llvm.pipeline +#pipeline = #llvm.pipeline + +// CHECK: #[[LOOP_MD:.*]] = #llvm.loop_md< +// CHECK-DAG: disableNonforced = false +// CHECK-DAG: mustProgress = true +// CHECK-DAG: unroll = #[[UNROLL]] +// CHECK-DAG: unrollAndJam = #[[UNROLL_AND_JAM]] +// CHECK-DAG: licm = #[[LICM]] +// CHECK-DAG: distribute = #[[DISTRIBUTE]] +// CHECK-DAG: pipeline = #[[PIPELINE]] +// CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2> +#loopMD = #llvm.loop_md + +// CHECK: llvm.func @loopMetadata +llvm.func @loopMetadata() { + // CHECK: llvm.br ^bb1 {llvm.loop = #[[LOOP_MD]] + llvm.br ^bb1 {llvm.loop = #loopMD} +^bb1: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -497,22 +497,6 @@ return } -module { - // CHECK-LABEL: @loopOptions - llvm.func @loopOptions() { - // CHECK: llvm.br - // CHECK-SAME: llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - llvm.br ^bb1 {llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - ^bb1: - llvm.return - } - // CHECK: llvm.metadata @metadata attributes {test_attribute} { - llvm.metadata @metadata attributes {test_attribute} { - // CHECK: llvm.access_group @group1 - llvm.access_group @group1 - } -} - // CHECK-LABEL: llvm.func @vararg_func llvm.func @vararg_func(%arg0: i32, ...) { // CHECK: %{{.*}} = llvm.mlir.constant(1 : i32) : i32 diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1923,44 +1923,6 @@ // ----- -module { - llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { - %0 = llvm.mlir.constant(0 : i32) : i32 - %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) - llvm.br ^bb3(%0 : i32) - ^bb3(%1: i32): - %2 = llvm.icmp "slt" %1, %arg1 : i32 - // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb4: - %3 = llvm.add %1, %arg2 : i32 - // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] - %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr - // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] - llvm.br ^bb3(%3 : i32) {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb5: - llvm.return - } - - llvm.metadata @metadata { - llvm.access_group @group1 - llvm.access_group @group2 - } -} - -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[PA_NODE:[0-9]+]], ![[UNROLL_DISABLE_NODE:[0-9]+]], ![[LICM_DISABLE_NODE:[0-9]+]], ![[INTERLEAVE_NODE:[0-9]+]], ![[PIPELINE_DISABLE_NODE:[0-9]+]], ![[II_NODE:[0-9]+]]} -// CHECK: ![[PA_NODE]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} -// CHECK: ![[GROUP_NODE1]] = distinct !{} -// CHECK: ![[GROUP_NODE2]] = distinct !{} -// CHECK: ![[UNROLL_DISABLE_NODE]] = !{!"llvm.loop.unroll.disable", i1 true} -// CHECK: ![[LICM_DISABLE_NODE]] = !{!"llvm.licm.disable", i1 true} -// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 1} -// CHECK: ![[PIPELINE_DISABLE_NODE]] = !{!"llvm.loop.pipeline.disable", i1 true} -// CHECK: ![[II_NODE]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} -// CHECK: ![[ACCESS_GROUPS_NODE]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]} - -// ----- - module { llvm.func @aliasScope(%arg1 : !llvm.ptr, %arg2 : !llvm.ptr, %arg3 : !llvm.ptr) { %0 = llvm.mlir.constant(0 : i32) : i32 diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir new file mode 100644 --- /dev/null +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -0,0 +1,239 @@ +// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s + +// CHECK-LABEL: @disableNonForced +llvm.func @disableNonForced() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} + +// ----- + +// CHECK-LABEL: @mustprogress +llvm.func @mustprogress() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.mustprogress"} + +// ----- + + +#followup = #llvm.loop_md + +// CHECK-LABEL: @vectorizeOptions +llvm.func @vectorizeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.predicate.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.scalable.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.width", i32 16} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_vectorized", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_epilogue", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @interleaveOptions +llvm.func @interleaveOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[INTERLEAVE_NODE:[0-9]+]]} +// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 32} + +// ----- + +#followup = #llvm.loop_md + +// CHECK-LABEL: @unrollOptions +llvm.func @unrollOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.count", i32 64} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.runtime.disable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_remainder", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @unrollOptions2 +llvm.func @unrollOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unroll.enable"} +// CHECK-DAG: ![[VEC_NODE2:[0-9]+]] = !{!"llvm.loop.unroll.full"} + +// ----- + +#followup = #llvm.loop_md + +// CHECK-LABEL: @unrollAndJamOptions +llvm.func @unrollAndJamOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.enable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.count", i32 8} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @licmOptions +llvm.func @licmOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.licm_versioning.disable"} + +// ----- + +// CHECK-LABEL: @licmOptions2 +llvm.func @licmOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.licm.disable"} + +// ----- + +#followup = #llvm.loop_md + +// CHECK-LABEL: @distributeOptions +llvm.func @distributeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_coincident", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_sequential", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_fallback", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @pipelineOptions +llvm.func @pipelineOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {llvm.loop = #llvm.loop_md>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 false} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 1} + +// ----- + +// CHECK-LABEL: @loopOptions +llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { + %0 = llvm.mlir.constant(0 : i32) : i32 + %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) + llvm.br ^bb3(%0 : i32) + ^bb3(%1: i32): + %2 = llvm.icmp "slt" %1, %arg1 : i32 + // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = #llvm.loop_md< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + ^bb4: + %3 = llvm.add %1, %arg2 : i32 + // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] + %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr + // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] + llvm.br ^bb3(%3 : i32) {llvm.loop = #llvm.loop_md< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + + ^bb5: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[PA_NODE:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} +// CHECK-DAG: ![[GROUP_NODE1:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[GROUP_NODE2:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[UNROLL_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: ![[LICM_DISABLE_NODE:[0-9]+]] = !{!"llvm.licm.disable"} +// CHECK-DAG: ![[INTERLEAVE_NODE:[0-9]+]] = !{!"llvm.loop.interleave.count", i32 1} +// CHECK-DAG: ![[PIPELINE_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 true} +// CHECK-DAG: ![[II_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} +// CHECK-DAG: ![[ACCESS_GROUPS_NODE:[0-9]+]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]}