diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h @@ -15,6 +15,7 @@ #define MLIR_DIALECT_LLVMIR_LLVMDIALECT_H_ #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/Dialect.h" @@ -34,8 +35,6 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" -#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h.inc" - namespace llvm { class Type; class LLVMContext; diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h new file mode 100644 --- /dev/null +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h @@ -0,0 +1,36 @@ +//===- LLVMInterfaces.h - LLVM Interfaces -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines op interfaces for the LLVM dialect in MLIR. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_LLVMIR_LLVMINTERFACES_H_ +#define MLIR_DIALECT_LLVMIR_LLVMINTERFACES_H_ + +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// Verifies the access groups attribute of memory operations that implement the +/// access group interface. +LogicalResult verifyAccessGroupOpInterface(Operation *op); + +/// Verifies the alias analysis attributes of memory operations that implement +/// the alias analysis interface. +LogicalResult verifyAliasAnalysisOpInterface(Operation *op); + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h.inc" + +#endif // MLIR_DIALECT_LLVMIR_LLVMINTERFACES_H_ diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td @@ -48,6 +48,125 @@ ]; } +def AccessGroupOpInterface : OpInterface<"AccessGroupOpInterface"> { + let description = [{ + An interface for memory operations that can carry access groups metadata. + It provides setters and getters for the operation's access groups attribute. + The default implementations of the interface methods expect the operation + to have an attribute of type ArrayAttr named access_groups. + }]; + + let cppNamespace = "::mlir::LLVM"; + let verify = [{ return detail::verifyAccessGroupOpInterface($_op); }]; + + let methods = [ + InterfaceMethod< + /*desc=*/ "Returns the access groups attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getAccessGroupsOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getAccessGroupsAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the access groups attribute", + /*returnType=*/ "void", + /*methodName=*/ "setAccessGroups", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setAccessGroupsAttr(attr); + }] + > + ]; +} + +def AliasAnalysisOpInterface : OpInterface<"AliasAnalysisOpInterface"> { + let description = [{ + An interface for memory operations that can carry alias analysis metadata. + It provides setters and getters for the operation's alias analysis + attributes. The default implementations of the interface methods expect + the operation to have attributes of type ArrayAttr named alias_scopes, + noalias_scopes, and tbaa. + }]; + + let cppNamespace = "::mlir::LLVM"; + let verify = [{ return detail::verifyAliasAnalysisOpInterface($_op); }]; + + let methods = [ + InterfaceMethod< + /*desc=*/ "Returns the alias scopes attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getAliasScopesOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getAliasScopesAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the alias scopes attribute", + /*returnType=*/ "void", + /*methodName=*/ "setAliasScopes", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setAliasScopesAttr(attr); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns the noalias scopes attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getNoAliasScopesOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getNoaliasScopesAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the noalias scopes attribute", + /*returnType=*/ "void", + /*methodName=*/ "setNoAliasScopes", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setNoaliasScopesAttr(attr); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns the tbaa attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getTBAATagsOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getTbaaAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the tbaa attribute", + /*returnType=*/ "void", + /*methodName=*/ "setTBAATags", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setTbaaAttr(attr); + }] + > + ]; +} + //===----------------------------------------------------------------------===// // LLVM dialect type interfaces. //===----------------------------------------------------------------------===// @@ -76,5 +195,4 @@ ]; } - #endif // LLVMIR_INTERFACES diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -269,16 +269,9 @@ } //===----------------------------------------------------------------------===// -// Base classes for LLVM dialect operations. +// Base classes for LLVM enum attributes. //===----------------------------------------------------------------------===// -// Base class for LLVM operations. All operations get an "llvm." prefix in -// their name automatically. LLVM operations have either zero or one result, -// this class is specialized below for both cases and should not be used -// directly. -class LLVM_Op traits = []> : - LLVM_OpBase; - // Case of the LLVM enum attribute backed by I64Attr with customized string // representation that corresponds to what is visible in the textual IR form. // The parameters are as follows: @@ -290,7 +283,6 @@ // is printed/parsed as `weak` in MLIR custom textual format. class LLVM_EnumAttrCase : I64EnumAttrCase { - // The name of the equivalent enumerant in LLVM. string llvmEnumerant = llvmSym; } @@ -314,8 +306,6 @@ list cases, list unsupportedCases = []> : I64EnumAttr { - - // List of unsupported cases that have no conversion to an MLIR value. list unsupported = unsupportedCases; @@ -331,11 +321,50 @@ string llvmClassName = llvmNS; } -// For every value in the list, substitutes the value in the place of "$0" in -// "pattern" and stores the list of strings as "lst". -class ListIntSubst values> { - list lst = !foreach(x, values, - !subst("$0", !cast(x), pattern)); +//===----------------------------------------------------------------------===// +// Patterns for LLVM dialect operations. +//===----------------------------------------------------------------------===// + +// Patterns with code to set flags and metadata of memory operations after their +// translation to LLVM IR instructions. Operations may use the patterns to +// implement their "llvmBuilder". The patterns assume the `op` and `inst` +// variables exist and refer to the original MLIR operation and the translated +// LLVM IR instruction, respectively. +class LLVM_MemOpPatterns { + code setAlignmentCode = [{ + if ($alignment.has_value()) { + auto align = *$alignment; + if (align != 0) + inst->setAlignment(llvm::Align(align)); + } + }]; + code setVolatileCode = [{ + inst->setVolatile($volatile_); + }]; + code setSyncScopeCode = [{ + if ($syncscope.has_value()) { + llvm::LLVMContext &llvmContext = builder.getContext(); + inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope)); + } + }]; + code setOrderingCode = [{ + inst->setAtomic(convertAtomicOrderingToLLVM($ordering)); + }]; + code setNonTemporalMetadataCode = [{ + if ($nontemporal) { + llvm::MDNode *metadata = llvm::MDNode::get( + inst->getContext(), llvm::ConstantAsMetadata::get( + builder.getInt32(1))); + inst->setMetadata(llvm::LLVMContext::MD_nontemporal, metadata); + } + }]; + code setAccessGroupsMetadataCode = [{ + moduleTranslation.setAccessGroupsMetadata(op, inst); + }]; + code setAliasAnalysisMetadataCode = [{ + moduleTranslation.setAliasScopeMetadata(op, inst); + moduleTranslation.setTBAAMetadata(op, inst); + }]; } // Patterns with code obtaining the LLVM IR type of the given operand or result @@ -352,6 +381,36 @@ .getBody()[$0])}]; } +// For every value in the list, substitutes the value in the place of "$0" in +// "pattern" and stores the list of strings as "lst". +class ListIntSubst values> { + list lst = !foreach(x, values, + !subst("$0", !cast(x), pattern)); +} + +//===----------------------------------------------------------------------===// +// Base classes for LLVM dialect operations. +//===----------------------------------------------------------------------===// + +// Base class for LLVM operations. All operations get an "llvm." prefix in +// their name automatically and should either have zero or one result. +class LLVM_Op traits = []> : + LLVM_OpBase; + +// Base class for LLVM memory access operations that implement the access group +// and alias analysis interfaces. The "aliasAttrs" list contains the arguments +// required by the access group and alias analysis interfaces. Derived +// operations should append the "aliasAttrs" to their argument list. +class LLVM_MemAccessOpBase traits = []> : + LLVM_Op, + DeclareOpInterfaceMethods], traits)>, + LLVM_MemOpPatterns { + dag aliasAttrs = (ins OptionalAttr:$access_groups, + OptionalAttr:$alias_scopes, + OptionalAttr:$noalias_scopes, + OptionalAttr:$tbaa); +} // Base class for LLVM intrinsics operation. It is similar to LLVM_Op, but // provides the "llvmBuilder" field for constructing the intrinsic. diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -169,48 +169,8 @@ def LLVM_FNegOp : LLVM_UnaryFloatArithmeticOp< LLVM_ScalarOrVectorOf, "fneg", "FNeg">; -// Common code definitions used to set memory operation attributes and flags. -class MemoryOpBase { - code setAlignmentCode = [{ - if ($alignment.has_value()) { - auto align = *$alignment; - if (align != 0) - inst->setAlignment(llvm::Align(align)); - } - }]; - code setVolatileCode = [{ - inst->setVolatile($volatile_); - }]; - code setSyncScopeCode = [{ - if ($syncscope.has_value()) { - llvm::LLVMContext &llvmContext = builder.getContext(); - inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope)); - } - }]; - code setOrderingCode = [{ - inst->setAtomic(convertAtomicOrderingToLLVM($ordering)); - }]; - code setNonTemporalMetadataCode = [{ - if ($nontemporal) { - llvm::MDNode *metadata = llvm::MDNode::get( - inst->getContext(), llvm::ConstantAsMetadata::get( - builder.getInt32(1))); - inst->setMetadata(llvm::LLVMContext::MD_nontemporal, metadata); - } - }]; - code setAccessGroupsMetadataCode = [{ - moduleTranslation.setAccessGroupsMetadata(op, inst); - }]; - code setAliasScopeMetadataCode = [{ - moduleTranslation.setAliasScopeMetadata(op, inst); - }]; - code setTBAAMetadataCode = [{ - moduleTranslation.setTBAAMetadata(op, inst); - }]; -} - // Memory-related operations. -def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpBase { +def LLVM_AllocaOp : LLVM_Op<"alloca">, LLVM_MemOpPatterns { let arguments = (ins AnyInteger:$arraySize, OptionalAttr:$alignment, OptionalAttr:$elem_type); @@ -347,18 +307,16 @@ let hasVerifier = 1; } -def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { - let arguments = (ins Arg, "", [MemRead]>:$addr, - OptionalAttr:$access_groups, - OptionalAttr:$alias_scopes, - OptionalAttr:$noalias_scopes, - OptionalAttr:$tbaa, - OptionalAttr:$alignment, - UnitAttr:$volatile_, - UnitAttr:$nontemporal, - DefaultValuedAttr:$ordering, - OptionalAttr:$syncscope); +def LLVM_LoadOp : LLVM_MemAccessOpBase<"load"> { + dag args = (ins Arg, "", [MemRead]>:$addr, + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr< + AtomicOrdering, "AtomicOrdering::not_atomic">:$ordering, + OptionalAttr:$syncscope); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); let results = (outs LLVM_LoadableType:$res); string llvmInstName = "Load"; let description = [{ @@ -392,16 +350,13 @@ }]; string llvmBuilder = [{ auto *inst = builder.CreateLoad($_resultType, $addr, $volatile_); + $res = inst; }] # setOrderingCode # setSyncScopeCode # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode - # setAliasScopeMetadataCode - # setTBAAMetadataCode - # [{ - $res = inst; - }]; + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *loadInst = cast(inst); unsigned alignment = loadInst->getAlign().value(); @@ -423,19 +378,17 @@ let hasVerifier = 1; } -def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { - let arguments = (ins LLVM_LoadableType:$value, - Arg,"",[MemWrite]>:$addr, - OptionalAttr:$access_groups, - OptionalAttr:$alias_scopes, - OptionalAttr:$noalias_scopes, - OptionalAttr:$tbaa, - OptionalAttr:$alignment, - UnitAttr:$volatile_, - UnitAttr:$nontemporal, - DefaultValuedAttr:$ordering, - OptionalAttr:$syncscope); +def LLVM_StoreOp : LLVM_MemAccessOpBase<"store"> { + dag args = (ins LLVM_LoadableType:$value, + Arg,"",[MemWrite]>:$addr, + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr< + AtomicOrdering, "AtomicOrdering::not_atomic">:$ordering, + OptionalAttr:$syncscope); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); string llvmInstName = "Store"; let description = [{ The `store` operation is used to write to memory. A store may be marked as @@ -473,8 +426,7 @@ # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode - # setAliasScopeMetadataCode - # setTBAAMetadataCode; + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *storeInst = cast(inst); unsigned alignment = storeInst->getAlign().value(); @@ -1754,15 +1706,17 @@ def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, AnyInteger]>; -def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ +def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [ TypesMatchWith<"result #0 and operand #1 have the same type", - "val", "res", "$_self">]>, MemoryOpBase { - let arguments = (ins AtomicBinOp:$bin_op, - LLVM_PointerTo:$ptr, - LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering, - OptionalAttr:$syncscope, - OptionalAttr:$alignment, - UnitAttr:$volatile_); + "val", "res", "$_self">]> { + dag args = (ins AtomicBinOp:$bin_op, + LLVM_PointerTo:$ptr, + LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$volatile_); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); let results = (outs LLVM_AtomicRMWType:$res); let assemblyFormat = [{ (`volatile` $volatile_^)? $bin_op $ptr `,` $val @@ -1777,7 +1731,9 @@ $res = inst; }] # setVolatileCode # setSyncScopeCode - # setAlignmentCode; + # setAlignmentCode + # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *atomicInst = cast(inst); unsigned alignment = atomicInst->getAlign().value(); @@ -1786,7 +1742,7 @@ convertAtomicOrderingFromLLVM(atomicInst->getOrdering()), getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile()); }]; - list llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1]; + list llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1]; let builders = [ OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val, "LLVM::AtomicOrdering":$ordering, @@ -1799,20 +1755,22 @@ def LLVM_AtomicCmpXchgType : AnyTypeOf<[AnyInteger, LLVM_AnyPointer]>; -def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ +def LLVM_AtomicCmpXchgOp : LLVM_MemAccessOpBase<"cmpxchg", [ TypesMatchWith<"operand #1 and operand #2 have the same type", "val", "cmp", "$_self">, TypesMatchWith<"result #0 has an LLVM struct type consisting of " "the type of operand #2 and a bool", "val", "res", - "getValAndBoolStructType($_self)">]>, MemoryOpBase { - let arguments = (ins LLVM_PointerTo:$ptr, - LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, - AtomicOrdering:$success_ordering, - AtomicOrdering:$failure_ordering, - OptionalAttr:$syncscope, - OptionalAttr:$alignment, - UnitAttr:$weak, - UnitAttr:$volatile_); + "getValAndBoolStructType($_self)">]> { + dag args = (ins LLVM_PointerTo:$ptr, + LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, + AtomicOrdering:$success_ordering, + AtomicOrdering:$failure_ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$weak, + UnitAttr:$volatile_); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); let results = (outs LLVM_AnyStruct:$res); let assemblyFormat = [{ (`weak` $weak^)? (`volatile` $volatile_^)? $ptr `,` $cmp `,` $val @@ -1828,7 +1786,9 @@ inst->setWeak($weak); }] # setVolatileCode # setSyncScopeCode - # setAlignmentCode; + # setAlignmentCode + # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *cmpXchgInst = cast(inst); unsigned alignment = cmpXchgInst->getAlign().value(); @@ -1851,7 +1811,7 @@ let hasVerifier = 1; } -def LLVM_FenceOp : LLVM_Op<"fence">, MemoryOpBase { +def LLVM_FenceOp : LLVM_Op<"fence">, LLVM_MemOpPatterns { let arguments = (ins AtomicOrdering:$ordering, OptionalAttr:$syncscope); let assemblyFormat = "(`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict"; diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -14,6 +14,7 @@ #ifndef MLIR_TARGET_LLVMIR_MODULETRANSLATION_H #define MLIR_TARGET_LLVMIR_MODULETRANSLATION_H +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/IR/Operation.h" #include "mlir/IR/SymbolTable.h" #include "mlir/IR/Value.h" @@ -130,9 +131,8 @@ // Sets LLVM metadata for memory operations that have alias scope information. void setAliasScopeMetadata(Operation *op, llvm::Instruction *inst); - /// Sets LLVM TBAA metadata for memory operations that have - /// TBAA attributes. - void setTBAAMetadata(Operation *op, llvm::Instruction *inst); + /// Sets LLVM TBAA metadata for memory operations that have TBAA attributes. + void setTBAAMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst); /// Sets LLVM loop metadata for branch operations that have a loop annotation /// attribute. diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt --- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt +++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt @@ -4,6 +4,7 @@ IR/FunctionCallUtils.cpp IR/LLVMAttrs.cpp IR/LLVMDialect.cpp + IR/LLVMInterfaces.cpp IR/LLVMTypes.cpp IR/LLVMTypeSyntax.cpp diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "TypeDetail.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinOps.h" @@ -46,8 +47,6 @@ static constexpr const char kElemTypeAttrName[] = "elem_type"; -#include "mlir/Dialect/LLVMIR/LLVMInterfaces.cpp.inc" - static auto processFMFAttr(ArrayRef attrs) { SmallVector filteredAttrs( llvm::make_filter_range(attrs, [&](NamedAttribute attr) { @@ -715,95 +714,6 @@ // LoadOp //===----------------------------------------------------------------------===// -/// Verifies the given array attribute contains symbol references and checks the -/// referenced symbol types using the provided verification function. -LogicalResult verifyMemOpSymbolRefs( - Operation *op, StringRef name, ArrayAttr symbolRefs, - llvm::function_ref - verifySymbolType) { - assert(symbolRefs && "expected a non-null attribute"); - - // Verify that the attribute is a symbol ref array attribute, - // because this constraint is not verified for all attribute - // names processed here (e.g. 'tbaa'). This verification - // is redundant in some cases. - if (!llvm::all_of(symbolRefs, [](Attribute attr) { - return attr && attr.isa(); - })) - return op->emitOpError("attribute '") - << name - << "' failed to satisfy constraint: symbol ref array attribute"; - - for (SymbolRefAttr symbolRef : symbolRefs.getAsRange()) { - StringAttr metadataName = symbolRef.getRootReference(); - StringAttr symbolName = symbolRef.getLeafReference(); - // We want @metadata::@symbol, not just @symbol - if (metadataName == symbolName) { - return op->emitOpError() << "expected '" << symbolRef - << "' to specify a fully qualified reference"; - } - auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to reference a metadata op"; - Operation *symbolOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); - if (!symbolOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to be a valid reference"; - if (failed(verifySymbolType(symbolOp, symbolRef))) { - return failure(); - } - } - - return success(); -} - -/// Verifies the given array attribute contains symbol references that point to -/// metadata operations of the given type. -template -static LogicalResult -verifyMemOpSymbolRefsPointTo(Operation *op, StringRef name, - std::optional symbolRefs) { - if (!symbolRefs) - return success(); - - auto verifySymbolType = [op](Operation *symbolOp, - SymbolRefAttr symbolRef) -> LogicalResult { - if (!isa(symbolOp)) { - return op->emitOpError() - << "expected '" << symbolRef << "' to resolve to a " - << OpTy::getOperationName(); - } - return success(); - }; - return verifyMemOpSymbolRefs(op, name, *symbolRefs, verifySymbolType); -} - -/// Verifies the types of the metadata operations referenced by aliasing and -/// access group metadata. -template -LogicalResult verifyMemOpMetadata(OpTy memOp) { - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getAccessGroupsAttrName(), memOp.getAccessGroups()))) - return failure(); - - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getAliasScopesAttrName(), memOp.getAliasScopes()))) - return failure(); - - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getNoaliasScopesAttrName(), memOp.getNoaliasScopes()))) - return failure(); - - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getTbaaAttrName(), memOp.getTbaa()))) - return failure(); - - return success(); -} - /// Returns true if the given type is supported by atomic operations. All /// integer and float types with limited bit width are supported. Additionally, /// depending on the operation pointers may be supported as well. @@ -850,12 +760,8 @@ LogicalResult LoadOp::verify() { Type valueType = getResult().getType(); - if (failed(verifyAtomicMemOp( - *this, valueType, - {AtomicOrdering::release, AtomicOrdering::acq_rel}))) - return failure(); - - return verifyMemOpMetadata(*this); + return verifyAtomicMemOp(*this, valueType, + {AtomicOrdering::release, AtomicOrdering::acq_rel}); } void LoadOp::build(OpBuilder &builder, OperationState &state, Value addr, @@ -871,12 +777,12 @@ bool isNonTemporal, AtomicOrdering ordering, StringRef syncscope) { build(builder, state, type, addr, - /*access_groups=*/nullptr, - /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, - /*tbaa=*/nullptr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, isNonTemporal, ordering, - syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope), + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, + /*tbaa=*/nullptr); } // Extract the pointee type from the LLVM pointer type wrapped in MLIR. Return @@ -930,23 +836,20 @@ LogicalResult StoreOp::verify() { Type valueType = getValue().getType(); - if (failed(verifyAtomicMemOp( - *this, valueType, - {AtomicOrdering::acquire, AtomicOrdering::acq_rel}))) - return failure(); - - return verifyMemOpMetadata(*this); + return verifyAtomicMemOp(*this, valueType, + {AtomicOrdering::acquire, AtomicOrdering::acq_rel}); } void StoreOp::build(OpBuilder &builder, OperationState &state, Value value, Value addr, unsigned alignment, bool isVolatile, bool isNonTemporal, AtomicOrdering ordering, StringRef syncscope) { - build(builder, state, value, addr, /*access_groups=*/nullptr, - /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, + build(builder, state, value, addr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, isNonTemporal, ordering, - syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope), + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); } /// Parses the StoreOp type either using the typed or opaque pointer format. @@ -2326,7 +2229,9 @@ unsigned alignment, bool isVolatile) { build(builder, state, val.getType(), binOp, ptr, val, ordering, !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, - alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile); + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); } LogicalResult AtomicRMWOp::verify() { @@ -2378,7 +2283,8 @@ successOrdering, failureOrdering, !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak, - isVolatile); + isVolatile, /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); } LogicalResult AtomicCmpXchgOp::verify() { @@ -2964,22 +2870,17 @@ bool isLegalToInline(Operation *op, Region *, bool, IRMapping &) const final { if (isPure(op)) return true; - return llvm::TypeSwitch(op) - .Case([&](auto memOp) { - // Some attributes on load and store operations require handling - // during inlining. Since this is not yet implemented, refuse to - // inline memory operations that have any of these attributes. - if (memOp.getAccessGroups()) - return false; - if (memOp.getAliasScopes()) - return false; - if (memOp.getNoaliasScopes()) - return false; - return true; - }) - .Case([](auto) { return true; }) - .Default([](auto) { return false; }); + // Some attributes on memory operations require handling during + // inlining. Since this is not yet implemented, refuse to inline memory + // operations that have any of these attributes. + if (auto iface = dyn_cast(op)) + if (iface.getAliasScopesOrNull() || iface.getNoAliasScopesOrNull()) + return false; + if (auto iface = dyn_cast(op)) + if (iface.getAccessGroupsOrNull()) + return false; + return isa(op); } /// Handle the given inlined return by replacing it with a branch. This diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp new file mode 100644 --- /dev/null +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp @@ -0,0 +1,115 @@ +//===- LLVMInterfaces.cpp - LLVM Interfaces ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines op interfaces for the LLVM dialect in MLIR. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +using namespace mlir; +using namespace mlir::LLVM; + +/// Verifies the given array attribute contains symbol references and checks the +/// referenced symbol types using the provided verification function. +static LogicalResult +verifySymbolRefs(Operation *op, StringRef name, ArrayAttr symbolRefs, + llvm::function_ref + verifySymbolType) { + assert(symbolRefs && "expected a non-null attribute"); + + // Verify that the attribute is a symbol ref array attribute, + // because this constraint is not verified for all attribute + // names processed here (e.g. 'tbaa'). This verification + // is redundant in some cases. + if (!llvm::all_of(symbolRefs, [](Attribute attr) { + return attr && attr.isa(); + })) + return op->emitOpError() << name + << " attribute failed to satisfy constraint: " + "symbol ref array attribute"; + + for (SymbolRefAttr symbolRef : symbolRefs.getAsRange()) { + StringAttr metadataName = symbolRef.getRootReference(); + StringAttr symbolName = symbolRef.getLeafReference(); + // We want @metadata::@symbol, not just @symbol + if (metadataName == symbolName) { + return op->emitOpError() << "expected '" << symbolRef + << "' to specify a fully qualified reference"; + } + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to reference a metadata op"; + Operation *symbolOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); + if (!symbolOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to be a valid reference"; + if (failed(verifySymbolType(symbolOp, symbolRef))) { + return failure(); + } + } + + return success(); +} + +/// Verifies the given array attribute contains symbol references that point to +/// metadata operations of the given type. +template +LogicalResult verifySymbolRefsPointTo(Operation *op, StringRef name, + ArrayAttr symbolRefs) { + if (!symbolRefs) + return success(); + + auto verifySymbolType = [op](Operation *symbolOp, + SymbolRefAttr symbolRef) -> LogicalResult { + if (!isa(symbolOp)) { + return op->emitOpError() + << "expected '" << symbolRef << "' to resolve to a " + << OpTy::getOperationName(); + } + return success(); + }; + return verifySymbolRefs(op, name, symbolRefs, verifySymbolType); +} + +//===----------------------------------------------------------------------===// +// AccessGroupOpInterface +//===----------------------------------------------------------------------===// + +LogicalResult mlir::LLVM::detail::verifyAccessGroupOpInterface(Operation *op) { + auto iface = cast(op); + if (failed(verifySymbolRefsPointTo( + iface, "access groups", iface.getAccessGroupsOrNull()))) + return failure(); + return success(); +} + +//===----------------------------------------------------------------------===// +// AliasAnalysisOpInterface +//===----------------------------------------------------------------------===// + +LogicalResult +mlir::LLVM::detail::verifyAliasAnalysisOpInterface(Operation *op) { + auto iface = cast(op); + if (failed(verifySymbolRefsPointTo( + iface, "alias scopes", iface.getAliasScopesOrNull()))) + return failure(); + if (failed(verifySymbolRefsPointTo( + iface, "noalias scopes", iface.getNoAliasScopesOrNull()))) + return failure(); + if (failed(verifySymbolRefsPointTo( + iface, "tbaa tags", iface.getTBAATagsOrNull()))) + return failure(); + return success(); +} + +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.cpp.inc" diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -12,6 +12,7 @@ #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Support/LLVM.h" #include "mlir/Target/LLVMIR/ModuleImport.h" @@ -77,30 +78,6 @@ return convertibleMetadata; } -namespace { -/// Helper class to attach metadata attributes to specific operation types. It -/// specializes TypeSwitch to take an Operation and return a LogicalResult. -template -struct AttributeSetter { - AttributeSetter(Operation *op) : op(op) {} - - /// Calls `attachFn` on the provided Operation if it has one of - /// the given operation types. Returns failure otherwise. - template - LogicalResult apply(CallableT &&attachFn) { - return llvm::TypeSwitch(op) - .Case([&attachFn](auto concreteOp) { - attachFn(concreteOp); - return success(); - }) - .Default([&](auto) { return failure(); }); - } - -private: - Operation *op; -}; -} // namespace - /// Converts the given profiling metadata `node` to an MLIR profiling attribute /// and attaches it to the imported operation if the translation succeeds. /// Returns failure otherwise. @@ -148,11 +125,13 @@ branchWeights.push_back(branchWeight->getZExtValue()); } - return AttributeSetter(op).apply( - [&](auto branchWeightOp) { + return TypeSwitch(op) + .Case([&](auto branchWeightOp) { branchWeightOp.setBranchWeightsAttr( builder.getI32VectorAttr(branchWeights)); - }); + return success(); + }) + .Default([](auto) { return failure(); }); } /// Searches the symbol reference pointing to the metadata operation that @@ -164,9 +143,12 @@ if (!tbaaTagSym) return failure(); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setTbaaAttr(ArrayAttr::get(memOp.getContext(), tbaaTagSym)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setTBAATags(ArrayAttr::get(iface.getContext(), tbaaTagSym)); + return success(); } /// Looks up all the symbol references pointing to the access group operations @@ -181,12 +163,14 @@ if (failed(accessGroups)) return failure(); - SmallVector accessGroupAttrs(accessGroups->begin(), - accessGroups->end()); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setAccessGroupsAttr( - ArrayAttr::get(memOp.getContext(), accessGroupAttrs)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setAccessGroups(ArrayAttr::get( + iface.getContext(), + SmallVector{accessGroups->begin(), accessGroups->end()})); + return success(); } /// Converts the given loop metadata node to an MLIR loop annotation attribute @@ -218,12 +202,14 @@ if (failed(aliasScopes)) return failure(); - SmallVector aliasScopeAttrs(aliasScopes->begin(), - aliasScopes->end()); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setAliasScopesAttr( - ArrayAttr::get(memOp.getContext(), aliasScopeAttrs)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setAliasScopes(ArrayAttr::get( + iface.getContext(), + SmallVector{aliasScopes->begin(), aliasScopes->end()})); + return success(); } /// Looks up all the symbol references pointing to the alias scope operations @@ -233,17 +219,19 @@ static LogicalResult setNoaliasScopesAttr(const llvm::MDNode *node, Operation *op, LLVM::ModuleImport &moduleImport) { - FailureOr> noaliasScopes = + FailureOr> noAliasScopes = moduleImport.lookupAliasScopeAttrs(node); - if (failed(noaliasScopes)) + if (failed(noAliasScopes)) return failure(); - SmallVector noaliasScopeAttrs(noaliasScopes->begin(), - noaliasScopes->end()); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setNoaliasScopesAttr( - ArrayAttr::get(memOp.getContext(), noaliasScopeAttrs)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setNoAliasScopes(ArrayAttr::get( + iface.getContext(), + SmallVector{noAliasScopes->begin(), noAliasScopes->end()})); + return success(); } namespace { diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -18,6 +18,7 @@ #include "LoopAnnotationTranslation.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h" #include "mlir/Dialect/OpenMP/OpenMPDialect.h" #include "mlir/IR/Attributes.h" @@ -1091,31 +1092,25 @@ return tbaaMetadataMapping.lookup(tagOp); } -void ModuleTranslation::setTBAAMetadata(Operation *op, +void ModuleTranslation::setTBAAMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst) { - auto populateTBAAMetadata = [&](std::optional tagRefs) { - if (!tagRefs || tagRefs->empty()) - return; - - // LLVM IR currently does not support attaching more than one - // TBAA access tag to a memory accessing instruction. - // It may be useful to support this in future, but for the time being - // just ignore the metadata if MLIR operation has multiple access tags. - if (tagRefs->size() > 1) { - op->emitWarning() << "TBAA access tags were not translated, because LLVM " - "IR only supports a single tag per instruction"; - return; - } + ArrayAttr tagRefs = op.getTBAATagsOrNull(); + if (!tagRefs || tagRefs.empty()) + return; - SymbolRefAttr tagRef = (*tagRefs)[0].cast(); - llvm::MDNode *node = getTBAANode(op, tagRef); - inst->setMetadata(llvm::LLVMContext::MD_tbaa, node); - }; + // LLVM IR currently does not support attaching more than one TBAA access tag + // to a memory accessing instruction. It may be useful to support this in + // future, but for the time being just ignore the metadata if MLIR operation + // has multiple access tags. + if (tagRefs.size() > 1) { + op.emitWarning() << "TBAA access tags were not translated, because LLVM " + "IR only supports a single tag per instruction"; + return; + } - llvm::TypeSwitch(op) - .Case( - [&](auto memOp) { populateTBAAMetadata(memOp.getTbaa()); }) - .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); + SymbolRefAttr tagRef = tagRefs[0].cast(); + llvm::MDNode *node = getTBAANode(op, tagRef); + inst->setMetadata(llvm::LLVMContext::MD_tbaa, node); } LogicalResult ModuleTranslation::createTBAAMetadata() { diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -938,9 +938,9 @@ // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : !llvm.ptr) { // expected-error@below {{expected '@func1' to specify a fully qualified reference}} - %0 = llvm.load %arg0 { "access_groups" = [@func1] } : !llvm.ptr + %0 = llvm.load %arg0 { "access_groups" = [@func1] } : !llvm.ptr -> i32 llvm.return } llvm.func @func1() { @@ -951,9 +951,9 @@ // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : i32, %arg1 : !llvm.ptr) { // expected-error@below {{expected '@accessGroups::@group1' to reference a metadata op}} - %0 = llvm.load %arg0 { "access_groups" = [@accessGroups::@group1] } : !llvm.ptr + llvm.store %arg0, %arg1 { "access_groups" = [@accessGroups::@group1] } : i32, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -963,9 +963,9 @@ // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : !llvm.ptr, %arg1 : f32) { // expected-error@below {{expected '@metadata::@group1' to be a valid reference}} - %0 = llvm.load %arg0 { "access_groups" = [@metadata::@group1] } : !llvm.ptr + %0 = llvm.atomicrmw fadd %arg0, %arg1 monotonic { "access_groups" = [@metadata::@group1] } : !llvm.ptr, f32 llvm.return } llvm.metadata @metadata { @@ -975,9 +975,9 @@ // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : !llvm.ptr, %arg1 : i32, %arg2 : i32) { // expected-error@below {{expected '@metadata::@scope' to resolve to a llvm.access_group}} - %0 = llvm.load %arg0 { "access_groups" = [@metadata::@scope] } : !llvm.ptr + %0 = llvm.cmpxchg %arg0, %arg1, %arg2 acq_rel monotonic { "access_groups" = [@metadata::@scope] } : !llvm.ptr, i32 llvm.return } llvm.metadata @metadata { @@ -989,9 +989,9 @@ // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @aliasScope(%arg0 : !llvm.ptr, %arg1 : i32, %arg2 : i32) { // expected-error@below {{attribute 'alias_scopes' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "alias_scopes" = "test" } : !llvm.ptr + %0 = llvm.cmpxchg %arg0, %arg1, %arg2 acq_rel monotonic { "alias_scopes" = "test" } : !llvm.ptr, i32 llvm.return } } @@ -999,9 +999,9 @@ // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @noAliasScopes(%arg0 : !llvm.ptr) { // expected-error@below {{attribute 'noalias_scopes' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "noalias_scopes" = "test" } : !llvm.ptr + %0 = llvm.load %arg0 { "noalias_scopes" = "test" } : !llvm.ptr -> i32 llvm.return } } @@ -1009,9 +1009,9 @@ // ----- module { - llvm.func @aliasScope(%arg0 : !llvm.ptr) { + llvm.func @aliasScope(%arg0 : i32, %arg1 : !llvm.ptr) { // expected-error@below {{expected '@metadata::@group' to resolve to a llvm.alias_scope}} - %0 = llvm.load %arg0 { "alias_scopes" = [@metadata::@group] } : !llvm.ptr + llvm.store %arg0, %arg1 { "alias_scopes" = [@metadata::@group] } : i32, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -1022,9 +1022,9 @@ // ----- module { - llvm.func @aliasScope(%arg0 : !llvm.ptr) { + llvm.func @aliasScope(%arg0 : !llvm.ptr, %arg1 : f32) { // expected-error@below {{expected '@metadata::@group' to resolve to a llvm.alias_scope}} - %0 = llvm.load %arg0 { "noalias_scopes" = [@metadata::@group] } : !llvm.ptr + %0 = llvm.atomicrmw fadd %arg0, %arg1 monotonic { "noalias_scopes" = [@metadata::@group] } : !llvm.ptr, f32 llvm.return } llvm.metadata @metadata { diff --git a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll --- a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll @@ -59,3 +59,23 @@ !3 = !{!3, !1} !4 = !{!2} !5 = !{!3} + +; // ----- + +; CHECK-LABEL: llvm.func @supported_ops +define void @supported_ops(ptr %arg1, float %arg2, i32 %arg3, i32 %arg4) { + ; CHECK: llvm.load {{.*}}alias_scopes = + %1 = load i32, ptr %arg1, !alias.scope !3 + ; CHECK: llvm.store {{.*}}alias_scopes = + store i32 %1, ptr %arg1, !alias.scope !3 + ; CHECK: llvm.atomicrmw {{.*}}alias_scopes = + %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !alias.scope !3 + ; CHECK: llvm.cmpxchg {{.*}}alias_scopes = + %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !alias.scope !3 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1} +!2 = !{!2, !0} +!3 = !{!2} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -27,6 +27,25 @@ ; // ----- +; CHECK-LABEL: llvm.func @supported_ops +define void @supported_ops(ptr %arg1, float %arg2, i32 %arg3, i32 %arg4) { + ; CHECK: llvm.load {{.*}}access_groups = + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ; CHECK: llvm.store {{.*}}access_groups = + store i32 %1, ptr %arg1, !llvm.access.group !0 + ; CHECK: llvm.atomicrmw {{.*}}access_groups = + %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !llvm.access.group !0 + ; CHECK: llvm.cmpxchg {{.*}}access_groups = + %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !llvm.access.group !0 + ret void +} + +!0 = !{!1, !2} +!1 = distinct !{} +!2 = distinct !{} + +; // ----- + ; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation ; CHECK-LABEL: @simple diff --git a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll --- a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll @@ -29,7 +29,7 @@ !4 = !{!"other scalar type", !5, i64 0} !5 = !{!"Other language TBAA"} -// ----- +; // ----- ; CHECK-LABEL: llvm.metadata @__llvm_global_metadata { ; CHECK-NEXT: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} @@ -68,3 +68,22 @@ !11 = !{!12, !13, i64 0} !12 = !{!"agg1_t", !13, i64 0, !13, i64 4} !13 = !{!"int", !9, i64 0} + +; // ----- + +; CHECK-LABEL: llvm.func @supported_ops +define void @supported_ops(ptr %arg1, float %arg2, i32 %arg3, i32 %arg4) { + ; CHECK: llvm.load {{.*}}tbaa = + %1 = load i32, ptr %arg1, !tbaa !0 + ; CHECK: llvm.store {{.*}}tbaa = + store i32 %1, ptr %arg1, !tbaa !0 + ; CHECK: llvm.atomicrmw {{.*}}tbaa = + %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !tbaa !0 + ; CHECK: llvm.cmpxchg {{.*}}tbaa = + %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !tbaa !0 + ret void +} + +!0 = !{!1, !1, i64 0} +!1 = !{!"scalar type", !2, i64 0} +!2 = !{!"Simple C/C++ TBAA"} diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1975,27 +1975,29 @@ // ----- module { - llvm.func @aliasScope(%arg1 : !llvm.ptr, %arg2 : !llvm.ptr, %arg3 : !llvm.ptr) { + llvm.func @aliasScope(%arg1 : !llvm.ptr) { %0 = llvm.mlir.constant(0 : i32) : i32 - llvm.store %0, %arg1 { alias_scopes = [@metadata::@scope1], noalias_scopes = [@metadata::@scope2, @metadata::@scope3] } : !llvm.ptr - llvm.store %0, %arg2 { alias_scopes = [@metadata::@scope2], noalias_scopes = [@metadata::@scope1, @metadata::@scope3] } : !llvm.ptr - %1 = llvm.load %arg3 { alias_scopes = [@metadata::@scope3], noalias_scopes = [@metadata::@scope1, @metadata::@scope2] } : !llvm.ptr + llvm.store %0, %arg1 {alias_scopes = [@metadata::@scope1], noalias_scopes = [@metadata::@scope2, @metadata::@scope3]} : i32, !llvm.ptr + %1 = llvm.load %arg1 {alias_scopes = [@metadata::@scope2], noalias_scopes = [@metadata::@scope1, @metadata::@scope3]} : !llvm.ptr -> i32 + %2 = llvm.atomicrmw add %arg1, %0 monotonic {alias_scopes = [@metadata::@scope3], noalias_scopes = [@metadata::@scope1, @metadata::@scope2]} : !llvm.ptr, i32 + %3 = llvm.cmpxchg %arg1, %1, %2 acq_rel monotonic {alias_scopes = [@metadata::@scope3]} : !llvm.ptr, i32 llvm.return } llvm.metadata @metadata { - llvm.alias_scope_domain @domain { description = "The domain"} - llvm.alias_scope @scope1 { domain = @domain, description = "The first scope" } - llvm.alias_scope @scope2 { domain = @domain } - llvm.alias_scope @scope3 { domain = @domain } + llvm.alias_scope_domain @domain {description = "The domain"} + llvm.alias_scope @scope1 {domain = @domain, description = "The first scope"} + llvm.alias_scope @scope2 {domain = @domain} + llvm.alias_scope @scope3 {domain = @domain} } } // Function // CHECK-LABEL: aliasScope // CHECK: store {{.*}}, !alias.scope ![[SCOPES1:[0-9]+]], !noalias ![[SCOPES23:[0-9]+]] -// CHECK: store {{.*}}, !alias.scope ![[SCOPES2:[0-9]+]], !noalias ![[SCOPES13:[0-9]+]] -// CHECK: load {{.*}}, !alias.scope ![[SCOPES3:[0-9]+]], !noalias ![[SCOPES12:[0-9]+]] +// CHECK: load {{.*}}, !alias.scope ![[SCOPES2:[0-9]+]], !noalias ![[SCOPES13:[0-9]+]] +// CHECK: atomicrmw {{.*}}, !alias.scope ![[SCOPES3:[0-9]+]], !noalias ![[SCOPES12:[0-9]+]] +// CHECK: cmpxchg {{.*}}, !alias.scope ![[SCOPES3]] // Metadata // CHECK-DAG: ![[DOMAIN:[0-9]+]] = distinct !{![[DOMAIN]], !"The domain"} diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir --- a/mlir/test/Target/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -45,7 +45,7 @@ llvm.func @vectorizeOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation >} ^bb1: @@ -111,7 +111,7 @@ llvm.return } -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} // CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unroll.enable"} // CHECK-DAG: ![[VEC_NODE2:[0-9]+]] = !{!"llvm.loop.unroll.full"} @@ -236,7 +236,7 @@ // CHECK-LABEL: @loopOptions llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { %0 = llvm.mlir.constant(0 : i32) : i32 - %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) + %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) llvm.br ^bb3(%0 : i32) ^bb3(%1: i32): %2 = llvm.icmp "slt" %1, %arg1 : i32 @@ -249,7 +249,13 @@ ^bb4: %3 = llvm.add %1, %arg2 : i32 // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] - %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr + %5 = llvm.load %4 {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr -> i32 + // CHECK: store i32 %{{.*}}, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + llvm.store %5, %4 {access_groups = [@metadata::@group1, @metadata::@group2]} : i32, !llvm.ptr + // CHECK: = atomicrmw add ptr %{{.*}}, i32 %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + %6 = llvm.atomicrmw add %4, %5 monotonic {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr, i32 + // CHECK: = cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + %7 = llvm.cmpxchg %4, %5, %6 acq_rel monotonic {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr, i32 // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] llvm.br ^bb3(%3 : i32) {loop_annotation = #llvm.loop_annotation< licm = , @@ -266,7 +272,7 @@ llvm.access_group @group2 } -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} // CHECK-DAG: ![[PA_NODE:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} // CHECK-DAG: ![[GROUP_NODE1:[0-9]+]] = distinct !{} // CHECK-DAG: ![[GROUP_NODE2:[0-9]+]] = distinct !{} diff --git a/mlir/test/Target/LLVMIR/tbaa.mlir b/mlir/test/Target/LLVMIR/tbaa.mlir --- a/mlir/test/Target/LLVMIR/tbaa.mlir +++ b/mlir/test/Target/LLVMIR/tbaa.mlir @@ -65,6 +65,10 @@ %5 = llvm.getelementptr inbounds %arg0[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg1_t", (i32, i32)> // CHECK: store i32 %{{.*}}, ptr %{{.*}},{{.*}}!tbaa ![[STAG:[0-9]*]] llvm.store %4, %5 {tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr + // CHECK: atomicrmw add ptr %{{.*}}, i32 %{{.*}} !tbaa ![[STAG]] + %6 = llvm.atomicrmw add %5, %4 monotonic {tbaa = [@__tbaa::@tbaa_tag_7]} : !llvm.ptr, i32 + // CHECK: cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} !tbaa ![[STAG]] + %7 = llvm.cmpxchg %5, %6, %4 acq_rel monotonic {tbaa = [@__tbaa::@tbaa_tag_7]} : !llvm.ptr, i32 llvm.return } }