diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -1095,6 +1095,7 @@ https://llvm.org/docs/LangRef.html#noalias-and-alias-scope-metadata }]; let assemblyFormat = "$sym_name attr-dict"; + let hasVerifier = 1; } def LLVM_AccessGroupMetadataOp : LLVM_Op<"access_group", [ diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -176,6 +176,12 @@ LoopAnnotationAttr translateLoopAnnotationAttr(const llvm::MDNode *node, Location loc) const; + /// Returns the symbol references pointing to the alias scope operations that + /// map to the alias scope nodes starting from the metadata `node`. Returns + /// failure, if any of the symbol references cannot be found. + FailureOr> + lookupAliasScopeAttrs(const llvm::MDNode *node) const; + private: /// Clears the block and value mapping before processing a new region. void clearBlockAndValueMapping() { @@ -268,6 +274,12 @@ /// symbol pointing to the translated operation. Returns success if all /// conversions succeed and failure otherwise. LogicalResult processAccessGroupMetadata(const llvm::MDNode *node); + /// Converts all LLVM alias scopes and domains starting from `node` to MLIR + /// alias scope and domain operations and stores a mapping from every nested + /// alias scope or alias domain node to the symbol pointing to the translated + /// operation. Returns success if all conversions succeed and failure + /// otherwise. + LogicalResult processAliasScopeMetadata(const llvm::MDNode *node); /// Builder pointing at where the next instruction should be generated. OpBuilder builder; @@ -298,9 +310,11 @@ /// operations for all operations that return no result. All operations that /// return a result have a valueMapping entry instead. DenseMap noResultOpMapping; - /// Mapping between LLVM TBAA metadata nodes and symbol references - /// to the LLVMIR dialect TBAA operations corresponding to these - /// nodes. + /// Mapping between LLVM alias scope and domain metadata nodes and symbol + /// references to the LLVM dialect operations corresponding to these nodes. + DenseMap aliasScopeMapping; + /// Mapping between LLVM TBAA metadata nodes and symbol references to the LLVM + /// dialect TBAA operations corresponding to these nodes. DenseMap tbaaMapping; /// The stateful type translator (contains named structs). LLVM::TypeFromLLVMIRTranslator typeTranslator; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2734,6 +2734,21 @@ return success(); } +//===----------------------------------------------------------------------===// +// AliasScopeMetadataOp +//===----------------------------------------------------------------------===// + +LogicalResult AliasScopeMetadataOp::verify() { + Operation *domainOp = SymbolTable::lookupNearestSymbolFrom( + this->getOperation(), getDomainAttr()); + if (!isa_and_nonnull(domainOp)) { + return this->emitOpError() + << "expected '" << getDomain() + << "' to reference a domain operation in the same region"; + } + return success(); +} + //===----------------------------------------------------------------------===// // OpAsmDialectInterface //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -71,8 +71,9 @@ /// dialect attributes. static ArrayRef getSupportedMetadataImpl() { static const SmallVector convertibleMetadata = { - llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa, - llvm::LLVMContext::MD_access_group, llvm::LLVMContext::MD_loop}; + llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa, + llvm::LLVMContext::MD_access_group, llvm::LLVMContext::MD_loop, + llvm::LLVMContext::MD_noalias, llvm::LLVMContext::MD_alias_scope}; return convertibleMetadata; } @@ -178,8 +179,9 @@ /// that map to the access group nodes starting from the access group metadata /// `node`, and attaches all of them to the imported operation if the lookups /// succeed. Returns failure otherwise. -static LogicalResult setAccessGroupAttr(const llvm::MDNode *node, Operation *op, - LLVM::ModuleImport &moduleImport) { +static LogicalResult setAccessGroupsAttr(const llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { FailureOr> accessGroups = moduleImport.lookupAccessGroupAttrs(node); if (failed(accessGroups)) @@ -211,6 +213,45 @@ .Default([](auto) { return failure(); }); } +/// Looks up all the symbol references pointing to the alias scope operations +/// that map to the alias scope nodes starting from the alias scope metadata +/// `node`, and attaches all of them to the imported operation if the lookups +/// succeed. Returns failure otherwise. +static LogicalResult setAliasScopesAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { + FailureOr> aliasScopes = + moduleImport.lookupAliasScopeAttrs(node); + if (failed(aliasScopes)) + return failure(); + + SmallVector aliasScopeAttrs(aliasScopes->begin(), + aliasScopes->end()); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setAliasScopesAttr( + ArrayAttr::get(memOp.getContext(), aliasScopeAttrs)); + }); +} + +/// Looks up all the symbol references pointing to the alias scope operations +/// that map to the alias scope nodes starting from the noalias metadata `node`, +/// and attaches all of them to the imported operation if the lookups succeed. +/// Returns failure otherwise. +static LogicalResult setNoaliasScopesAttr(const llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { + FailureOr> noaliasScopes = + moduleImport.lookupAliasScopeAttrs(node); + if (failed(noaliasScopes)) + return failure(); + + SmallVector noaliasScopeAttrs(noaliasScopes->begin(), + noaliasScopes->end()); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setNoaliasScopesAttr( + ArrayAttr::get(memOp.getContext(), noaliasScopeAttrs)); + }); +} + namespace { /// Implementation of the dialect interface that converts operations belonging @@ -238,9 +279,13 @@ if (kind == llvm::LLVMContext::MD_tbaa) return setTBAAAttr(node, op, moduleImport); if (kind == llvm::LLVMContext::MD_access_group) - return setAccessGroupAttr(node, op, moduleImport); + return setAccessGroupsAttr(node, op, moduleImport); if (kind == llvm::LLVMContext::MD_loop) return setLoopAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_alias_scope) + return setAliasScopesAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_noalias) + return setNoaliasScopesAttr(node, op, moduleImport); // A handler for a supported metadata kind is missing. llvm_unreachable("unknown metadata type"); diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -521,6 +521,99 @@ return success(); } +LogicalResult +ModuleImport::processAliasScopeMetadata(const llvm::MDNode *node) { + Location loc = mlirModule.getLoc(); + // Helper that verifies the node has a self reference operand. + auto verifySelfRef = [](const llvm::MDNode *node) { + return node->getNumOperands() != 0 && + node == dyn_cast(node->getOperand(0)); + }; + // Helper that verifies the given operand is a string or does not exist. + auto verifyDescription = [](const llvm::MDNode *node, unsigned idx) { + return idx >= node->getNumOperands() || + isa(node->getOperand(idx)); + }; + // Helper that creates an alias scope domain operation. + auto createAliasScopeDomainOp = [&](const llvm::MDNode *aliasDomain) { + StringAttr description = nullptr; + if (aliasDomain->getNumOperands() >= 2) + if (auto *operand = dyn_cast(aliasDomain->getOperand(1))) + description = builder.getStringAttr(operand->getString()); + std::string name = llvm::formatv("domain_{0}", aliasScopeMapping.size()); + return builder.create(loc, name, description); + }; + + // Collect the alias scopes and domains to translate them. + for (const llvm::MDOperand &operand : node->operands()) { + if (const auto *scope = dyn_cast(operand)) { + llvm::AliasScopeNode aliasScope(scope); + const llvm::MDNode *domain = aliasScope.getDomain(); + + // Verify the scope node points to valid scope metadata which includes + // verifying its domain. Perform the verification before looking it up in + // the alias scope mapping since it could have been inserted as a domain + // node before. + if (!verifySelfRef(scope) || !domain || !verifyDescription(scope, 2)) + return emitError(loc) << "unsupported alias scope node: " + << diagMD(scope, llvmModule.get()); + if (!verifySelfRef(domain) || !verifyDescription(domain, 1)) + return emitError(loc) << "unsupported alias domain node: " + << diagMD(domain, llvmModule.get()); + + if (aliasScopeMapping.count(scope)) + continue; + + // Set the insertion point to the end of the global metadata operation. + MetadataOp metadataOp = getGlobalMetadataOp(); + StringAttr metadataOpName = + SymbolTable::getSymbolName(getGlobalMetadataOp()); + OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToEnd(&metadataOp.getBody().back()); + + // Convert the domain metadata node if it has not been translated before. + auto it = aliasScopeMapping.find(aliasScope.getDomain()); + if (it == aliasScopeMapping.end()) { + auto aliasScopeDomainOp = createAliasScopeDomainOp(domain); + auto symbolRef = SymbolRefAttr::get( + builder.getContext(), metadataOpName, + FlatSymbolRefAttr::get(builder.getContext(), + aliasScopeDomainOp.getSymName())); + it = aliasScopeMapping.try_emplace(domain, symbolRef).first; + } + + // Convert the scope metadata node if it has not been converted before. + StringAttr description = nullptr; + if (!aliasScope.getName().empty()) + description = builder.getStringAttr(aliasScope.getName()); + std::string name = llvm::formatv("scope_{0}", aliasScopeMapping.size()); + auto aliasScopeOp = builder.create( + loc, name, it->getSecond().getLeafReference().getValue(), + description); + auto symbolRef = + SymbolRefAttr::get(builder.getContext(), metadataOpName, + FlatSymbolRefAttr::get(builder.getContext(), + aliasScopeOp.getSymName())); + aliasScopeMapping.try_emplace(aliasScope.getNode(), symbolRef); + } + } + return success(); +} + +FailureOr> +ModuleImport::lookupAliasScopeAttrs(const llvm::MDNode *node) const { + SmallVector aliasScopes; + aliasScopes.reserve(node->getNumOperands()); + for (const llvm::MDOperand &operand : node->operands()) { + auto *node = cast(operand.get()); + aliasScopes.push_back(aliasScopeMapping.lookup(node)); + } + // Return failure if one of the alias scope lookups failed. + if (llvm::is_contained(aliasScopes, nullptr)) + return failure(); + return aliasScopes; +} + LogicalResult ModuleImport::convertMetadata() { OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToEnd(mlirModule.getBody()); @@ -539,8 +632,12 @@ if (aliasAnalysisNodes.TBAA) if (failed(processTBAAMetadata(aliasAnalysisNodes.TBAA))) return failure(); - - // TODO: Support noalias and scope metadata nodes. + if (aliasAnalysisNodes.Scope) + if (failed(processAliasScopeMetadata(aliasAnalysisNodes.Scope))) + return failure(); + if (aliasAnalysisNodes.NoAlias) + if (failed(processAliasScopeMetadata(aliasAnalysisNodes.NoAlias))) + return failure(); } } return success(); diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -964,6 +964,28 @@ // ----- +module { + llvm.metadata @metadata { + llvm.access_group @group + // expected-error@below {{expected 'group' to reference a domain operation in the same region}} + llvm.alias_scope @scope { domain = @group } + } +} + +// ----- + +module { + llvm.metadata @metadata { + // expected-error@below {{expected 'domain' to reference a domain operation in the same region}} + llvm.alias_scope @scope { domain = @domain } + } + llvm.metadata @other_metadata { + llvm.alias_scope_domain @domain + } +} + +// ----- + llvm.func @wmmaLoadOp_invalid_mem_space(%arg0: !llvm.ptr, %arg1: i32) { // expected-error@+1 {{'nvvm.wmma.load' op expected source pointer in memory space 0, 1, 3}} %0 = nvvm.wmma.load %arg0, %arg1 diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -572,3 +572,39 @@ } !0 = !{!"branch_weights", i32 64} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported alias scope node: ![[NODE:[0-9]+]] = distinct !{![[NODE]]} +define void @alias_scope(ptr %arg1) { + %1 = load i32, ptr %arg1, !alias.scope !0 + ret void +} + +!0 = !{!0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported alias scope node: ![[NODE:[0-9]+]] = distinct !{![[NODE]], !"The domain"} +define void @alias_scope(ptr %arg1) { + %1 = load i32, ptr %arg1, !alias.scope !1 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = !{!1, !0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported alias domain node: ![[NODE:[0-9]+]] = distinct !{![[NODE]], ![[NODE]]} +define void @alias_scope_domain(ptr %arg1) { + %1 = load i32, ptr %arg1, !alias.scope !2 + ret void +} + +!0 = distinct !{!0, !0} +!1 = !{!1, !0} +!2 = !{!1} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll new file mode 100644 --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll @@ -0,0 +1,61 @@ +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.alias_scope_domain @[[DOMAIN:.*]] {description = "The domain"} +; CHECK: llvm.alias_scope @[[$SCOPE0:.*]] {description = "The first scope", domain = @[[DOMAIN]]} +; CHECK: llvm.alias_scope @[[$SCOPE1:.*]] {domain = @[[DOMAIN]]} +; CHECK: llvm.alias_scope @[[$SCOPE2:.*]] {domain = @[[DOMAIN]]} +; CHECK: } + +; CHECK-LABEL: llvm.func @alias_scope +define void @alias_scope(ptr %arg1) { + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]], @__llvm_global_metadata::@[[$SCOPE2]]] + %1 = load i32, ptr %arg1, !alias.scope !4, !noalias !7 + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]], @__llvm_global_metadata::@[[$SCOPE2]]] + %2 = load i32, ptr %arg1, !alias.scope !5, !noalias !8 + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE2]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]], @__llvm_global_metadata::@[[$SCOPE1]]] + %3 = load i32, ptr %arg1, !alias.scope !6, !noalias !9 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1, !0, !"The first scope"} +!2 = distinct !{!2, !0} +!3 = distinct !{!3, !0} +!4 = !{!1} +!5 = !{!2} +!6 = !{!3} +!7 = !{!2, !3} +!8 = !{!1, !3} +!9 = !{!1, !2} + +; // ----- + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.alias_scope_domain @[[DOMAIN0:.*]] {description = "The domain"} +; CHECK: llvm.alias_scope @[[$SCOPE0:.*]] {domain = @[[DOMAIN0]]} +; CHECK: llvm.alias_scope_domain @[[DOMAIN1:.*]] +; CHECK: llvm.alias_scope @[[$SCOPE1:.*]] {domain = @[[DOMAIN1]]} +; CHECK: } + +; CHECK-LABEL: llvm.func @two_domains +define void @two_domains(ptr %arg1) { + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]]] + %1 = load i32, ptr %arg1, !alias.scope !4, !noalias !5 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1} +!2 = !{!2, !0} +!3 = !{!3, !1} +!4 = !{!2} +!5 = !{!3}