diff --git a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td --- a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td +++ b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td @@ -40,7 +40,7 @@ class ArmNeon_IntrOp overloadedResults, list overloadedOperands, int numResults, list traits = [], bit requiresAccessGroup = 0, - bit requiresAliasScope = 0> + bit requiresAliasAnalysis = 0> : LLVM_IntrOpBase; + /*requiresAliasAnalysis=*/requiresAliasAnalysis>; // ArmNeon dialect op that corresponds to an LLVM IR intrinsic with one // overloaded result. diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -4,7 +4,6 @@ include "mlir/IR/OpBase.td" include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td" include "mlir/Dialect/LLVMIR/LLVMOpBase.td" -include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td" include "mlir/Interfaces/InferTypeOpInterface.td" // Operations that correspond to LLVM intrinsics. With MLIR operation set being @@ -130,25 +129,42 @@ def LLVM_UMaxOp : LLVM_BinarySameArgsIntrOpI<"umax">; def LLVM_UMinOp : LLVM_BinarySameArgsIntrOpI<"umin">; -def LLVM_MemcpyOp : LLVM_ZeroResultIntrOp<"memcpy", [0, 1, 2]> { - let arguments = (ins Arg:$dst, - Arg:$src, - AnySignlessInteger:$len, I1:$isVolatile); -} -def LLVM_MemcpyInlineOp : LLVM_ZeroResultIntrOp<"memcpy.inline", [0, 1, 2]> { - let arguments = (ins Arg:$dst, - Arg:$src, - AnySignlessInteger:$len, I1:$isVolatile); -} -def LLVM_MemmoveOp : LLVM_ZeroResultIntrOp<"memmove", [0, 1, 2]> { - let arguments = (ins Arg:$dst, - Arg:$src, - AnySignlessInteger:$len, I1:$isVolatile); -} - -def LLVM_MemsetOp : LLVM_ZeroResultIntrOp<"memset", [0, 2]> { - let arguments = (ins Arg:$dst, - I8:$val, AnySignlessInteger:$len, I1:$isVolatile); +class LLVM_MemcpyIntrOpBase : + LLVM_ZeroResultIntrOp { + dag args = (ins Arg:$dst, + Arg:$src, + AnySignlessInteger:$len, I1:$isVolatile); + // Append the alias attributes defined by LLVM_IntrOpBase. + let arguments = !con(args, aliasAttrs); + let builders = [ + OpBuilder<(ins "Value":$dst, "Value":$src, "Value":$len, + "Value":$isVolatile), [{ + build($_builder, $_state, dst, src, len, isVolatile, + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); + }] + >]; +} + +def LLVM_MemcpyOp : LLVM_MemcpyIntrOpBase<"memcpy">; +def LLVM_MemcpyInlineOp : LLVM_MemcpyIntrOpBase<"memcpy.inline">; +def LLVM_MemmoveOp : LLVM_MemcpyIntrOpBase<"memmove">; + +def LLVM_MemsetOp : LLVM_ZeroResultIntrOp<"memset", [0, 2], [], + /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> { + dag args = (ins Arg:$dst, + I8:$val, AnySignlessInteger:$len, I1:$isVolatile); + // Append the alias attributes defined by LLVM_IntrOpBase. + let arguments = !con(args, aliasAttrs); + let builders = [ + OpBuilder<(ins "Value":$dst, "Value":$val, "Value":$len, + "Value":$isVolatile), [{ + build($_builder, $_state, dst, val, len, isVolatile, + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); + }] + >]; } //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -426,18 +426,36 @@ // intrinsic and "enumName" contains the name of the intrinsic as appears in // `llvm::Intrinsic` enum; one usually wants these to be related. Additionally, // the base class also defines the "mlirBuilder" field to support the inverse -// translation starting from an LLVM IR intrinsic. +// translation starting from an LLVM IR intrinsic. The "requiresAccessGroup", +// "requiresAliasAnalysis", and "requiresFastmath" flags specify which +// interfaces the intrinsic implements. If the corresponding flags are set, the +// "aliasAttrs" list contains the arguments required by the access group and +// alias analysis interfaces. Derived intrinsics should append the "aliasAttrs" +// to their argument list if they set one of the flags. class LLVM_IntrOpBase overloadedResults, list overloadedOperands, list traits, int numResults, - bit requiresAccessGroup = 0, bit requiresAliasScope = 0, + bit requiresAccessGroup = 0, bit requiresAliasAnalysis = 0, bit requiresFastmath = 0> : LLVM_OpBase], []), + !if(!gt(requiresAliasAnalysis, 0), + [DeclareOpInterfaceMethods], []), !if(!gt(requiresFastmath, 0), - [DeclareOpInterfaceMethods], - []), + [DeclareOpInterfaceMethods], []), traits)>, + LLVM_MemOpPatterns, Results { + dag aliasAttrs = !con( + !if(!gt(requiresAccessGroup, 0), + (ins OptionalAttr:$access_groups), + (ins )), + !if(!gt(requiresAccessGroup, 0), + (ins OptionalAttr:$alias_scopes, + OptionalAttr:$noalias_scopes, + OptionalAttr:$tbaa), + (ins ))); string resultPattern = !if(!gt(numResults, 1), LLVM_IntrPatterns.structResult, LLVM_IntrPatterns.result); @@ -453,14 +471,12 @@ overloadedOperands>.lst), ", ") # [{ }); auto operands = moduleTranslation.lookupValues(opInst.getOperands()); - }] # [{auto *inst = builder.CreateCall(fn, operands); - }] # !if(!gt(requiresAccessGroup, 0), - "moduleTranslation.setAccessGroupsMetadata(op, inst);", - "(void) inst;") - # !if(!gt(requiresAliasScope, 0), - "moduleTranslation.setAliasScopeMetadata(op, inst);", - "(void) inst;") - # !if(!gt(numResults, 0), "$res = inst;", ""); + }] # [{ + auto *inst = builder.CreateCall(fn, operands); + (void) inst; + }] # !if(!gt(requiresAccessGroup, 0), setAccessGroupsMetadataCode, "") + # !if(!gt(requiresAliasAnalysis, 0), setAliasAnalysisMetadataCode, "") + # !if(!gt(numResults, 0), "$res = inst;", ""); string mlirBuilder = [{ FailureOr> mlirOperands = @@ -481,10 +497,10 @@ class LLVM_IntrOp overloadedResults, list overloadedOperands, list traits, int numResults, bit requiresAccessGroup = 0, - bit requiresAliasScope = 0, bit requiresFastmath = 0> + bit requiresAliasAnalysis = 0, bit requiresFastmath = 0> : LLVM_IntrOpBase; // Base class for LLVM intrinsic operations returning no results. Places the @@ -502,8 +518,11 @@ // one (as indicated by `[0]`) is necessary to resolve the overloaded intrinsic. // The Op has no results. class LLVM_ZeroResultIntrOp overloadedOperands = [], - list traits = []> - : LLVM_IntrOp; + list traits = [], + bit requiresAccessGroup = 0, + bit requiresAliasAnalysis = 0> + : LLVM_IntrOp; // Base class for LLVM intrinsic operations returning one result. Places the // intrinsic into the LLVM dialect and prefixes its name with "intr.". This is @@ -516,7 +535,7 @@ list traits = [], bit requiresFastmath = 0> : LLVM_IntrOp; def LLVM_OneResultOpBuilder : diff --git a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll --- a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll @@ -72,9 +72,16 @@ %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !alias.scope !3 ; CHECK: llvm.cmpxchg {{.*}}alias_scopes = %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !alias.scope !3 + ; CHECK: "llvm.intr.memcpy"{{.*}}alias_scopes = + call void @llvm.memcpy.p0.p0.i32(ptr %arg1, ptr %arg1, i32 4, i1 false), !alias.scope !3 + ; CHECK: "llvm.intr.memset"{{.*}}alias_scopes = + call void @llvm.memset.p0.i32(ptr %arg1, i8 42, i32 4, i1 false), !alias.scope !3 ret void } +declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) +declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) + !0 = distinct !{!0, !"The domain"} !1 = distinct !{!1} !2 = !{!2, !0} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -37,9 +37,16 @@ %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !llvm.access.group !0 ; CHECK: llvm.cmpxchg {{.*}}access_groups = %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !llvm.access.group !0 + ; CHECK: "llvm.intr.memcpy"{{.*}}access_groups = + call void @llvm.memcpy.p0.p0.i32(ptr %arg1, ptr %arg1, i32 4, i1 false), !llvm.access.group !0 + ; CHECK: "llvm.intr.memset"{{.*}}access_groups = + call void @llvm.memset.p0.i32(ptr %arg1, i8 42, i32 4, i1 false), !llvm.access.group !0 ret void } +declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) +declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) + !0 = !{!1, !2} !1 = distinct !{} !2 = distinct !{} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll --- a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll @@ -81,9 +81,16 @@ %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !tbaa !0 ; CHECK: llvm.cmpxchg {{.*}}tbaa = %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !tbaa !0 + ; CHECK: "llvm.intr.memcpy"{{.*}}tbaa = + call void @llvm.memcpy.p0.p0.i32(ptr %arg1, ptr %arg1, i32 4, i1 false), !tbaa !0 + ; CHECK: "llvm.intr.memset"{{.*}}tbaa = + call void @llvm.memset.p0.i32(ptr %arg1, i8 42, i32 4, i1 false), !tbaa !0 ret void } +declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) +declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) + !0 = !{!1, !1, i64 0} !1 = !{!"scalar type", !2, i64 0} !2 = !{!"Simple C/C++ TBAA"} diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1981,6 +1981,10 @@ %1 = llvm.load %arg1 {alias_scopes = [@metadata::@scope2], noalias_scopes = [@metadata::@scope1, @metadata::@scope3]} : !llvm.ptr -> i32 %2 = llvm.atomicrmw add %arg1, %0 monotonic {alias_scopes = [@metadata::@scope3], noalias_scopes = [@metadata::@scope1, @metadata::@scope2]} : !llvm.ptr, i32 %3 = llvm.cmpxchg %arg1, %1, %2 acq_rel monotonic {alias_scopes = [@metadata::@scope3]} : !llvm.ptr, i32 + %4 = llvm.mlir.constant(0 : i1) : i1 + %5 = llvm.mlir.constant(42 : i8) : i8 + "llvm.intr.memcpy"(%arg1, %arg1, %0, %4) {alias_scopes = [@metadata::@scope3]} : (!llvm.ptr, !llvm.ptr, i32, i1) -> () + "llvm.intr.memset"(%arg1, %5, %0, %4) {noalias_scopes = [@metadata::@scope3]} : (!llvm.ptr, i8, i32, i1) -> () llvm.return } @@ -1998,6 +2002,8 @@ // CHECK: load {{.*}}, !alias.scope ![[SCOPES2:[0-9]+]], !noalias ![[SCOPES13:[0-9]+]] // CHECK: atomicrmw {{.*}}, !alias.scope ![[SCOPES3:[0-9]+]], !noalias ![[SCOPES12:[0-9]+]] // CHECK: cmpxchg {{.*}}, !alias.scope ![[SCOPES3]] +// CHECK: llvm.memcpy{{.*}}, !alias.scope ![[SCOPES3]] +// CHECK: llvm.memset{{.*}}, !noalias ![[SCOPES3]] // Metadata // CHECK-DAG: ![[DOMAIN:[0-9]+]] = distinct !{![[DOMAIN]], !"The domain"} diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir --- a/mlir/test/Target/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -256,6 +256,12 @@ %6 = llvm.atomicrmw add %4, %5 monotonic {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr, i32 // CHECK: = cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] %7 = llvm.cmpxchg %4, %5, %6 acq_rel monotonic {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr, i32 + %8 = llvm.mlir.constant(0 : i1) : i1 + %9 = llvm.mlir.constant(42 : i8) : i8 + // CHECK: llvm.memcpy{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + "llvm.intr.memcpy"(%4, %4, %0, %8) {access_groups = [@metadata::@group1, @metadata::@group2]} : (!llvm.ptr, !llvm.ptr, i32, i1) -> () + // CHECK: llvm.memset{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + "llvm.intr.memset"(%4, %9, %0, %8) {access_groups = [@metadata::@group1, @metadata::@group2]} : (!llvm.ptr, i8, i32, i1) -> () // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] llvm.br ^bb3(%3 : i32) {loop_annotation = #llvm.loop_annotation< licm = , diff --git a/mlir/test/Target/LLVMIR/tbaa.mlir b/mlir/test/Target/LLVMIR/tbaa.mlir --- a/mlir/test/Target/LLVMIR/tbaa.mlir +++ b/mlir/test/Target/LLVMIR/tbaa.mlir @@ -69,6 +69,12 @@ %6 = llvm.atomicrmw add %5, %4 monotonic {tbaa = [@__tbaa::@tbaa_tag_7]} : !llvm.ptr, i32 // CHECK: cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} !tbaa ![[STAG]] %7 = llvm.cmpxchg %5, %6, %4 acq_rel monotonic {tbaa = [@__tbaa::@tbaa_tag_7]} : !llvm.ptr, i32 + %8 = llvm.mlir.constant(0 : i1) : i1 + %9 = llvm.mlir.constant(42 : i8) : i8 + // CHECK: llvm.memcpy{{.*}} !tbaa ![[STAG]] + "llvm.intr.memcpy"(%arg1, %arg1, %0, %8) {tbaa = [@__tbaa::@tbaa_tag_7]} : (!llvm.ptr, !llvm.ptr, i32, i1) -> () + // CHECK: llvm.memset{{.*}} !tbaa ![[STAG]] + "llvm.intr.memset"(%arg1, %9, %0, %8) {tbaa = [@__tbaa::@tbaa_tag_7]} : (!llvm.ptr, i8, i32, i1) -> () llvm.return } } diff --git a/mlir/test/mlir-tblgen/llvm-intrinsics.td b/mlir/test/mlir-tblgen/llvm-intrinsics.td --- a/mlir/test/mlir-tblgen/llvm-intrinsics.td +++ b/mlir/test/mlir-tblgen/llvm-intrinsics.td @@ -24,9 +24,9 @@ // CHECK: [NoMemoryEffect] // It has a result. // CHECK: 1, -// It does not require an access group. +// It does not implement the access group interface. // CHECK: 0, -// It does not require alias scopes. +// It does not implement the alias analysis interface. // CHECK: 0> // CHECK: Arguments<(ins LLVM_Type, LLVM_Type @@ -45,20 +45,20 @@ // GROUPS: [NoMemoryEffect] // It has a result. // GROUPS: 1, -// It requires generation of an access group LLVM metadata. +// It implements the access group interface. // GROUPS: 1, -// It does not require alias scopes. +// It does not implement the alias analysis interface. // GROUPS: 0> // It has an access group attribute. // GROUPS: OptionalAttr:$access_groups //---------------------------------------------------------------------------// -// This checks that we can define an op that takes in alias scopes metadata. +// This checks that we can define an op that takes in alias analysis metadata. // // RUN: cat %S/../../../llvm/include/llvm/IR/Intrinsics.td \ // RUN: | grep -v "llvm/IR/Intrinsics" \ -// RUN: | mlir-tblgen -gen-llvmir-intrinsics -I %S/../../../llvm/include/ --llvmir-intrinsics-filter=ptrmask --llvmir-intrinsics-alias-scopes-regexp=ptrmask \ +// RUN: | mlir-tblgen -gen-llvmir-intrinsics -I %S/../../../llvm/include/ --llvmir-intrinsics-filter=ptrmask --llvmir-intrinsics-alias-analysis-regexp=ptrmask \ // RUN: | FileCheck --check-prefix=ALIAS %s // ALIAS-LABEL: def LLVM_ptrmask @@ -67,13 +67,14 @@ // ALIAS: [NoMemoryEffect] // It has a result. // ALIAS: 1, -// It does not require an access group. +// It does not implement the access group interface. // ALIAS: 0, -// It requires generation of alias scopes LLVM metadata. +// It implements the alias analysis interface. // ALIAS: 1> -// It has alias scopes and noalias. +// It has alias scopes, noalias, and tbaa. // ALIAS: OptionalAttr:$alias_scopes // ALIAS: OptionalAttr:$noalias_scopes +// ALIAS: OptionalAttr:$tbaa //---------------------------------------------------------------------------// diff --git a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp --- a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp +++ b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp @@ -44,10 +44,10 @@ "regexp as taking an access group metadata"), llvm::cl::cat(intrinsicGenCat)); -static llvm::cl::opt aliasScopesRegexp( - "llvmir-intrinsics-alias-scopes-regexp", +static llvm::cl::opt aliasAnalysisRegexp( + "llvmir-intrinsics-alias-analysis-regexp", llvm::cl::desc("Mark intrinsics that match the specified " - "regexp as taking alias.scopes and noalias metadata"), + "regexp as taking alias.scopes, noalias, and tbaa metadata"), llvm::cl::cat(intrinsicGenCat)); // Used to represent the indices of overloadable operands/results. @@ -202,9 +202,9 @@ bool requiresAccessGroup = !accessGroupRegexp.empty() && accessGroupMatcher.match(record.getName()); - llvm::Regex aliasScopesMatcher(aliasScopesRegexp); - bool requiresAliasScopes = - !aliasScopesRegexp.empty() && aliasScopesMatcher.match(record.getName()); + llvm::Regex aliasAnalysisMatcher(aliasAnalysisRegexp); + bool requiresAliasAnalysis = !aliasAnalysisRegexp.empty() && + aliasAnalysisMatcher.match(record.getName()); // Prepare strings for traits, if any. llvm::SmallVector traits; @@ -218,9 +218,10 @@ "LLVM_Type"); if (requiresAccessGroup) operands.push_back("OptionalAttr:$access_groups"); - if (requiresAliasScopes) { + if (requiresAliasAnalysis) { operands.push_back("OptionalAttr:$alias_scopes"); operands.push_back("OptionalAttr:$noalias_scopes"); + operands.push_back("OptionalAttr:$tbaa"); } // Emit the definition. @@ -233,7 +234,7 @@ printBracketedRange(traits, os); os << ", " << intr.getNumResults() << ", " << (requiresAccessGroup ? "1" : "0") << ", " - << (requiresAliasScopes ? "1" : "0") << ">, Arguments<(ins" + << (requiresAliasAnalysis ? "1" : "0") << ">, Arguments<(ins" << (operands.empty() ? "" : " "); llvm::interleaveComma(operands, os); os << ")>;\n\n";