diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -19603,6 +19603,82 @@ This function returns the same values as the libm ``trunc`` functions would and handles error conditions in the same way. +.. _int_experimental_noalias_scope_decl: + +'``llvm.experimental.noalias.scope.decl``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + + +:: + + declare void @llvm.experimental.noalias.scope.decl(metadata !id.scope.list) + +Overview: +""""""""" + +The ``llvm.experimental.noalias.scope.decl`` intrinsic identifies where a +noalias scope is declared. When the intrinsic is duplicated, a decision must +also be made about the scope: depending on the reason of the duplication, +the scope might need to be duplicated as well. + + +Arguments: +"""""""""" + +The ``!id.scope.list`` argument is metadata that is a list of ``noalias`` +metadata references. The format is identical to that required for ``noalias`` +metadata. This list must have exactly one element. + +Semantics: +"""""""""" + +The ``llvm.experimental.noalias.scope.decl`` intrinsic identifies where a +noalias scope is declared. When the intrinsic is duplicated, a decision must +also be made about the scope: depending on the reason of the duplication, +the scope might need to be duplicated as well. + +For example, when the intrinsic is used inside a loop body, and that loop is +unrolled, the associated noalias scope must also be duplicated. Otherwise, the +noalias property it signifies would spill across loop iterations, whereas it +was only valid within a single iteration. + +.. code-block:: llvm + + ; This examples shows two possible positions for noalias.decl and how they impact the semantics: + ; If it is outside the loop (Version 1), then %a and %b are noalias across *all* iterations. + ; If it is inside the loop (Version 2), then %a and %b are noalias only within *one* iteration. + declare void @decl_in_loop(i8* %a.base, i8* %b.base) { + entry: + ; call void @llvm.experimental.noalias.scope.decl(metadata !2) ; Version 1: noalias decl outside loop + br label %loop + + loop: + %a = phi i8* [ %a.base, %entry ], [ %a.inc, %loop ] + %b = phi i8* [ %b.base, %entry ], [ %b.inc, %loop ] + ; call void @llvm.experimental.noalias.scope.decl(metadata !2) ; Version 2: noalias decl inside loop + %val = load i8, i8* %a, !alias.scope !2 + store i8 %val, i8* %b, !noalias !2 + %a.inc = getelementptr inbounds i8, i8* %a, i64 1 + %b.inc = getelementptr inbounds i8, i8* %b, i64 1 + %cond = call i1 @cond() + br i1 %cond, label %loop, label %exit + + exit: + ret void + } + + !0 = !{!0} ; domain + !1 = !{!1, !0} ; scope + !2 = !{!1} ; scope list + +Multiple calls to `@llvm.experimental.noalias.scope.decl` for the same scope +are possible, but one should never dominate another. Violations are pointed out +by the verifier as they indicate a problem in either a transformation pass or +the input. + Floating Point Environment Manipulation intrinsics -------------------------------------------------- diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -858,6 +858,13 @@ CallInst *CreateAssumption(Value *Cond, ArrayRef OpBundles = llvm::None); + /// Create a llvm.experimental.noalias.scope.decl intrinsic call. + Instruction *CreateNoAliasScopeDeclaration(Value *Scope); + Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) { + return CreateNoAliasScopeDeclaration( + MetadataAsValue::get(Context, ScopeTag)); + } + /// Create a call to the experimental.gc.statepoint intrinsic to /// start a new statepoint sequence. CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -34,6 +34,9 @@ /// function known by LLVM. The enum values are returned by /// Function::getIntrinsicID(). namespace Intrinsic { + // Abstraction for the arguments of the noalias intrinsics + static const int NoAliasScopeDeclScopeArg = 0; + // Intrinsic ID type. This is an opaque typedef to facilitate splitting up // the enum into target-specific enums. typedef unsigned ID; diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -539,6 +539,16 @@ def int_assume : DefaultAttrsIntrinsic<[], [llvm_i1_ty], [IntrWillReturn, NoUndef>]>; +// 'llvm.experimental.noalias.scope.decl' intrinsic: Inserted at the location of +// noalias scope declaration. Makes it possible to identify that a noalias scope +// is only valid inside the body of a loop. +// +// Purpose of the different arguments: +// - arg0: id.scope: metadata representing the scope declaration. +def int_experimental_noalias_scope_decl + : DefaultAttrsIntrinsic<[], [llvm_metadata_ty], + [IntrInaccessibleMemOnly]>; // blocks LICM and some more + // Stack Protector Intrinsic - The stackprotector intrinsic writes the stack // guard to the correct place on the stack frame. def int_stackprotector : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptrptr_ty], []>; diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp --- a/llvm/lib/CodeGen/IntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp @@ -329,6 +329,7 @@ break; case Intrinsic::assume: + case Intrinsic::experimental_noalias_scope_decl: case Intrinsic::var_annotation: break; // Strip out these intrinsics diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1252,6 +1252,8 @@ case Intrinsic::sideeffect: // Neither does the assume intrinsic; it's also OK not to codegen its operand. case Intrinsic::assume: + // Neither does the llvm.experimental.noalias.scope.decl intrinsic + case Intrinsic::experimental_noalias_scope_decl: return true; case Intrinsic::dbg_declare: { const DbgDeclareInst *DI = cast(II); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -6466,10 +6466,13 @@ // Drop the intrinsic, but forward the value setValue(&I, getValue(I.getOperand(0))); return; + case Intrinsic::assume: + case Intrinsic::experimental_noalias_scope_decl: case Intrinsic::var_annotation: case Intrinsic::sideeffect: - // Discard annotate attributes, assumptions, and artificial side-effects. + // Discard annotate attributes, noalias scope declarations, assumptions, and + // artificial side-effects. return; case Intrinsic::codeview_annotation: { diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -452,6 +452,13 @@ return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles); } +Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) { + Module *M = BB->getModule(); + auto *FnIntrinsic = Intrinsic::getDeclaration( + M, Intrinsic::experimental_noalias_scope_decl, {}); + return createCallHelper(FnIntrinsic, {Scope}, this); +} + /// Create a call to a Masked Load intrinsic. /// \p Ptr - base pointer for the load /// \p Alignment - alignment of the source location diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -115,6 +115,11 @@ using namespace llvm; +static cl::opt VerifyNoAliasScopeDomination( + "verify-noalias-scope-decl-dom", cl::Hidden, cl::init(false), + cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical " + "scopes are not dominating")); + namespace llvm { struct VerifierSupport { @@ -313,6 +318,8 @@ TBAAVerifier TBAAVerifyHelper; + SmallVector NoAliasScopeDecls; + void checkAtomicMemAccessSize(Type *Ty, const Instruction *I); public: @@ -360,6 +367,8 @@ LandingPadResultTy = nullptr; SawFrameEscape = false; SiblingFuncletInfo.clear(); + verifyNoAliasScopeDecl(); + NoAliasScopeDecls.clear(); return !Broken; } @@ -536,6 +545,9 @@ /// Verify all-or-nothing property of DIFile source attribute within a CU. void verifySourceDebugInfo(const DICompileUnit &U, const DIFile &F); + + /// Verify the llvm.experimental.noalias.scope.decl declarations + void verifyNoAliasScopeDecl(); }; } // end anonymous namespace @@ -5163,6 +5175,10 @@ &Call); break; } + case Intrinsic::experimental_noalias_scope_decl: { + NoAliasScopeDecls.push_back(cast(&Call)); + break; + } }; } @@ -5513,6 +5529,76 @@ "inconsistent use of embedded source"); } +void Verifier::verifyNoAliasScopeDecl() { + if (NoAliasScopeDecls.empty()) + return; + + // only a single scope must be declared at a time. + for (auto *II : NoAliasScopeDecls) { + assert(II->getIntrinsicID() == Intrinsic::experimental_noalias_scope_decl && + "Not a llvm.experimental.noalias.scope.decl ?"); + const auto *ScopeListMV = dyn_cast( + II->getOperand(Intrinsic::NoAliasScopeDeclScopeArg)); + Assert(ScopeListMV != nullptr, + "llvm.experimental.noalias.scope.decl must have a MetadataAsValue " + "argument", + II); + + const auto *ScopeListMD = dyn_cast(ScopeListMV->getMetadata()); + Assert(ScopeListMD != nullptr, "!id.scope.list must point to an MDNode", + II); + Assert(ScopeListMD->getNumOperands() == 1, + "!id.scope.list must point to a list with a single scope", II); + } + + // Only check the domination rule when requested. Once all passes have been + // adapted this option can go away. + if (!VerifyNoAliasScopeDomination) + return; + + // Now sort the intrinsics based on the scope MDNode so that declarations of + // the same scopes are next to each other. + auto GetScope = [](IntrinsicInst *II) { + const auto *ScopeListMV = cast( + II->getOperand(Intrinsic::NoAliasScopeDeclScopeArg)); + return &cast(ScopeListMV->getMetadata())->getOperand(0); + }; + + // We are sorting on MDNode pointers here. For valid input IR this is ok. + // TODO: Sort on Metadata ID to avoid non-deterministic error messages. + auto Compare = [GetScope](IntrinsicInst *Lhs, IntrinsicInst *Rhs) { + return GetScope(Lhs) < GetScope(Rhs); + }; + + llvm::sort(NoAliasScopeDecls, Compare); + + // Go over the intrinsics and check that for the same scope, they are not + // dominating each other. + auto ItCurrent = NoAliasScopeDecls.begin(); + while (ItCurrent != NoAliasScopeDecls.end()) { + auto CurScope = GetScope(*ItCurrent); + auto ItNext = ItCurrent; + do { + ++ItNext; + } while (ItNext != NoAliasScopeDecls.end() && + GetScope(*ItNext) == CurScope); + + // [ItCurrent, ItNext[ represents the declarations for the same scope. + // Ensure they are not dominating each other + for (auto *I : llvm::make_range(ItCurrent, ItNext)) { + for (auto *J : llvm::make_range(ItCurrent, ItNext)) { + if (I != J) { + Assert(!DT.dominates(I, J), + "llvm.experimental.noalias.scope.decl dominates another one " + "with the same scope", + I); + } + } + } + ItCurrent = ItNext; + } +} + //===----------------------------------------------------------------------===// // Implement the public interfaces to this file... //===----------------------------------------------------------------------===// diff --git a/llvm/test/Verifier/noalias_scope_decl.ll b/llvm/test/Verifier/noalias_scope_decl.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Verifier/noalias_scope_decl.ll @@ -0,0 +1,61 @@ +; RUN: not llvm-as -disable-output --verify-noalias-scope-decl-dom < %s 2>&1 | FileCheck %s + +define void @test_single_scope01() nounwind ssp { + tail call void @llvm.experimental.noalias.scope.decl(metadata !2) + ret void +} + +define void @test_single_scope02() nounwind ssp { + tail call void @llvm.experimental.noalias.scope.decl(metadata !5) + ret void +} +; CHECK: !id.scope.list must point to a list with a single scope +; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata !5) + +define void @test_single_scope03() nounwind ssp { + tail call void @llvm.experimental.noalias.scope.decl(metadata !"test") + ret void +} +; CHECK-NEXT: !id.scope.list must point to an MDNode +; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata !"test") + +define void @test_dom01() nounwind ssp { + tail call void @llvm.experimental.noalias.scope.decl(metadata !2) + tail call void @llvm.experimental.noalias.scope.decl(metadata !8) + ret void +} + +define void @test_dom02() nounwind ssp { + tail call void @llvm.experimental.noalias.scope.decl(metadata !2) + tail call void @llvm.experimental.noalias.scope.decl(metadata !6) + ret void +} +; CHECK-NEXT: llvm.experimental.noalias.scope.decl dominates another one with the same scope +; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata !2) + +define void @test_dom03() nounwind ssp { + tail call void @llvm.experimental.noalias.scope.decl(metadata !2) + tail call void @llvm.experimental.noalias.scope.decl(metadata !2) + ret void +} +; CHECK-NEXT: llvm.experimental.noalias.scope.decl dominates another one with the same scope +; CHECK-NEXT: tail call void @llvm.experimental.noalias.scope.decl(metadata !2) + +; CHECK-NOT: llvm.experimental.noalias.scope.decl + +; Function Attrs: inaccessiblememonly nounwind +declare void @llvm.experimental.noalias.scope.decl(metadata) #1 + +attributes #1 = { inaccessiblememonly nounwind } +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang"} +!2 = !{!3} +!3 = distinct !{!3, !4, !"test: pA"} +!4 = distinct !{!4, !"test"} +!5 = !{!3, !3} +!6 = !{!3} +!7 = distinct !{!7, !4, !"test: pB"} +!8 = !{!7}