diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -2318,6 +2318,26 @@ :ref:`stackmap entry `. See the intrinsic description for further details. + +.. _nocapture_use: + +No Capture Use Operand Bundles + +A "nocapture_use" operand bundle indicates that the instruction uses the +operands of the operand bundle but does not capture them. It can be used in +combination with the ``!nocapture`` metadata on stores to express that a +pointer is stored into memory and passed to a function without being captured. +That is, the pointer store is not capturing the pointer, nor is the use of the +memory in the instruction with the "nocapture_use" operand bundle but the +indirection via memory is only needed for ABI reasons. The reason the +"nocapture_use" is required is that the instruction annotated with it might +still read or write the pointer, a fact that was prior implied by passing the +memory the pointer was stored to. Since the store is not capturing the pointer +anymore, due to ``!nocapture``, we need to make the potential use explicit. +Note that other uses of the pointer might be captured by the instruction +annotated with "nocapture_use". + + .. _moduleasm: Module-Level Inline Assembly @@ -9427,7 +9447,7 @@ :: store [volatile] , * [, align ][, !nontemporal !][, !invariant.group !] ; yields void - store atomic [volatile] , * [syncscope("")] , align [, !invariant.group !] ; yields void + store atomic [volatile] , * [syncscope("")] , align [, !invariant.group !] [, !nocapture !] ; yields void ! = !{ i32 1 } ! = !{} @@ -9486,6 +9506,18 @@ The optional ``!invariant.group`` metadata must reference a single metadata name ````. See ``invariant.group`` metadata. +The optional ``!nocapture`` metadata must reference a single metadata name +```` corresponding to a node with no entries. The existence of +``!nocapture`` metadata on the instruction tells the optimizer that the pointer +stored is not captured in the sense that all uses of the pointer are explicitly +marked otherwise and the storing can be ignored during capture analysis. +The ``!nocapture`` metadata can be used with the :ref:`"nocapture_use" operand +bundle ` to indicate a store is a necessasity, e.g., of a given +ABI, but the user of the memory the pointer is stored into is only using the +pointer value without capturing it. If a pointer stored via an instruction +with the ``!nocpture`` metadata is loaded from that memory and captured, the +behavior is undefined. + Semantics: """""""""" diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -42,3 +42,4 @@ LLVM_FIXED_MD_KIND(MD_vcall_visibility, "vcall_visibility", 28) LLVM_FIXED_MD_KIND(MD_noundef, "noundef", 29) LLVM_FIXED_MD_KIND(MD_annotation, "annotation", 30) +LLVM_FIXED_MD_KIND(MD_nocapture, "nocapture", 31) diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp --- a/llvm/lib/Analysis/CaptureTracking.cpp +++ b/llvm/lib/Analysis/CaptureTracking.cpp @@ -26,6 +26,7 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/Support/CommandLine.h" using namespace llvm; @@ -314,9 +315,15 @@ // (think of self-referential objects). if (Call->isDataOperand(U) && !Call->doesNotCapture(Call->getDataOperandNo(U))) { - // The parameter is not marked 'nocapture' - captured. - if (Tracker->captured(U)) - return; + // The parameter is not marked 'nocapture', now we check for the + // nocapture_use operand bundle use which, as the name suggests, does + // not capture the pointer. + if (!Call->isBundleOperand(U) || + Call->getBundleOpInfoForOperand(Call->getDataOperandNo(U)) + .Tag->getKey() != "nocapture_use") { + if (Tracker->captured(U)) + return; + } } break; } @@ -331,10 +338,13 @@ break; case Instruction::Store: // Stored the pointer - conservatively assume it may be captured. - // Volatile stores make the address observable. + // Volatile stores make the address observable. We do however ignore + // stores with the !nocapture metadata as it guarantees the pointer + // is not captured (in any way). if (U->getOperandNo() == 0 || cast(I)->isVolatile()) - if (Tracker->captured(U)) - return; + if (!cast(I)->hasMetadata(LLVMContext::MD_nocapture)) + if (Tracker->captured(U)) + return; break; case Instruction::AtomicRMW: { // atomicrmw conceptually includes both a load and store from diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -428,6 +428,7 @@ void visitDereferenceableMetadata(Instruction &I, MDNode *MD); void visitProfMetadata(Instruction &I, MDNode *MD); void visitAnnotationMetadata(MDNode *Annotation); + void visitNocaptureMetadata(Instruction &I, MDNode *Nocapture); template bool isValidMetadataArray(const MDTuple &N); #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N); @@ -4288,6 +4289,13 @@ Assert(isa(Op.get()), "operands must be strings"); } +void Verifier::visitNocaptureMetadata(Instruction &I, MDNode *Nocapture) { + Assert(isa(Nocapture), "nocapture must be a tuple"); + Assert(Nocapture->getNumOperands() == 0, + "nocaptrue must have at least one operand"); + Assert(isa(I), "nocapture must be attached to a store"); +} + /// verifyInstruction - Verify that an instruction is well formed. /// void Verifier::visitInstruction(Instruction &I) { @@ -4451,6 +4459,9 @@ if (MDNode *Annotation = I.getMetadata(LLVMContext::MD_annotation)) visitAnnotationMetadata(Annotation); + if (MDNode *Nocapture = I.getMetadata(LLVMContext::MD_nocapture)) + visitNocaptureMetadata(I, Nocapture); + if (MDNode *N = I.getDebugLoc().getAsMDNode()) { AssertDI(isa(N), "invalid !dbg metadata attachment", &I, N); visitMDNode(*N, AreDebugLocsAllowed::Yes); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -512,6 +512,7 @@ case LLVMContext::MD_nontemporal: case LLVMContext::MD_mem_parallel_loop_access: case LLVMContext::MD_access_group: + case LLVMContext::MD_nocapture: // All of these directly apply. NewStore->setMetadata(ID, N); break; diff --git a/llvm/test/Transforms/InstCombine/nocapture_use.ll b/llvm/test/Transforms/InstCombine/nocapture_use.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/nocapture_use.ll @@ -0,0 +1,85 @@ +; RUN: opt -instcombine -S < %s | FileCheck %s + +%struct.S = type { i32* } + +define i32 @base_negative() { +; The base case, we cannot propagate 1 to the return. +; CHECK: @base_negative +; CHECK: ret i32 %tmp2 + + %local = alloca i32 + %s = alloca %struct.S + %tmp = bitcast i32* %local to i8* + %ptr = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 0 + store i32* %local, i32** %ptr + %call = call i32* @unknown_nocapture_pointer_in_arg(%struct.S* nonnull %s) + store i32 1, i32* %local + store i32 2, i32* %call + %tmp2 = load i32, i32* %local + ret i32 %tmp2 +} + +define i32 @nocapture_late_positive() { +; The nocapture case, store late, we can propagate 1 to the return. +; CHECK: @nocapture_late_positive +; CHECK: ret i32 1 + + %local = alloca i32 + %s = alloca %struct.S + %tmp = bitcast i32* %local to i8* + %ptr = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 0 + store i32* %local, i32** %ptr, !nocapture !0 + %call = call i32* @unknown_nocapture_pointer_in_arg(%struct.S* nonnull %s) ["nocapture_use"(i32* %local)] + store i32 1, i32* %local + store i32 2, i32* %call + %tmp2 = load i32, i32* %local + ret i32 %tmp2 +} + +define i32 @nocapture_early_negative() { +; The nocapture case, store early, we cannot propagate 1 to the return. +; +; The purpose of this test is to confirm that the use in +; @unknown_nocapture_pointer_in_arg can still overwrite +; the value stored in %local even if the pointer is not captured. +; +; CHECK: @nocapture_early_negative +; CHECK: ret i32 %tmp2 + + %local = alloca i32 + %s = alloca %struct.S + %tmp = bitcast i32* %local to i8* + %ptr = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 0 + store i32* %local, i32** %ptr, !nocapture !0 + store i32 1, i32* %local + %call = call i32* @unknown_nocapture_pointer_in_arg(%struct.S* nonnull %s) ["nocapture_use"(i32* %local)] + store i32 2, i32* %call + %tmp2 = load i32, i32* %local + ret i32 %tmp2 +} + +define i32 @nocapture_early_positive_no_operand_bundle() { +; The nocapture case, store early, we can propagate 1 to the return. +; +; The purpose of this test is to confirm that it is the operand bundle +; use in the call to @unknown_nocapture_pointer_in_arg which can still +; overwrite the value stored in %local even if the pointer is not captured. +; +; CHECK: @nocapture_early_positive_no_operand_bundle +; CHECK: ret i32 1 + + %local = alloca i32 + %s = alloca %struct.S + %tmp = bitcast i32* %local to i8* + %ptr = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 0 + store i32* %local, i32** %ptr, !nocapture !0 + store i32 1, i32* %local + %call = call i32* @unknown_nocapture_pointer_in_arg(%struct.S* nonnull %s) + store i32 2, i32* %call + %tmp2 = load i32, i32* %local + ret i32 %tmp2 +} + +declare dso_local i32* @unknown_nocapture_pointer_in_arg(%struct.S*) + +!0 = !{} diff --git a/llvm/unittests/Analysis/CaptureTrackingTest.cpp b/llvm/unittests/Analysis/CaptureTrackingTest.cpp --- a/llvm/unittests/Analysis/CaptureTrackingTest.cpp +++ b/llvm/unittests/Analysis/CaptureTrackingTest.cpp @@ -20,7 +20,7 @@ TEST(CaptureTracking, MaxUsesToExplore) { StringRef Assembly = R"( ; Function Attrs: nounwind ssp uwtable - declare void @doesnt_capture(i8* nocapture, i8* nocapture, i8* nocapture, + declare void @doesnt_capture(i8* nocapture, i8* nocapture, i8* nocapture, i8* nocapture, i8* nocapture) ; %arg has 5 uses @@ -132,3 +132,83 @@ EXPECT_EQ(ICmp, CT.Captures[6]->getUser()); EXPECT_EQ(1u, CT.Captures[6]->getOperandNo()); } + +TEST(CaptureTracking, NoCaptureMetadata) { + StringRef Assembly = R"( + define void @test(i8* %arg, i8** %ptr) { + store i8* %arg, i8** %ptr, !nocapture !0 + ret void + } + + !0 = !{} + )"; + + LLVMContext Context; + SMDiagnostic Error; + auto M = parseAssemblyString(Assembly, Error, Context); + ASSERT_TRUE(M) << "Bad assembly?"; + + Function *F = M->getFunction("test"); + Value *Arg1 = F->getArg(0); + Value *Arg2 = F->getArg(1); + + CollectingCaptureTracker CT; + PointerMayBeCaptured(Arg1, &CT); + PointerMayBeCaptured(Arg2, &CT); + EXPECT_EQ(0u, CT.Captures.size()); +} + +TEST(CaptureTracking, NoCaptureUseOperandBundle) { + StringRef Assembly = R"( + declare void @u() + + define void @test(i8* %arg, i8** %ptr) { + call void @u() ["nocapture_use"(i8** %ptr), "unknown"(i8* %arg)] + ret void + } + )"; + + LLVMContext Context; + SMDiagnostic Error; + auto M = parseAssemblyString(Assembly, Error, Context); + ASSERT_TRUE(M) << "Bad assembly?"; + + Function *F = M->getFunction("test"); + Value *Arg1 = F->getArg(0); + Value *Arg2 = F->getArg(1); + + CollectingCaptureTracker CT; + PointerMayBeCaptured(Arg1, &CT); + PointerMayBeCaptured(Arg2, &CT); + ASSERT_EQ(1u, CT.Captures.size()); + EXPECT_EQ(Arg1, CT.Captures[0]->get()); +} + +TEST(CaptureTracking, NoCaptureMetadataAndNocaptureUseOperandBundle) { + StringRef Assembly = R"( + declare void @u(i8**) + + define void @test(i8* %arg, i8** %ptr) { + store i8* %arg, i8** %ptr, !nocapture !0 + call void @u(i8** %ptr) ["nocapture_use"(i8* %arg)] + ret void + } + + !0 = !{} + )"; + + LLVMContext Context; + SMDiagnostic Error; + auto M = parseAssemblyString(Assembly, Error, Context); + ASSERT_TRUE(M) << "Bad assembly?"; + + Function *F = M->getFunction("test"); + Value *Arg1 = F->getArg(0); + Value *Arg2 = F->getArg(1); + + CollectingCaptureTracker CT; + PointerMayBeCaptured(Arg1, &CT); + PointerMayBeCaptured(Arg2, &CT); + ASSERT_EQ(1u, CT.Captures.size()); + EXPECT_EQ(Arg2, CT.Captures[0]->get()); +}