Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -6766,7 +6766,7 @@ :: - = load [volatile] , * [, align ][, !nontemporal !][, !invariant.load !][, !invariant.group !][, !nonnull !][, !dereferenceable !][, !dereferenceable_or_null !][, !align !] + = load [volatile] , * [, align ][, !nontemporal !][, !invariant.load !][, !invariant.group !][, !nonnull !][, !dereferenceable !][, !unconditionally_dereferenceable !][, !dereferenceable_or_null !][, !align !] = load atomic [volatile] * [singlethread] , align [, !invariant.group !] ! = !{ i32 1 } ! = !{i64 } @@ -6838,7 +6838,20 @@ The optional ``!dereferenceable`` metadata must reference a single metadata name ```` corresponding to a metadata node with one ``i64`` entry. The existence of the ``!dereferenceable`` metadata on the instruction -tells the optimizer that the value loaded is known to be dereferenceable. +tells the optimizer that the value loaded is known to be dereferenceable, +possibly predicated on some condition that dominates the instruction. +The number of bytes known to be dereferenceable is specified by the integer +value in the metadata node. This is analogous to the ''dereferenceable'' +attribute on parameters and return values. This metadata can only be applied +to loads of a pointer type. + +The optional ``!unconditionally_dereferenceable`` metadata must reference +a single metadata name ```` corresponding to a metadata +node with one ``i64`` entry. The existence of the +``!unconditionally_dereferenceable`` metadata on the instruction tells +the optimizer that the value loaded is known to be dereferenceable regardless +of any conditions. This means that the loaded pointer stays dereferenceable +even if the load is speculated. The number of bytes known to be dereferenceable is specified by the integer value in the metadata node. This is analogous to the ''dereferenceable'' attribute on parameters and return values. This metadata can only be applied @@ -6848,7 +6861,8 @@ metadata name ```` corresponding to a metadata node with one ``i64`` entry. The existence of the ``!dereferenceable_or_null`` metadata on the instruction tells the optimizer that the value loaded is known to be either -dereferenceable or null. +dereferenceable or null, possibly predicated on some condition that dominates +the instruction. The number of bytes known to be dereferenceable is specified by the integer value in the metadata node. This is analogous to the ''dereferenceable_or_null'' attribute on parameters and return values. This metadata can only be applied Index: include/llvm/IR/LLVMContext.h =================================================================== --- include/llvm/IR/LLVMContext.h +++ include/llvm/IR/LLVMContext.h @@ -60,10 +60,11 @@ MD_nonnull = 11, // "nonnull" MD_dereferenceable = 12, // "dereferenceable" MD_dereferenceable_or_null = 13, // "dereferenceable_or_null" - MD_make_implicit = 14, // "make.implicit" - MD_unpredictable = 15, // "unpredictable" - MD_invariant_group = 16, // "invariant.group" - MD_align = 17 // "align" + MD_unconditionally_dereferenceable = 14, // "unconditionally_dereferenceable" + MD_make_implicit = 15, // "make.implicit" + MD_unpredictable = 16, // "unpredictable" + MD_invariant_group = 17, // "invariant.group" + MD_align = 18, // "align" }; /// Known operand bundle tag IDs, which always have the same value. All Index: lib/Analysis/ValueTracking.cpp =================================================================== --- lib/Analysis/ValueTracking.cpp +++ lib/Analysis/ValueTracking.cpp @@ -3156,12 +3156,18 @@ CheckForNonNull = true; } } else if (const LoadInst *LI = dyn_cast(BV)) { - if (MDNode *MD = LI->getMetadata(LLVMContext::MD_dereferenceable)) { + if (MDNode *MD = LI->getMetadata(LLVMContext::MD_unconditionally_dereferenceable)) { ConstantInt *CI = mdconst::extract(MD->getOperand(0)); DerefBytes = CI->getLimitedValue(); } if (!DerefBytes.getBoolValue()) { - if (MDNode *MD = + if (MDNode *MD = LI->getMetadata(LLVMContext::MD_dereferenceable)) { + ConstantInt *CI = mdconst::extract(MD->getOperand(0)); + DerefBytes = CI->getLimitedValue(); + } + } + if (!DerefBytes.getBoolValue()) { + if (MDNode *MD = LI->getMetadata(LLVMContext::MD_dereferenceable_or_null)) { ConstantInt *CI = mdconst::extract(MD->getOperand(0)); DerefBytes = CI->getLimitedValue(); Index: lib/IR/LLVMContext.cpp =================================================================== --- lib/IR/LLVMContext.cpp +++ lib/IR/LLVMContext.cpp @@ -105,6 +105,12 @@ "dereferenceable_or_null kind id drifted"); (void)DereferenceableOrNullID; + // Create the 'unconditionally_dereferenceable' metadata kind. + unsigned UnconditionallyDereferenceableID = getMDKindID("unconditionally_dereferenceable"); + assert(UnconditionallyDereferenceableID == MD_unconditionally_dereferenceable && + "unconditionally_dereferenceable kind id drifted"); + (void)UnconditionallyDereferenceableID; + // Create the 'make.implicit' metadata kind. unsigned MakeImplicitID = getMDKindID("make.implicit"); assert(MakeImplicitID == MD_make_implicit && Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -328,7 +328,8 @@ void visitFunction(const Function &F); void visitBasicBlock(BasicBlock &BB); void visitRangeMetadata(Instruction& I, MDNode* Range, Type* Ty); - void visitDereferenceableMetadata(Instruction& I, MDNode* MD); + void visitDereferenceableMetadata(Instruction& I, MDNode* MD, + StringRef Name); template bool isValidMetadataArray(const MDTuple &N); #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N); @@ -3358,17 +3359,16 @@ "Instruction does not dominate all uses!", Op, &I); } -void Verifier::visitDereferenceableMetadata(Instruction& I, MDNode* MD) { - Assert(I.getType()->isPointerTy(), "dereferenceable, dereferenceable_or_null " - "apply only to pointer types", &I); +void Verifier::visitDereferenceableMetadata(Instruction& I, MDNode* MD, + StringRef Name) { + Assert(I.getType()->isPointerTy(), Name + " applies only to pointer types", &I); Assert(isa(I), - "dereferenceable, dereferenceable_or_null apply only to load" - " instructions, use attributes for calls or invokes", &I); - Assert(MD->getNumOperands() == 1, "dereferenceable, dereferenceable_or_null " - "take one operand!", &I); + Name + " applies only to load instructions, use attributes " + "for calls or invokes", &I); + Assert(MD->getNumOperands() == 1, Name + " takes one operand!", &I); ConstantInt *CI = mdconst::dyn_extract(MD->getOperand(0)); - Assert(CI && CI->getType()->isIntegerTy(64), "dereferenceable, " - "dereferenceable_or_null metadata value must be an i64!", &I); + Assert(CI && CI->getType()->isIntegerTy(64), Name + " metadata value " + "must be an i64!", &I); } /// verifyInstruction - Verify that an instruction is well formed. @@ -3493,10 +3493,13 @@ } if (MDNode *MD = I.getMetadata(LLVMContext::MD_dereferenceable)) - visitDereferenceableMetadata(I, MD); + visitDereferenceableMetadata(I, MD, "dereferenceable"); if (MDNode *MD = I.getMetadata(LLVMContext::MD_dereferenceable_or_null)) - visitDereferenceableMetadata(I, MD); + visitDereferenceableMetadata(I, MD, "dereferenceable_or_null"); + + if (MDNode *MD = I.getMetadata(LLVMContext::MD_unconditionally_dereferenceable)) + visitDereferenceableMetadata(I, MD, "unconditionally_dereferenceable"); if (MDNode *AlignMD = I.getMetadata(LLVMContext::MD_align)) { Assert(I.getType()->isPointerTy(), "align applies only to pointer types", Index: lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -372,6 +372,7 @@ break; case LLVMContext::MD_align: case LLVMContext::MD_dereferenceable: + case LLVMContext::MD_unconditionally_dereferenceable: case LLVMContext::MD_dereferenceable_or_null: // These only directly apply if the new type is also a pointer. if (NewTy->isPointerTy()) @@ -429,6 +430,7 @@ case LLVMContext::MD_range: case LLVMContext::MD_align: case LLVMContext::MD_dereferenceable: + case LLVMContext::MD_unconditionally_dereferenceable: case LLVMContext::MD_dereferenceable_or_null: // These don't apply for stores. break; @@ -803,6 +805,7 @@ LLVMContext::MD_invariant_load, LLVMContext::MD_nonnull, LLVMContext::MD_invariant_group, LLVMContext::MD_align, LLVMContext::MD_dereferenceable, + LLVMContext::MD_unconditionally_dereferenceable, LLVMContext::MD_dereferenceable_or_null}; combineMetadata(NLI, &LI, KnownIDs); }; Index: lib/Transforms/InstCombine/InstCombinePHI.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombinePHI.cpp +++ lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -361,6 +361,7 @@ LLVMContext::MD_nonnull, LLVMContext::MD_align, LLVMContext::MD_dereferenceable, + LLVMContext::MD_unconditionally_dereferenceable, LLVMContext::MD_dereferenceable_or_null, }; Index: lib/Transforms/Scalar/LICM.cpp =================================================================== --- lib/Transforms/Scalar/LICM.cpp +++ lib/Transforms/Scalar/LICM.cpp @@ -724,8 +724,10 @@ I.moveBefore(Preheader->getTerminator()); // Metadata can be dependent on the condition we are hoisting above. - // Conservatively strip all metadata on the instruction. - I.dropUnknownNonDebugMetadata(); + // Conservatively strip all metadata on the instruction except + // !unconditionally_dereferenceable. + I.dropUnknownNonDebugMetadata( + LLVMContext::MD_unconditionally_dereferenceable); if (isa(I)) ++NumMovedLoads; else if (isa(I)) ++NumMovedCalls; Index: lib/Transforms/Utils/Local.cpp =================================================================== --- lib/Transforms/Utils/Local.cpp +++ lib/Transforms/Utils/Local.cpp @@ -1519,6 +1519,7 @@ MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD)); break; case LLVMContext::MD_dereferenceable: + case LLVMContext::MD_unconditionally_dereferenceable: case LLVMContext::MD_dereferenceable_or_null: K->setMetadata(Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD)); Index: lib/Transforms/Utils/SimplifyCFG.cpp =================================================================== --- lib/Transforms/Utils/SimplifyCFG.cpp +++ lib/Transforms/Utils/SimplifyCFG.cpp @@ -1140,6 +1140,7 @@ LLVMContext::MD_fpmath, LLVMContext::MD_invariant_load, LLVMContext::MD_nonnull, LLVMContext::MD_invariant_group, LLVMContext::MD_align, LLVMContext::MD_dereferenceable, + LLVMContext::MD_unconditionally_dereferenceable, LLVMContext::MD_dereferenceable_or_null}; combineMetadata(I1, I2, KnownIDs); I2->eraseFromParent(); Index: test/Transforms/InstCombine/load-combine-metadata-5.ll =================================================================== --- /dev/null +++ test/Transforms/InstCombine/load-combine-metadata-5.ll @@ -0,0 +1,20 @@ +; RUN: opt -instcombine -S < %s | FileCheck %s + +target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128" + +; CHECK-LABEL: @test_load_load_combine_metadata( +; Check that unconditionally_dereferenceable metadata is combined +; CHECK: load i32*, i32** %0 +; CHECK-SAME: !unconditionally_dereferenceable ![[DEREF:[0-9]+]] +define void @test_load_load_combine_metadata(i32**, i32**, i32**) { + %a = load i32*, i32** %0, !unconditionally_dereferenceable !0 + %b = load i32*, i32** %0, !unconditionally_dereferenceable !1 + store i32 0, i32* %a + store i32 0, i32* %b + ret void +} + +; CHECK: ![[DEREF]] = !{i64 4} + +!0 = !{i64 4} +!1 = !{i64 8} Index: test/Transforms/InstCombine/loadstore-metadata.ll =================================================================== --- test/Transforms/InstCombine/loadstore-metadata.ll +++ test/Transforms/InstCombine/loadstore-metadata.ll @@ -90,6 +90,17 @@ ret i8* %c } +define i8* @test_load_cast_combine_uncond_deref(i32** %ptr) { +; Ensure (cast (load (...))) -> (load (cast (...))) preserves +; unconditionally_dereferenceable metadata. +; CHECK-LABEL: @test_load_cast_combine_uncond_deref( +; CHECK: load i8*, i8** %{{.*}}, !unconditionally_dereferenceable !5 +entry: + %l = load i32*, i32** %ptr, !unconditionally_dereferenceable !5 + %c = bitcast i32* %l to i8* + ret i8* %c +} + define void @test_load_cast_combine_loop(float* %src, i32* %dst, i32 %n) { ; Ensure (cast (load (...))) -> (load (cast (...))) preserves loop access ; metadata. Index: test/Transforms/InstCombine/phi-load-metadata-4.ll =================================================================== --- /dev/null +++ test/Transforms/InstCombine/phi-load-metadata-4.ll @@ -0,0 +1,30 @@ +; RUN: opt -instcombine -S < %s | FileCheck %s + +declare void @bar() +declare void @baz() + +; Check that unconditionally_dereferenceable metadata is combined +; CHECK-LABEL: cont: +; CHECK: load i32*, i32** +; CHECK-SAME: !unconditionally_dereferenceable ![[DEREF:[0-9]+]] +define i32* @test_phi_combine_load_metadata(i1 %c, i32** dereferenceable(8) %p1, i32** dereferenceable(8) %p2) { + br i1 %c, label %t, label %f +t: + call void @bar() + %v1 = load i32*, i32** %p1, align 8, !unconditionally_dereferenceable !0 + br label %cont + +f: + call void @baz() + %v2 = load i32*, i32** %p2, align 8, !unconditionally_dereferenceable !1 + br label %cont + +cont: + %res = phi i32* [ %v1, %t ], [ %v2, %f ] + ret i32* %res +} + +; CHECK: ![[DEREF]] = !{i64 8} + +!0 = !{i64 8} +!1 = !{i64 16} Index: test/Transforms/LICM/hoist-nested-deref-load.ll =================================================================== --- /dev/null +++ test/Transforms/LICM/hoist-nested-deref-load.ll @@ -0,0 +1,60 @@ +; RUN: opt -S -licm < %s | FileCheck %s + +target datalayout = "E-m:e-p:32:32-i8:8:8-i16:16:16-i64:32:32-f64:32:32-v64:32:32-v128:32:32-a0:0:32-n32" + +%a = type { %b* } +%b = type { i32 } + +; This test represents the following function: +; class B: +; __immutable_fields__ = {"x"} +; def __init__(self): +; self.x = 10 +; class A: +; __immutable_fields__ = {"f"} +; def __init__(self): +; self.f = A() +; def foo(self): +; for _ in range(10): +; consume(self.f.x) +; in a memory-safe language where every pointer that is ever computed +; is fully dereferenceable. +; +; We want to check that loads of immutable fields are hoisted out of +; loops even when nested; if they were marked just !dereferenceable instead +; of !unconditionally_dereferenceable, LICM would strip that metadata, as it +; cannot prove that the loaded pointer is still dereferenceable in the loop +; preheader. + +; CHECK-LABEL: @test +; CHECK: entry: +; CHECK: %val.f = load %b*, %b** %ptr.f, !unconditionally_dereferenceable !0 +; CHECK: %val.x = load i32, i32* %ptr.x +; CHECK: for.head: + +define void @test(%a* dereferenceable(4) %arg) { +entry: + br label %for.head + +for.head: + %IND = phi i32 [ 0, %entry ], [ %IND.new, %for.body ] + %CMP = icmp slt i32 %IND, 10 + br i1 %CMP, label %for.body, label %exit + +for.body: + %ptr.f = getelementptr inbounds %a, %a* %arg, i32 0, i32 0 + %val.f = load %b*, %b** %ptr.f, !invariant.load !0, !unconditionally_dereferenceable !1 + %ptr.x = getelementptr inbounds %b, %b* %val.f, i32 0, i32 0 + %val.x = load i32, i32* %ptr.x, !invariant.load !0 + call void @consume(i32 %val.x) + %IND.new = add i32 %IND, 1 + br label %for.head + +exit: + ret void +} + +declare void @consume(i32) + +!0 = !{} +!1 = !{ i64 4 } Index: test/Transforms/SimplifyCFG/preserve-load-metadata-4.ll =================================================================== --- /dev/null +++ test/Transforms/SimplifyCFG/preserve-load-metadata-4.ll @@ -0,0 +1,32 @@ +; RUN: opt < %s -simplifycfg -S | FileCheck %s + +declare void @bar(i32*) +declare void @baz(i32*) + +; CHECK-LABEL: @test_load_combine_metadata( +; Check that unconditionally_dereferenceable metadata is combined +; CHECK: load i32*, i32** %p +; CHECK-SAME: !unconditionally_dereferenceable ![[DEREF:[0-9]+]] +; CHECK: t: +; CHECK: f: +define void @test_load_combine_metadata(i1 %c, i32** %p) { + br i1 %c, label %t, label %f + +t: + %v1 = load i32*, i32** %p, !unconditionally_dereferenceable !0 + call void @bar(i32* %v1) + br label %cont + +f: + %v2 = load i32*, i32** %p, !unconditionally_dereferenceable !1 + call void @baz(i32* %v2) + br label %cont + +cont: + ret void +} + +; CHECK: ![[DEREF]] = !{i64 8} + +!0 = !{i64 8} +!1 = !{i64 16} Index: test/Verifier/dereferenceable-md.ll =================================================================== --- test/Verifier/dereferenceable-md.ll +++ test/Verifier/dereferenceable-md.ll @@ -7,7 +7,7 @@ call i8* @foo(), !dereferenceable !{i64 2} ret void } -; CHECK: dereferenceable, dereferenceable_or_null apply only to load instructions, use attributes for calls or invokes +; CHECK: dereferenceable applies only to load instructions, use attributes for calls or invokes ; CHECK-NEXT: call i8* @foo() define void @f2() { @@ -15,7 +15,15 @@ call i8* @foo(), !dereferenceable_or_null !{i64 2} ret void } -; CHECK: dereferenceable, dereferenceable_or_null apply only to load instructions, use attributes for calls or invokes +; CHECK: dereferenceable_or_null applies only to load instructions, use attributes for calls or invokes +; CHECK-NEXT: call i8* @foo() + +define void @g1() { +entry: + call i8* @foo(), !unconditionally_dereferenceable !{i64 2} + ret void +} +; CHECK: unconditionally_dereferenceable applies only to load instructions, use attributes for calls or invokes ; CHECK-NEXT: call i8* @foo() define i8 @f3(i8* %x) { @@ -23,7 +31,7 @@ %y = load i8, i8* %x, !dereferenceable !{i64 2} ret i8 %y } -; CHECK: dereferenceable, dereferenceable_or_null apply only to pointer types +; CHECK: dereferenceable applies only to pointer types ; CHECK-NEXT: load i8, i8* %x define i8 @f4(i8* %x) { @@ -31,7 +39,15 @@ %y = load i8, i8* %x, !dereferenceable_or_null !{i64 2} ret i8 %y } -; CHECK: dereferenceable, dereferenceable_or_null apply only to pointer types +; CHECK: dereferenceable_or_null applies only to pointer types +; CHECK-NEXT: load i8, i8* %x + +define i8 @g3(i8* %x) { +entry: + %y = load i8, i8* %x, !unconditionally_dereferenceable !{i64 2} + ret i8 %y +} +; CHECK: unconditionally_dereferenceable applies only to pointer types ; CHECK-NEXT: load i8, i8* %x define i8* @f5(i8** %x) { @@ -39,16 +55,23 @@ %y = load i8*, i8** %x, !dereferenceable !{} ret i8* %y } -; CHECK: dereferenceable, dereferenceable_or_null take one operand +; CHECK: dereferenceable takes one operand ; CHECK-NEXT: load i8*, i8** %x - define i8* @f6(i8** %x) { entry: %y = load i8*, i8** %x, !dereferenceable_or_null !{} ret i8* %y } -; CHECK: dereferenceable, dereferenceable_or_null take one operand +; CHECK: dereferenceable_or_null takes one operand +; CHECK-NEXT: load i8*, i8** %x + +define i8* @g5(i8** %x) { +entry: + %y = load i8*, i8** %x, !unconditionally_dereferenceable !{} + ret i8* %y +} +; CHECK: unconditionally_dereferenceable takes one operand ; CHECK-NEXT: load i8*, i8** %x define i8* @f7(i8** %x) { @@ -56,16 +79,23 @@ %y = load i8*, i8** %x, !dereferenceable !{!"str"} ret i8* %y } -; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64! +; CHECK: dereferenceable metadata value must be an i64! ; CHECK-NEXT: load i8*, i8** %x - define i8* @f8(i8** %x) { entry: %y = load i8*, i8** %x, !dereferenceable_or_null !{!"str"} ret i8* %y } -; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64! +; CHECK: dereferenceable_or_null metadata value must be an i64! +; CHECK-NEXT: load i8*, i8** %x + +define i8* @g7(i8** %x) { +entry: + %y = load i8*, i8** %x, !unconditionally_dereferenceable !{!"str"} + ret i8* %y +} +; CHECK: unconditionally_dereferenceable metadata value must be an i64! ; CHECK-NEXT: load i8*, i8** %x define i8* @f9(i8** %x) { @@ -73,14 +103,21 @@ %y = load i8*, i8** %x, !dereferenceable !{i32 2} ret i8* %y } -; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64! +; CHECK: dereferenceable metadata value must be an i64! ; CHECK-NEXT: load i8*, i8** %x - define i8* @f10(i8** %x) { entry: %y = load i8*, i8** %x, !dereferenceable_or_null !{i32 2} ret i8* %y } -; CHECK: dereferenceable, dereferenceable_or_null metadata value must be an i64! -; CHECK-NEXT: load i8*, i8** %x \ No newline at end of file +; CHECK: dereferenceable_or_null metadata value must be an i64! +; CHECK-NEXT: load i8*, i8** %x + +define i8* @g9(i8** %x) { +entry: + %y = load i8*, i8** %x, !unconditionally_dereferenceable !{i32 2} + ret i8* %y +} +; CHECK: unconditionally_dereferenceable metadata value must be an i64! +; CHECK-NEXT: load i8*, i8** %x