diff --git a/llvm/include/llvm/Analysis/AssumeBundleQueries.h b/llvm/include/llvm/Analysis/AssumeBundleQueries.h --- a/llvm/include/llvm/Analysis/AssumeBundleQueries.h +++ b/llvm/include/llvm/Analysis/AssumeBundleQueries.h @@ -106,6 +106,9 @@ ArgValue == Other.ArgValue; } bool operator!=(RetainedKnowledge Other) const { return !(*this == Other); } + bool operator<(RetainedKnowledge Other) const { + return ArgValue < Other.ArgValue; + } operator bool() const { return AttrKind != Attribute::None; } static RetainedKnowledge none() { return RetainedKnowledge{}; } }; diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -467,7 +467,8 @@ /// assume intrinsic, I, at the point in the control-flow identified by the /// context instruction, CxtI. bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool AllowEphemeral = false); enum class OverflowResult { /// Always overflows in the direction of signed/unsigned min value. diff --git a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h --- a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h +++ b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h @@ -16,6 +16,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H #define LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H +#include "llvm/Analysis/AssumeBundleQueries.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/PassManager.h" @@ -41,6 +42,13 @@ void salvageKnowledge(Instruction *I, AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr); +/// Build and return a new assume created from the provided knowledge +/// if the knowledge in the assume is fully redundant this will return nullptr +IntrinsicInst *buildAssumeFromKnowledge(ArrayRef Knowledge, + Instruction *CtxI, + AssumptionCache *AC = nullptr, + DominatorTree *DT = nullptr); + /// This pass attempts to minimize the number of assume without loosing any /// information. struct AssumeSimplifyPass : public PassInfoMixin { diff --git a/llvm/lib/Analysis/AssumeBundleQueries.cpp b/llvm/lib/Analysis/AssumeBundleQueries.cpp --- a/llvm/lib/Analysis/AssumeBundleQueries.cpp +++ b/llvm/lib/Analysis/AssumeBundleQueries.cpp @@ -180,8 +180,8 @@ RetainedKnowledge llvm::getKnowledgeValidInContext( const Value *V, ArrayRef AttrKinds, const Instruction *CtxI, const DominatorTree *DT, AssumptionCache *AC) { - return getKnowledgeForValue(V, AttrKinds, AC, - [&](auto, Instruction *I, auto) { - return isValidAssumeForContext(I, CtxI, DT); - }); + return getKnowledgeForValue( + V, AttrKinds, AC, [&](auto, Instruction *I, auto) { + return isValidAssumeForContext(I, CtxI, DT, /*AllowEphemeral*/ true); + }); } diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -16,6 +16,7 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/Analysis/AssumeBundleQueries.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalVariable.h" @@ -75,6 +76,23 @@ return isAligned(V, Offset, Alignment, DL); } + RetainedKnowledge AlignRK; + RetainedKnowledge DerefRK; + getKnowledgeForValue(V, {Attribute::Dereferenceable, Attribute::Alignment}, + nullptr, + [&](RetainedKnowledge RK, Instruction *Assume, auto) { + if (!isValidAssumeForContext(Assume, CtxI)) + return false; + if (RK.AttrKind == Attribute::Alignment) + AlignRK = std::max(AlignRK, RK); + if (RK.AttrKind == Attribute::Dereferenceable) + DerefRK = std::max(DerefRK, RK); + return false; + }); + if (AlignRK && DerefRK && AlignRK.ArgValue >= Alignment.value() && + DerefRK.ArgValue >= Size.getZExtValue()) + return true; + // For GEPs, determine if the indexing lands within the allocated object. if (const GEPOperator *GEP = dyn_cast(V)) { const Value *Base = GEP->getPointerOperand(); diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -619,7 +619,8 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv, const Instruction *CxtI, - const DominatorTree *DT) { + const DominatorTree *DT, + bool AllowEphemeral) { // There are two restrictions on the use of an assume: // 1. The assume must dominate the context (or the control flow must // reach the assume whenever it reaches the context). @@ -647,7 +648,7 @@ if (!isGuaranteedToTransferExecutionToSuccessor(&*I)) return false; - return !isEphemeralValueOf(Inv, CxtI); + return AllowEphemeral || !isEphemeralValueOf(Inv, CxtI); } // Inv and CxtI are in different blocks. @@ -744,6 +745,18 @@ unsigned BitWidth = Known.getBitWidth(); + if (V->getType()->isPointerTy()) { + if (getKnowledgeForValue( + V, {Attribute::Alignment}, Q.AC, + [&](RetainedKnowledge RK, Instruction *Assume, auto) { + if (isValidAssumeForContext(Assume, Q.CxtI, Q.DT, + /*AllowEphemeral*/ true)) + Known.Zero.setLowBits(countTrailingZeros(RK.ArgValue)); + return false; /// There may be a better assume later so keep going + })) + return; + } + // Note that the patterns below need to be kept in sync with the code // in AssumptionCache::updateAffectedValues. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -68,6 +68,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/InstCombine/InstCombineWorklist.h" +#include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/SimplifyLibCalls.h" #include @@ -90,6 +91,8 @@ cl::desc("How wide an instruction window to bypass looking for " "another guard")); +extern cl::opt EnableKnowledgeRetention; + /// Return the specified type promoted as it would be to pass though a va_arg /// area. static Type *getPromotedType(Type *Ty) { @@ -4220,12 +4223,21 @@ break; case Intrinsic::assume: { Value *IIOperand = II->getArgOperand(0); + SmallVector OpBundles; + II->getOperandBundlesAsDefs(OpBundles); + auto RemoveConditionFromAssume = [&](Instruction *Assume) -> Instruction * { + assert(isa(Assume)); + if (isAssumeWithEmptyBundle(*cast(II))) + return eraseInstFromFunction(CI); + replaceUse(II->getOperandUse(0), ConstantInt::getTrue(II->getContext())); + return nullptr; + }; // Remove an assume if it is followed by an identical assume. // TODO: Do we need this? Unless there are conflicting assumptions, the // computeKnownBits(IIOperand) below here eliminates redundant assumes. Instruction *Next = II->getNextNonDebugInstruction(); if (match(Next, m_Intrinsic(m_Specific(IIOperand)))) - return eraseInstFromFunction(CI); + return RemoveConditionFromAssume(Next); // Canonicalize assume(a && b) -> assume(a); assume(b); // Note: New assumption intrinsics created here are registered by @@ -4247,9 +4259,59 @@ return eraseInstFromFunction(*II); } + // Convert nonnull assume like: + // %A = icmp ne i32* %PTR, null + // call void @llvm.assume(i1 %A) + // into + // call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ] + CmpInst::Predicate Pred; + if (EnableKnowledgeRetention && + match(IIOperand, m_Cmp(Pred, m_Value(A), m_Zero())) && + Pred == CmpInst::ICMP_NE && A->getType()->isPointerTy()) { + if (IntrinsicInst *Replacement = buildAssumeFromKnowledge( + {RetainedKnowledge{Attribute::NonNull, 0, A}}, Next, &AC, &DT)) { + + Replacement->insertBefore(Next); + AC.registerAssumption(Replacement); + } + return RemoveConditionFromAssume(II); + } + + // Convert alignment assume like: + // %A = icmp ne i32* %PTR, null + // %B = ptrtoint i32* %A to i64 + // %C = and i64 %B, Constant + // %D = icmp eq i64 %C, 0 + // call void @llvm.assume(i1 %D) + // into + // call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 Constant + 1)] + uint64_t AlignMask; + if (EnableKnowledgeRetention && + match(IIOperand, + m_Cmp(Pred, m_And(m_Value(A), m_ConstantInt(AlignMask)), + m_Zero())) && + Pred == CmpInst::ICMP_EQ) { + if (isPowerOf2_64(AlignMask + 1)) { + uint64_t Offset = 0; + match(A, m_Add(m_Value(A), m_ConstantInt(Offset))); + if (match(A, m_PtrToInt(m_Value(A)))) { + /// Note: this doesn't preserve the offset information but merges + /// offset and alignememt. + RetainedKnowledge RK{Attribute::Alignment, + (unsigned)MinAlign(Offset, AlignMask + 1), A}; + if (IntrinsicInst *Replacement = + buildAssumeFromKnowledge(RK, Next, &AC, &DT)) { + + Replacement->insertBefore(Next); + AC.registerAssumption(Replacement); + } + return RemoveConditionFromAssume(II); + } + } + } + // assume( (load addr) != null ) -> add 'nonnull' metadata to load // (if assume is valid at the load) - CmpInst::Predicate Pred; Instruction *LHS; if (match(IIOperand, m_ICmp(Pred, m_Instruction(LHS), m_Zero())) && Pred == ICmpInst::ICMP_NE && LHS->getOpcode() == Instruction::Load && @@ -4257,7 +4319,8 @@ isValidAssumeForContext(II, LHS, &DT)) { MDNode *MD = MDNode::get(II->getContext(), None); LHS->setMetadata(LLVMContext::MD_nonnull, MD); - return eraseInstFromFunction(*II); + if (isAssumeWithEmptyBundle(*II)) + return eraseInstFromFunction(*II); // TODO: apply nonnull return attributes to calls and invokes // TODO: apply range metadata for range check patterns? diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp --- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp +++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp @@ -280,6 +280,16 @@ } } +IntrinsicInst * +llvm::buildAssumeFromKnowledge(ArrayRef Knowledge, + Instruction *CtxI, AssumptionCache *AC, + DominatorTree *DT) { + AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT); + for (const RetainedKnowledge &RK : Knowledge) + Builder.addKnowledge(RK); + return Builder.build(); +} + namespace { struct AssumeSimplify { diff --git a/llvm/test/Analysis/BasicAA/featuretest.ll b/llvm/test/Analysis/BasicAA/featuretest.ll --- a/llvm/test/Analysis/BasicAA/featuretest.ll +++ b/llvm/test/Analysis/BasicAA/featuretest.ll @@ -128,7 +128,7 @@ ; USE_ASSUME-LABEL: @gep_distance_test3( ; USE_ASSUME-NEXT: [[C1:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 1 ; USE_ASSUME-NEXT: [[C:%.*]] = bitcast i32* [[C1]] to i8* -; USE_ASSUME-NEXT: store i8 42, i8* [[C]], align 1 +; USE_ASSUME-NEXT: store i8 42, i8* [[C]], align 4 ; USE_ASSUME-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[A]], i64 4), "nonnull"(i32* [[A]]), "align"(i32* [[A]], i64 4) ] ; USE_ASSUME-NEXT: ret i32 0 ; diff --git a/llvm/test/Analysis/ValueTracking/assume.ll b/llvm/test/Analysis/ValueTracking/assume.ll --- a/llvm/test/Analysis/ValueTracking/assume.ll +++ b/llvm/test/Analysis/ValueTracking/assume.ll @@ -59,20 +59,20 @@ define dso_local i32 @test4(i32* readonly %0, i1 %cond) { ; CHECK-LABEL: @test4( -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4), "align"(i32* [[TMP0]], i32 8) ] ; CHECK-NEXT: br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]] ; CHECK: B: ; CHECK-NEXT: br label [[A]] ; CHECK: A: ; CHECK-NEXT: br i1 false, label [[TMP4:%.*]], label [[TMP2:%.*]] ; CHECK: 2: -; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 8 ; CHECK-NEXT: br label [[TMP4]] ; CHECK: 4: ; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ [[TMP3]], [[TMP2]] ], [ 0, [[A]] ] ; CHECK-NEXT: ret i32 [[TMP5]] ; - call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4)] + call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4), "align"(i32* %0, i32 8)] br i1 %cond, label %A, label %B B: diff --git a/llvm/test/Transforms/InstCombine/assume.ll b/llvm/test/Transforms/InstCombine/assume.ll --- a/llvm/test/Transforms/InstCombine/assume.ll +++ b/llvm/test/Transforms/InstCombine/assume.ll @@ -6,15 +6,22 @@ target triple = "x86_64-unknown-linux-gnu" define i32 @foo1(i32* %a) #0 { -; SAME-LABEL: define {{[^@]+}}@foo1 -; SAME-SAME: (i32* [[A:%.*]]) #0 -; SAME-NEXT: entry: -; SAME-NEXT: [[TMP0:%.*]] = load i32, i32* [[A]], align 32 -; SAME-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64 -; SAME-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 -; SAME-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 -; SAME-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) -; SAME-NEXT: ret i32 [[TMP0]] +; DEFAULT-LABEL: define {{[^@]+}}@foo1 +; DEFAULT-SAME: (i32* [[A:%.*]]) #0 +; DEFAULT-NEXT: entry: +; DEFAULT-NEXT: [[TMP0:%.*]] = load i32, i32* [[A]], align 32 +; DEFAULT-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64 +; DEFAULT-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 +; DEFAULT-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) +; DEFAULT-NEXT: ret i32 [[TMP0]] +; +; BUNDLES-LABEL: define {{[^@]+}}@foo1 +; BUNDLES-SAME: (i32* [[A:%.*]]) #0 +; BUNDLES-NEXT: entry: +; BUNDLES-NEXT: [[TMP0:%.*]] = load i32, i32* [[A]], align 32 +; BUNDLES-NEXT: call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 32) ] +; BUNDLES-NEXT: ret i32 [[TMP0]] ; entry: %0 = load i32, i32* %a, align 4 @@ -31,15 +38,22 @@ } define i32 @foo2(i32* %a) #0 { -; SAME-LABEL: define {{[^@]+}}@foo2 -; SAME-SAME: (i32* [[A:%.*]]) #0 -; SAME-NEXT: entry: -; SAME-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64 -; SAME-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 -; SAME-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 -; SAME-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) -; SAME-NEXT: [[TMP0:%.*]] = load i32, i32* [[A]], align 32 -; SAME-NEXT: ret i32 [[TMP0]] +; DEFAULT-LABEL: define {{[^@]+}}@foo2 +; DEFAULT-SAME: (i32* [[A:%.*]]) #0 +; DEFAULT-NEXT: entry: +; DEFAULT-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64 +; DEFAULT-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 +; DEFAULT-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) +; DEFAULT-NEXT: [[TMP0:%.*]] = load i32, i32* [[A]], align 32 +; DEFAULT-NEXT: ret i32 [[TMP0]] +; +; BUNDLES-LABEL: define {{[^@]+}}@foo2 +; BUNDLES-SAME: (i32* [[A:%.*]]) #0 +; BUNDLES-NEXT: entry: +; BUNDLES-NEXT: call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 32) ] +; BUNDLES-NEXT: [[TMP0:%.*]] = load i32, i32* [[A]], align 32 +; BUNDLES-NEXT: ret i32 [[TMP0]] ; entry: ; Same check as in @foo1, but make sure it works if the assume is first too. @@ -256,6 +270,18 @@ ret i32 %load } +define i32 @bundle1A(i32* %P) { +; SAME-LABEL: define {{[^@]+}}@bundle1A +; SAME-SAME: (i32* [[P:%.*]]) +; SAME-NEXT: tail call void @llvm.assume(i1 true) [ "align"(i32* [[P]], i32 64) ] +; SAME-NEXT: [[LOAD:%.*]] = load i32, i32* [[P]], align 64 +; SAME-NEXT: ret i32 [[LOAD]] +; + tail call void @llvm.assume(i1 true) ["align"(i32* %P, i32 64)] + %load = load i32, i32* %P + ret i32 %load +} + define i32 @bundle2(i32* %P) { ; SAME-LABEL: define {{[^@]+}}@bundle2 ; SAME-SAME: (i32* [[P:%.*]]) @@ -268,11 +294,18 @@ } define i1 @nonnull1(i32** %a) { -; SAME-LABEL: define {{[^@]+}}@nonnull1 -; SAME-SAME: (i32** [[A:%.*]]) -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8, !nonnull !6 -; SAME-NEXT: tail call void @escape(i32* nonnull [[LOAD]]) -; SAME-NEXT: ret i1 false +; DEFAULT-LABEL: define {{[^@]+}}@nonnull1 +; DEFAULT-SAME: (i32** [[A:%.*]]) +; DEFAULT-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8, !nonnull !6 +; DEFAULT-NEXT: tail call void @escape(i32* nonnull [[LOAD]]) +; DEFAULT-NEXT: ret i1 false +; +; BUNDLES-LABEL: define {{[^@]+}}@nonnull1 +; BUNDLES-SAME: (i32** [[A:%.*]]) +; BUNDLES-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 +; BUNDLES-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]) ] +; BUNDLES-NEXT: tail call void @escape(i32* nonnull [[LOAD]]) +; BUNDLES-NEXT: ret i1 false ; %load = load i32*, i32** %a %cmp = icmp ne i32* %load, null @@ -304,19 +337,31 @@ ; if the assume is control dependent on something else define i1 @nonnull3(i32** %a, i1 %control) { -; SAME-LABEL: define {{[^@]+}}@nonnull3 -; SAME-SAME: (i32** [[A:%.*]], i1 [[CONTROL:%.*]]) -; SAME-NEXT: entry: -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: br i1 [[CONTROL]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] -; SAME: taken: -; SAME-NEXT: tail call void @llvm.assume(i1 [[CMP]]) -; SAME-NEXT: ret i1 false -; SAME: not_taken: -; SAME-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null -; SAME-NEXT: ret i1 [[RVAL_2]] +; DEFAULT-LABEL: define {{[^@]+}}@nonnull3 +; DEFAULT-SAME: (i32** [[A:%.*]], i1 [[CONTROL:%.*]]) +; DEFAULT-NEXT: entry: +; DEFAULT-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 +; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null +; DEFAULT-NEXT: br i1 [[CONTROL]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; DEFAULT: taken: +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[CMP]]) +; DEFAULT-NEXT: ret i1 false +; DEFAULT: not_taken: +; DEFAULT-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; DEFAULT-NEXT: ret i1 [[RVAL_2]] +; +; BUNDLES-LABEL: define {{[^@]+}}@nonnull3 +; BUNDLES-SAME: (i32** [[A:%.*]], i1 [[CONTROL:%.*]]) +; BUNDLES-NEXT: entry: +; BUNDLES-NEXT: br i1 [[CONTROL]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; BUNDLES: taken: +; BUNDLES-NEXT: ret i1 false +; BUNDLES: not_taken: +; BUNDLES-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 +; BUNDLES-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; BUNDLES-NEXT: ret i1 [[RVAL_2]] ; +; FIXME: in the BUNDLES version we could duplicate the load and keep the assume nonnull. entry: %load = load i32*, i32** %a %cmp = icmp ne i32* %load, null @@ -335,13 +380,20 @@ ; interrupted by an exception being thrown define i1 @nonnull4(i32** %a) { -; SAME-LABEL: define {{[^@]+}}@nonnull4 -; SAME-SAME: (i32** [[A:%.*]]) -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 -; SAME-NEXT: tail call void @escape(i32* [[LOAD]]) -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: tail call void @llvm.assume(i1 [[CMP]]) -; SAME-NEXT: ret i1 false +; DEFAULT-LABEL: define {{[^@]+}}@nonnull4 +; DEFAULT-SAME: (i32** [[A:%.*]]) +; DEFAULT-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 +; DEFAULT-NEXT: tail call void @escape(i32* [[LOAD]]) +; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[CMP]]) +; DEFAULT-NEXT: ret i1 false +; +; BUNDLES-LABEL: define {{[^@]+}}@nonnull4 +; BUNDLES-SAME: (i32** [[A:%.*]]) +; BUNDLES-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 +; BUNDLES-NEXT: tail call void @escape(i32* [[LOAD]]) +; BUNDLES-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]) ] +; BUNDLES-NEXT: ret i1 false ; %load = load i32*, i32** %a ;; This call may throw! @@ -397,15 +449,25 @@ ; get in the way of the fold. define void @debug_interference(i8 %x) { -; SAME-LABEL: define {{[^@]+}}@debug_interference -; SAME-SAME: (i8 [[X:%.*]]) -; SAME-NEXT: [[CMP2:%.*]] = icmp ne i8 [[X]], 0 -; SAME-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9 -; SAME-NEXT: tail call void @llvm.assume(i1 false) -; SAME-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9 -; SAME-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9 -; SAME-NEXT: tail call void @llvm.assume(i1 [[CMP2]]) -; SAME-NEXT: ret void +; DEFAULT-LABEL: define {{[^@]+}}@debug_interference +; DEFAULT-SAME: (i8 [[X:%.*]]) +; DEFAULT-NEXT: [[CMP2:%.*]] = icmp ne i8 [[X]], 0 +; DEFAULT-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9 +; DEFAULT-NEXT: tail call void @llvm.assume(i1 false) +; DEFAULT-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9 +; DEFAULT-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9 +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[CMP2]]) +; DEFAULT-NEXT: ret void +; +; BUNDLES-LABEL: define {{[^@]+}}@debug_interference +; BUNDLES-SAME: (i8 [[X:%.*]]) +; BUNDLES-NEXT: [[CMP2:%.*]] = icmp ne i8 [[X]], 0 +; BUNDLES-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !6, metadata !DIExpression()), !dbg !8 +; BUNDLES-NEXT: tail call void @llvm.assume(i1 false) +; BUNDLES-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !6, metadata !DIExpression()), !dbg !8 +; BUNDLES-NEXT: tail call void @llvm.dbg.value(metadata i32 5, metadata !6, metadata !DIExpression()), !dbg !8 +; BUNDLES-NEXT: tail call void @llvm.assume(i1 [[CMP2]]) +; BUNDLES-NEXT: ret void ; %cmp1 = icmp eq i8 %x, 0 %cmp2 = icmp ne i8 %x, 0 @@ -439,18 +501,29 @@ } define i1 @nonnull3A(i32** %a, i1 %control) { -; SAME-LABEL: define {{[^@]+}}@nonnull3A -; SAME-SAME: (i32** [[A:%.*]], i1 [[CONTROL:%.*]]) -; SAME-NEXT: entry: -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 -; SAME-NEXT: br i1 [[CONTROL]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] -; SAME: taken: -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: call void @llvm.assume(i1 [[CMP]]) -; SAME-NEXT: ret i1 true -; SAME: not_taken: -; SAME-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null -; SAME-NEXT: ret i1 [[RVAL_2]] +; DEFAULT-LABEL: define {{[^@]+}}@nonnull3A +; DEFAULT-SAME: (i32** [[A:%.*]], i1 [[CONTROL:%.*]]) +; DEFAULT-NEXT: entry: +; DEFAULT-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 +; DEFAULT-NEXT: br i1 [[CONTROL]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; DEFAULT: taken: +; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null +; DEFAULT-NEXT: call void @llvm.assume(i1 [[CMP]]) +; DEFAULT-NEXT: ret i1 true +; DEFAULT: not_taken: +; DEFAULT-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; DEFAULT-NEXT: ret i1 [[RVAL_2]] +; +; BUNDLES-LABEL: define {{[^@]+}}@nonnull3A +; BUNDLES-SAME: (i32** [[A:%.*]], i1 [[CONTROL:%.*]]) +; BUNDLES-NEXT: entry: +; BUNDLES-NEXT: br i1 [[CONTROL]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; BUNDLES: taken: +; BUNDLES-NEXT: ret i1 true +; BUNDLES: not_taken: +; BUNDLES-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 +; BUNDLES-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; BUNDLES-NEXT: ret i1 [[RVAL_2]] ; entry: %load = load i32*, i32** %a @@ -469,13 +542,13 @@ ; SAME-LABEL: define {{[^@]+}}@nonnull3B ; SAME-SAME: (i32** [[A:%.*]], i1 [[CONTROL:%.*]]) ; SAME-NEXT: entry: +; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 ; SAME-NEXT: br i1 [[CONTROL]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] ; SAME: taken: -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A]], align 8 -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: call void @llvm.assume(i1 [[CMP]]) [ "nonnull"(i32* [[LOAD]]), "nonnull"(i1 [[CMP]]) ] +; SAME-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]), "nonnull"(i1 true) ] ; SAME-NEXT: ret i1 true ; SAME: not_taken: +; SAME-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]), "ignore"(i1 undef) ] ; SAME-NEXT: ret i1 [[CONTROL]] ; entry: