diff --git a/llvm/include/llvm/Analysis/AssumeBundleQueries.h b/llvm/include/llvm/Analysis/AssumeBundleQueries.h --- a/llvm/include/llvm/Analysis/AssumeBundleQueries.h +++ b/llvm/include/llvm/Analysis/AssumeBundleQueries.h @@ -106,6 +106,14 @@ ArgValue == Other.ArgValue; } bool operator!=(RetainedKnowledge Other) const { return !(*this == Other); } + /// This is only intended for use in std::min/std::max between attribute that + /// only differ in ArgValue. + bool operator<(RetainedKnowledge Other) const { + assert(AttrKind == Other.AttrKind && WasOn == Other.WasOn && + "This is only intend for use in min/max to select the best for " + "RetainedKnowledge that is otherwise equal"); + return ArgValue < Other.ArgValue; + } operator bool() const { return AttrKind != Attribute::None; } static RetainedKnowledge none() { return RetainedKnowledge{}; } }; diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -480,7 +480,8 @@ /// assume intrinsic, I, at the point in the control-flow identified by the /// context instruction, CxtI. bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool AllowEphemeral = false); enum class OverflowResult { /// Always overflows in the direction of signed/unsigned min value. diff --git a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h --- a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h +++ b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h @@ -16,6 +16,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H #define LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H +#include "llvm/Analysis/AssumeBundleQueries.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/PassManager.h" @@ -41,6 +42,13 @@ void salvageKnowledge(Instruction *I, AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr); +/// Build and return a new assume created from the provided knowledge +/// if the knowledge in the assume is fully redundant this will return nullptr +IntrinsicInst *buildAssumeFromKnowledge(ArrayRef Knowledge, + Instruction *CtxI, + AssumptionCache *AC = nullptr, + DominatorTree *DT = nullptr); + /// This pass attempts to minimize the number of assume without loosing any /// information. struct AssumeSimplifyPass : public PassInfoMixin { @@ -55,6 +63,13 @@ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; +/// return a canonical RetainedKnowledge from RK, Assume holding RK, +/// Will return a none RetainedKnowledge if RK is useless. +RetainedKnowledge simplifyRetainedKnowledge(CallBase *Assume, + RetainedKnowledge RK, + AssumptionCache *AC, + DominatorTree *DT); + } // namespace llvm #endif diff --git a/llvm/lib/Analysis/AssumeBundleQueries.cpp b/llvm/lib/Analysis/AssumeBundleQueries.cpp --- a/llvm/lib/Analysis/AssumeBundleQueries.cpp +++ b/llvm/lib/Analysis/AssumeBundleQueries.cpp @@ -209,8 +209,8 @@ RetainedKnowledge llvm::getKnowledgeValidInContext( const Value *V, ArrayRef AttrKinds, const Instruction *CtxI, const DominatorTree *DT, AssumptionCache *AC) { - return getKnowledgeForValue(V, AttrKinds, AC, - [&](auto, Instruction *I, auto) { - return isValidAssumeForContext(I, CtxI, DT); - }); + return getKnowledgeForValue( + V, AttrKinds, AC, [&](auto, Instruction *I, auto) { + return isValidAssumeForContext(I, CtxI, DT, /*AllowEphemeral*/ true); + }); } diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -16,6 +16,7 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/Analysis/AssumeBundleQueries.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalVariable.h" @@ -75,6 +76,29 @@ return isAligned(V, Offset, Alignment, DL); } + /// Look through assumes to see if both dereferencability and alignment can be + /// provent by an assume + RetainedKnowledge AlignRK; + RetainedKnowledge DerefRK; + if (getKnowledgeForValue( + V, {Attribute::Dereferenceable, Attribute::Alignment}, nullptr, + [&](RetainedKnowledge RK, Instruction *Assume, auto) { + if (!isValidAssumeForContext(Assume, CtxI)) + return false; + if (RK.AttrKind == Attribute::Alignment) + AlignRK = std::max(AlignRK, RK); + if (RK.AttrKind == Attribute::Dereferenceable) + DerefRK = std::max(DerefRK, RK); + if (AlignRK && DerefRK && AlignRK.ArgValue >= Alignment.value() && + DerefRK.ArgValue >= Size.getZExtValue()) + return true; // We have found what we needed so here + return false; // Other assumes may have better information. so keep + // looking + })) + return true; + /// TODO refactor this function to be able to search independently for + /// Dereferencability and Alignment requirements. + // For GEPs, determine if the indexing lands within the allocated object. if (const GEPOperator *GEP = dyn_cast(V)) { const Value *Base = GEP->getPointerOperand(); diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -617,7 +617,8 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv, const Instruction *CxtI, - const DominatorTree *DT) { + const DominatorTree *DT, + bool AllowEphemeral) { // There are two restrictions on the use of an assume: // 1. The assume must dominate the context (or the control flow must // reach the assume whenever it reaches the context). @@ -645,7 +646,7 @@ if (!isGuaranteedToTransferExecutionToSuccessor(&*I)) return false; - return !isEphemeralValueOf(Inv, CxtI); + return AllowEphemeral || !isEphemeralValueOf(Inv, CxtI); } // Inv and CxtI are in different blocks. @@ -742,6 +743,18 @@ unsigned BitWidth = Known.getBitWidth(); + if (V->getType()->isPointerTy()) { + if (getKnowledgeForValue( + V, {Attribute::Alignment}, Q.AC, + [&](RetainedKnowledge RK, Instruction *Assume, auto) { + if (isValidAssumeForContext(Assume, Q.CxtI, Q.DT, + /*AllowEphemeral*/ true)) + Known.Zero.setLowBits(countTrailingZeros(RK.ArgValue)); + return false; /// There may be a better assume later so keep going + })) + return; + } + // Note that the patterns below need to be kept in sync with the code // in AssumptionCache::updateAffectedValues. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -67,6 +67,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/InstCombine/InstCombineWorklist.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" +#include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/SimplifyLibCalls.h" #include @@ -89,6 +90,10 @@ cl::desc("How wide an instruction window to bypass looking for " "another guard")); +/// enable preservation of attributes in assume like: +/// call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ] +extern cl::opt EnableKnowledgeRetention; + /// Return the specified type promoted as it would be to pass though a va_arg /// area. static Type *getPromotedType(Type *Ty) { @@ -1463,15 +1468,23 @@ Value *IIOperand = II->getArgOperand(0); SmallVector OpBundles; II->getOperandBundlesAsDefs(OpBundles); - bool HasOpBundles = !OpBundles.empty(); + + /// This will remove the boolean Condition from the assume given as + /// arguement and remove the assume if it becomes useless. + /// always returns nullptr for use as a return values. + auto RemoveConditionFromAssume = [&](Instruction *Assume) -> Instruction * { + assert(isa(Assume)); + if (isAssumeWithEmptyBundle(*cast(II))) + return eraseInstFromFunction(CI); + replaceUse(II->getOperandUse(0), ConstantInt::getTrue(II->getContext())); + return nullptr; + }; // Remove an assume if it is followed by an identical assume. // TODO: Do we need this? Unless there are conflicting assumptions, the // computeKnownBits(IIOperand) below here eliminates redundant assumes. Instruction *Next = II->getNextNonDebugInstruction(); - if (HasOpBundles && - match(Next, m_Intrinsic(m_Specific(IIOperand))) && - !cast(Next)->hasOperandBundles()) - return eraseInstFromFunction(CI); + if (match(Next, m_Intrinsic(m_Specific(IIOperand)))) + return RemoveConditionFromAssume(Next); // Canonicalize assume(a && b) -> assume(a); assume(b); // Note: New assumption intrinsics created here are registered by @@ -1504,13 +1517,93 @@ isValidAssumeForContext(II, LHS, &DT)) { MDNode *MD = MDNode::get(II->getContext(), None); LHS->setMetadata(LLVMContext::MD_nonnull, MD); - if (!HasOpBundles) - return eraseInstFromFunction(*II); + return RemoveConditionFromAssume(II); // TODO: apply nonnull return attributes to calls and invokes // TODO: apply range metadata for range check patterns? } + // Convert nonnull assume like: + // %A = icmp ne i32* %PTR, null + // call void @llvm.assume(i1 %A) + // into + // call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ] + if (EnableKnowledgeRetention && + match(IIOperand, m_Cmp(Pred, m_Value(A), m_Zero())) && + Pred == CmpInst::ICMP_NE && A->getType()->isPointerTy()) { + if (IntrinsicInst *Replacement = buildAssumeFromKnowledge( + {RetainedKnowledge{Attribute::NonNull, 0, A}}, Next, &AC, &DT)) { + + Replacement->insertBefore(Next); + AC.registerAssumption(Replacement); + } + return RemoveConditionFromAssume(II); + } + + // Convert alignment assume like: + // %A = icmp ne i32* %PTR, null + // %B = ptrtoint i32* %A to i64 + // %C = and i64 %B, Constant + // %D = icmp eq i64 %C, 0 + // call void @llvm.assume(i1 %D) + // into + // call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 Constant + 1)] + uint64_t AlignMask; + if (EnableKnowledgeRetention && + match(IIOperand, + m_Cmp(Pred, m_And(m_Value(A), m_ConstantInt(AlignMask)), + m_Zero())) && + Pred == CmpInst::ICMP_EQ) { + if (isPowerOf2_64(AlignMask + 1)) { + uint64_t Offset = 0; + match(A, m_Add(m_Value(A), m_ConstantInt(Offset))); + if (match(A, m_PtrToInt(m_Value(A)))) { + /// Note: this doesn't preserve the offset information but merges + /// offset and alignment. + /// TODO: we can generate a GEP instead of merging the alignment with + /// the offset. + RetainedKnowledge RK{Attribute::Alignment, + (unsigned)MinAlign(Offset, AlignMask + 1), A}; + if (IntrinsicInst *Replacement = + buildAssumeFromKnowledge(RK, Next, &AC, &DT)) { + + Replacement->insertAfter(II); + AC.registerAssumption(Replacement); + } + return RemoveConditionFromAssume(II); + } + } + } + + /// Canonicalize Knowledge help in operand bundles. + if (II->hasOperandBundles()) { + for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) { + auto &BOI = II->bundle_op_info_begin()[Idx]; + RetainedKnowledge RK = llvm::getKnowledgeFromBundle(*II, BOI); + if (BOI.End - BOI.Begin > 2) + continue; // Prevent reducing knowledge in an align with offset since + // extracting a RetainedKnowledge form then looses + RetainedKnowledge CanonRK = llvm::simplifyRetainedKnowledge( + II, RK, &getAssumptionCache(), &getDominatorTree()); + if (CanonRK != RK) { + if (!CanonRK) { + if (BOI.End - BOI.Begin > 0) + Value::dropDroppableUse(II->op_begin()[BOI.Begin]); + return nullptr; + } + assert(RK.AttrKind == CanonRK.AttrKind); + if (BOI.End - BOI.Begin > 0) + II->op_begin()[BOI.Begin].set(CanonRK.WasOn); + if (BOI.End - BOI.Begin > 1) + II->op_begin()[BOI.Begin + 1].set(ConstantInt::get( + Type::getInt64Ty(II->getContext()), CanonRK.ArgValue)); + if (RK.WasOn) + Worklist.pushValue(RK.WasOn); + return II; + } + } + } + // If there is a dominating assume with the same condition as this one, // then this one is redundant, and should be removed. KnownBits Known(1); diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp --- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp +++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp @@ -6,8 +6,6 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "assume-builder" - #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/MapVector.h" @@ -37,6 +35,8 @@ cl::desc( "enable preservation of attributes throughout code transformation")); +#define DEBUG_TYPE "assume-builder" + STATISTIC(NumAssumeBuilt, "Number of assume built by the assume builder"); STATISTIC(NumBundlesInAssumes, "Total number of Bundles in the assume built"); STATISTIC(NumAssumesMerged, @@ -64,7 +64,7 @@ /// This function will try to transform the given knowledge into a more /// canonical one. the canonical knowledge maybe the given one. -RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK, Module *M) { +RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK, DataLayout DL) { switch (RK.AttrKind) { default: return RK; @@ -75,8 +75,7 @@ Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) { if (auto *GEP = dyn_cast(Strip)) RK.ArgValue = - MinAlign(RK.ArgValue, - GEP->getMaxPreservedAlignment(M->getDataLayout()).value()); + MinAlign(RK.ArgValue, GEP->getMaxPreservedAlignment(DL).value()); }); RK.WasOn = V; return RK; @@ -84,8 +83,8 @@ case Attribute::Dereferenceable: case Attribute::DereferenceableOrNull: { int64_t Offset = 0; - Value *V = GetPointerBaseWithConstantOffset( - RK.WasOn, Offset, M->getDataLayout(), /*AllowNonInBounds*/ false); + Value *V = GetPointerBaseWithConstantOffset(RK.WasOn, Offset, DL, + /*AllowNonInBounds*/ false); if (Offset < 0) return RK; RK.ArgValue = RK.ArgValue + Offset; @@ -102,16 +101,16 @@ using MapKey = std::pair; SmallMapVector AssumedKnowledgeMap; - Instruction *InstBeingRemoved = nullptr; + Instruction *InstBeingModified = nullptr; AssumptionCache* AC = nullptr; DominatorTree* DT = nullptr; AssumeBuilderState(Module *M, Instruction *I = nullptr, AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr) - : M(M), InstBeingRemoved(I), AC(AC), DT(DT) {} + : M(M), InstBeingModified(I), AC(AC), DT(DT) {} bool tryToPreserveWithoutAddingAssume(RetainedKnowledge RK) { - if (!InstBeingRemoved || !RK.WasOn) + if (!InstBeingModified || !RK.WasOn) return false; bool HasBeenPreserved = false; Use* ToUpdate = nullptr; @@ -119,12 +118,12 @@ RK.WasOn, {RK.AttrKind}, AC, [&](RetainedKnowledge RKOther, Instruction *Assume, const CallInst::BundleOpInfo *Bundle) { - if (!isValidAssumeForContext(Assume, InstBeingRemoved, DT)) + if (!isValidAssumeForContext(Assume, InstBeingModified, DT)) return false; if (RKOther.ArgValue >= RK.ArgValue) { HasBeenPreserved = true; return true; - } else if (isValidAssumeForContext(InstBeingRemoved, Assume, + } else if (isValidAssumeForContext(InstBeingModified, Assume, DT)) { HasBeenPreserved = true; IntrinsicInst *Intr = cast(Assume); @@ -161,14 +160,14 @@ if (RK.WasOn->use_empty()) return false; Use *SingleUse = RK.WasOn->getSingleUndroppableUse(); - if (SingleUse && SingleUse->getUser() == InstBeingRemoved) + if (SingleUse && SingleUse->getUser() == InstBeingModified) return false; } return true; } void addKnowledge(RetainedKnowledge RK) { - RK = canonicalizedKnowledge(RK, M); + RK = canonicalizedKnowledge(RK, M->getDataLayout()); if (!isKnowledgeWorthPreserving(RK)) return; @@ -298,6 +297,33 @@ } } +IntrinsicInst * +llvm::buildAssumeFromKnowledge(ArrayRef Knowledge, + Instruction *CtxI, AssumptionCache *AC, + DominatorTree *DT) { + AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT); + for (const RetainedKnowledge &RK : Knowledge) + Builder.addKnowledge(RK); + return Builder.build(); +} + +RetainedKnowledge llvm::simplifyRetainedKnowledge(CallBase *Assume, + RetainedKnowledge RK, + AssumptionCache *AC, + DominatorTree *DT) { + if (!Assume) + return canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout()); + AssumeBuilderState Builder(Assume->getModule(), Assume, AC, DT); + RK = canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout()); + + if (!Builder.isKnowledgeWorthPreserving(RK)) + return RetainedKnowledge::none(); + + if (Builder.tryToPreserveWithoutAddingAssume(RK)) + return RetainedKnowledge::none(); + return RK; +} + namespace { struct AssumeSimplify { diff --git a/llvm/test/Analysis/BasicAA/featuretest.ll b/llvm/test/Analysis/BasicAA/featuretest.ll --- a/llvm/test/Analysis/BasicAA/featuretest.ll +++ b/llvm/test/Analysis/BasicAA/featuretest.ll @@ -20,7 +20,6 @@ ; CHECK-NEXT: [[ARRAY22:%.*]] = alloca [200 x i32], align 4 ; CHECK-NEXT: [[ARRAY22_SUB:%.*]] = getelementptr inbounds [200 x i32], [200 x i32]* [[ARRAY22]], i64 0, i64 0 ; CHECK-NEXT: [[ARRAY11_SUB:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* [[ARRAY11]], i64 0, i64 0 -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i32* [[ARRAY11_SUB]], i32 4) ] ; CHECK-NEXT: call void @external(i32* nonnull [[ARRAY11_SUB]]) ; CHECK-NEXT: call void @external(i32* nonnull [[ARRAY22_SUB]]) ; CHECK-NEXT: [[POINTER2:%.*]] = getelementptr [200 x i32], [200 x i32]* [[ARRAY22]], i64 0, i64 [[B:%.*]] @@ -128,7 +127,7 @@ ; USE_ASSUME-LABEL: @gep_distance_test3( ; USE_ASSUME-NEXT: [[C1:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 1 ; USE_ASSUME-NEXT: [[C:%.*]] = bitcast i32* [[C1]] to i8* -; USE_ASSUME-NEXT: store i8 42, i8* [[C]], align 1 +; USE_ASSUME-NEXT: store i8 42, i8* [[C]], align 4 ; USE_ASSUME-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[A]], i64 4), "nonnull"(i32* [[A]]), "align"(i32* [[A]], i64 4) ] ; USE_ASSUME-NEXT: ret i32 0 ; diff --git a/llvm/test/Analysis/ValueTracking/assume-queries-counter.ll b/llvm/test/Analysis/ValueTracking/assume-queries-counter.ll --- a/llvm/test/Analysis/ValueTracking/assume-queries-counter.ll +++ b/llvm/test/Analysis/ValueTracking/assume-queries-counter.ll @@ -1,9 +1,9 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; REQUIRES: asserts -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=0,assume-queries-counter-count=1 -S | FileCheck %s --check-prefixes=SAME,COUNTER1 ; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=1,assume-queries-counter-count=2 -S | FileCheck %s --check-prefixes=SAME,COUNTER2 -; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=2,assume-queries-counter-count=1 -S | FileCheck %s --check-prefixes=SAME,COUNTER3 +; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=2,assume-queries-counter-count=5 -S | FileCheck %s --check-prefixes=SAME,COUNTER3 declare i1 @get_val() declare void @llvm.assume(i1) @@ -11,12 +11,12 @@ define dso_local i1 @test1(i32* readonly %0) { ; COUNTER1-LABEL: @test1( ; COUNTER1-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ] -; COUNTER1-NEXT: ret i1 false +; COUNTER1-NEXT: [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null +; COUNTER1-NEXT: ret i1 [[TMP2]] ; ; COUNTER2-LABEL: @test1( ; COUNTER2-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ] -; COUNTER2-NEXT: [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null -; COUNTER2-NEXT: ret i1 [[TMP2]] +; COUNTER2-NEXT: ret i1 false ; ; COUNTER3-LABEL: @test1( ; COUNTER3-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ] @@ -35,13 +35,13 @@ ; COUNTER1-NEXT: ret i1 [[TMP2]] ; ; COUNTER2-LABEL: @test2( -; COUNTER2-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ] -; COUNTER2-NEXT: ret i1 false +; COUNTER2-NEXT: [[TMP2:%.*]] = icmp eq i32* [[TMP0:%.*]], null +; COUNTER2-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0]]) ] +; COUNTER2-NEXT: ret i1 [[TMP2]] ; ; COUNTER3-LABEL: @test2( -; COUNTER3-NEXT: [[TMP2:%.*]] = icmp eq i32* [[TMP0:%.*]], null -; COUNTER3-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0]]) ] -; COUNTER3-NEXT: ret i1 [[TMP2]] +; COUNTER3-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ] +; COUNTER3-NEXT: ret i1 false ; %2 = icmp eq i32* %0, null call void @llvm.assume(i1 true) ["nonnull"(i32* %0)] @@ -49,48 +49,20 @@ } define dso_local i32 @test4(i32* readonly %0, i1 %cond) { -; COUNTER1-LABEL: @test4( -; COUNTER1-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ] -; COUNTER1-NEXT: br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]] -; COUNTER1: B: -; COUNTER1-NEXT: br label [[A]] -; COUNTER1: A: -; COUNTER1-NEXT: [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null -; COUNTER1-NEXT: br i1 [[TMP2]], label [[TMP5:%.*]], label [[TMP3:%.*]] -; COUNTER1: 3: -; COUNTER1-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP0]], align 4 -; COUNTER1-NEXT: br label [[TMP5]] -; COUNTER1: 5: -; COUNTER1-NEXT: [[TMP6:%.*]] = phi i32 [ [[TMP4]], [[TMP3]] ], [ 0, [[A]] ] -; COUNTER1-NEXT: ret i32 [[TMP6]] -; -; COUNTER2-LABEL: @test4( -; COUNTER2-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ] -; COUNTER2-NEXT: br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]] -; COUNTER2: B: -; COUNTER2-NEXT: br label [[A]] -; COUNTER2: A: -; COUNTER2-NEXT: br i1 false, label [[TMP4:%.*]], label [[TMP2:%.*]] -; COUNTER2: 2: -; COUNTER2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 4 -; COUNTER2-NEXT: br label [[TMP4]] -; COUNTER2: 4: -; COUNTER2-NEXT: [[TMP5:%.*]] = phi i32 [ [[TMP3]], [[TMP2]] ], [ 0, [[A]] ] -; COUNTER2-NEXT: ret i32 [[TMP5]] -; -; COUNTER3-LABEL: @test4( -; COUNTER3-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ] -; COUNTER3-NEXT: br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]] -; COUNTER3: B: -; COUNTER3-NEXT: br label [[A]] -; COUNTER3: A: -; COUNTER3-NEXT: br i1 false, label [[TMP4:%.*]], label [[TMP2:%.*]] -; COUNTER3: 2: -; COUNTER3-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 4 -; COUNTER3-NEXT: br label [[TMP4]] -; COUNTER3: 4: -; COUNTER3-NEXT: [[TMP5:%.*]] = phi i32 [ [[TMP3]], [[TMP2]] ], [ 0, [[A]] ] -; COUNTER3-NEXT: ret i32 [[TMP5]] +; SAME-LABEL: @test4( +; SAME-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ] +; SAME-NEXT: br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]] +; SAME: B: +; SAME-NEXT: br label [[A]] +; SAME: A: +; SAME-NEXT: [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null +; SAME-NEXT: br i1 [[TMP2]], label [[TMP5:%.*]], label [[TMP3:%.*]] +; SAME: 3: +; SAME-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP0]], align 4 +; SAME-NEXT: br label [[TMP5]] +; SAME: 5: +; SAME-NEXT: [[TMP6:%.*]] = phi i32 [ [[TMP4]], [[TMP3]] ], [ 0, [[A]] ] +; SAME-NEXT: ret i32 [[TMP6]] ; call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4)] br i1 %cond, label %A, label %B diff --git a/llvm/test/Analysis/ValueTracking/assume.ll b/llvm/test/Analysis/ValueTracking/assume.ll --- a/llvm/test/Analysis/ValueTracking/assume.ll +++ b/llvm/test/Analysis/ValueTracking/assume.ll @@ -59,20 +59,20 @@ define dso_local i32 @test4(i32* readonly %0, i1 %cond) { ; CHECK-LABEL: @test4( -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4), "align"(i32* [[TMP0]], i32 8) ] ; CHECK-NEXT: br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]] ; CHECK: B: ; CHECK-NEXT: br label [[A]] ; CHECK: A: ; CHECK-NEXT: br i1 false, label [[TMP4:%.*]], label [[TMP2:%.*]] ; CHECK: 2: -; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 8 ; CHECK-NEXT: br label [[TMP4]] ; CHECK: 4: ; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ [[TMP3]], [[TMP2]] ], [ 0, [[A]] ] ; CHECK-NEXT: ret i32 [[TMP5]] ; - call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4)] + call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4), "align"(i32* %0, i32 8)] br i1 %cond, label %A, label %B B: diff --git a/llvm/test/Transforms/InstCombine/assume.ll b/llvm/test/Transforms/InstCombine/assume.ll --- a/llvm/test/Transforms/InstCombine/assume.ll +++ b/llvm/test/Transforms/InstCombine/assume.ll @@ -11,13 +11,18 @@ ; been removed: define i32 @foo1(i32* %a) #0 { -; SAME-LABEL: @foo1( -; SAME-NEXT: [[T0:%.*]] = load i32, i32* [[A:%.*]], align 32 -; SAME-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64 -; SAME-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 -; SAME-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 -; SAME-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) -; SAME-NEXT: ret i32 [[T0]] +; DEFAULT-LABEL: @foo1( +; DEFAULT-NEXT: [[T0:%.*]] = load i32, i32* [[A:%.*]], align 32 +; DEFAULT-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64 +; DEFAULT-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 +; DEFAULT-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) +; DEFAULT-NEXT: ret i32 [[T0]] +; +; BUNDLES-LABEL: @foo1( +; BUNDLES-NEXT: [[T0:%.*]] = load i32, i32* [[A:%.*]], align 32 +; BUNDLES-NEXT: call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 32) ] +; BUNDLES-NEXT: ret i32 [[T0]] ; %t0 = load i32, i32* %a, align 4 %ptrint = ptrtoint i32* %a to i64 @@ -30,13 +35,18 @@ ; Same check as in @foo1, but make sure it works if the assume is first too. define i32 @foo2(i32* %a) #0 { -; SAME-LABEL: @foo2( -; SAME-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A:%.*]] to i64 -; SAME-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 -; SAME-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 -; SAME-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) -; SAME-NEXT: [[T0:%.*]] = load i32, i32* [[A]], align 32 -; SAME-NEXT: ret i32 [[T0]] +; DEFAULT-LABEL: @foo2( +; DEFAULT-NEXT: [[PTRINT:%.*]] = ptrtoint i32* [[A:%.*]] to i64 +; DEFAULT-NEXT: [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31 +; DEFAULT-NEXT: [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0 +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]]) +; DEFAULT-NEXT: [[T0:%.*]] = load i32, i32* [[A]], align 32 +; DEFAULT-NEXT: ret i32 [[T0]] +; +; BUNDLES-LABEL: @foo2( +; BUNDLES-NEXT: call void @llvm.assume(i1 true) [ "align"(i32* [[A:%.*]], i64 32) ] +; BUNDLES-NEXT: [[T0:%.*]] = load i32, i32* [[A]], align 32 +; BUNDLES-NEXT: ret i32 [[T0]] ; %ptrint = ptrtoint i32* %a to i64 %maskedptr = and i64 %ptrint, 31 @@ -211,6 +221,17 @@ ret i32 %load } +define i32 @bundle1A(i32* %P) { +; SAME-LABEL: @bundle1A( +; SAME-NEXT: tail call void @llvm.assume(i1 true) [ "align"(i32* [[P:%.*]], i32 64) ] +; SAME-NEXT: [[LOAD:%.*]] = load i32, i32* [[P]], align 64 +; SAME-NEXT: ret i32 [[LOAD]] +; + tail call void @llvm.assume(i1 true) ["align"(i32* %P, i32 64)] + %load = load i32, i32* %P + ret i32 %load +} + define i32 @bundle2(i32* %P) { ; SAME-LABEL: @bundle2( ; SAME-NEXT: [[LOAD:%.*]] = load i32, i32* [[P:%.*]], align 4 @@ -256,17 +277,28 @@ ; if the assume is control dependent on something else define i1 @nonnull3(i32** %a, i1 %control) { -; SAME-LABEL: @nonnull3( -; SAME-NEXT: entry: -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] -; SAME: taken: -; SAME-NEXT: tail call void @llvm.assume(i1 [[CMP]]) -; SAME-NEXT: ret i1 false -; SAME: not_taken: -; SAME-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null -; SAME-NEXT: ret i1 [[RVAL_2]] +; FIXME: in the BUNDLES version we could duplicate the load and keep the assume nonnull. +; DEFAULT-LABEL: @nonnull3( +; DEFAULT-NEXT: entry: +; DEFAULT-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 +; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null +; DEFAULT-NEXT: br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; DEFAULT: taken: +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[CMP]]) +; DEFAULT-NEXT: ret i1 false +; DEFAULT: not_taken: +; DEFAULT-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; DEFAULT-NEXT: ret i1 [[RVAL_2]] +; +; BUNDLES-LABEL: @nonnull3( +; BUNDLES-NEXT: entry: +; BUNDLES-NEXT: br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; BUNDLES: taken: +; BUNDLES-NEXT: ret i1 false +; BUNDLES: not_taken: +; BUNDLES-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 +; BUNDLES-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; BUNDLES-NEXT: ret i1 [[RVAL_2]] ; entry: %load = load i32*, i32** %a @@ -286,12 +318,18 @@ ; interrupted by an exception being thrown define i1 @nonnull4(i32** %a) { -; SAME-LABEL: @nonnull4( -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 -; SAME-NEXT: tail call void @escape(i32* [[LOAD]]) -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: tail call void @llvm.assume(i1 [[CMP]]) -; SAME-NEXT: ret i1 false +; DEFAULT-LABEL: @nonnull4( +; DEFAULT-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 +; DEFAULT-NEXT: tail call void @escape(i32* [[LOAD]]) +; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null +; DEFAULT-NEXT: tail call void @llvm.assume(i1 [[CMP]]) +; DEFAULT-NEXT: ret i1 false +; +; BUNDLES-LABEL: @nonnull4( +; BUNDLES-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 +; BUNDLES-NEXT: tail call void @escape(i32* [[LOAD]]) +; BUNDLES-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]) ] +; BUNDLES-NEXT: ret i1 false ; %load = load i32*, i32** %a ;; This call may throw! @@ -347,7 +385,6 @@ define void @debug_interference(i8 %x) { ; SAME-LABEL: @debug_interference( ; SAME-NEXT: [[CMP2:%.*]] = icmp ne i8 [[X:%.*]], 0 -; SAME-NEXT: tail call void @llvm.assume(i1 false) ; SAME-NEXT: tail call void @llvm.dbg.value(metadata i32 5, [[META7:metadata !.*]], metadata !DIExpression()), [[DBG9:!dbg !.*]] ; SAME-NEXT: tail call void @llvm.assume(i1 false) ; SAME-NEXT: tail call void @llvm.dbg.value(metadata i32 5, [[META7]], metadata !DIExpression()), [[DBG9]] @@ -386,17 +423,27 @@ } define i1 @nonnull3A(i32** %a, i1 %control) { -; SAME-LABEL: @nonnull3A( -; SAME-NEXT: entry: -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 -; SAME-NEXT: br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] -; SAME: taken: -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: call void @llvm.assume(i1 [[CMP]]) -; SAME-NEXT: ret i1 true -; SAME: not_taken: -; SAME-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null -; SAME-NEXT: ret i1 [[RVAL_2]] +; DEFAULT-LABEL: @nonnull3A( +; DEFAULT-NEXT: entry: +; DEFAULT-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 +; DEFAULT-NEXT: br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; DEFAULT: taken: +; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null +; DEFAULT-NEXT: call void @llvm.assume(i1 [[CMP]]) +; DEFAULT-NEXT: ret i1 true +; DEFAULT: not_taken: +; DEFAULT-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; DEFAULT-NEXT: ret i1 [[RVAL_2]] +; +; BUNDLES-LABEL: @nonnull3A( +; BUNDLES-NEXT: entry: +; BUNDLES-NEXT: br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] +; BUNDLES: taken: +; BUNDLES-NEXT: ret i1 true +; BUNDLES: not_taken: +; BUNDLES-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 +; BUNDLES-NEXT: [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null +; BUNDLES-NEXT: ret i1 [[RVAL_2]] ; entry: %load = load i32*, i32** %a @@ -414,13 +461,13 @@ define i1 @nonnull3B(i32** %a, i1 %control) { ; SAME-LABEL: @nonnull3B( ; SAME-NEXT: entry: +; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 ; SAME-NEXT: br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]] ; SAME: taken: -; SAME-NEXT: [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8 -; SAME-NEXT: [[CMP:%.*]] = icmp ne i32* [[LOAD]], null -; SAME-NEXT: call void @llvm.assume(i1 [[CMP]]) [ "nonnull"(i32* [[LOAD]]), "nonnull"(i1 [[CMP]]) ] +; SAME-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]), "nonnull"(i1 true) ] ; SAME-NEXT: ret i1 true ; SAME: not_taken: +; SAME-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]), "ignore"(i1 undef) ] ; SAME-NEXT: ret i1 [[CONTROL]] ; entry: @@ -635,6 +682,17 @@ unreachable } +define void @canonicalize_assume(i32* %0) { +; SAME-LABEL: @canonicalize_assume( +; SAME-NEXT: call void @llvm.assume(i1 true) [ "align"(i32* [[TMP0:%.*]], i64 8) ] +; SAME-NEXT: ret void +; + %2 = getelementptr inbounds i32, i32* %0, i64 2 + %3 = bitcast i32* %2 to i8* + call void @llvm.assume(i1 true) [ "align"(i8* %3, i64 16) ] + ret void +} + declare void @llvm.dbg.value(metadata, metadata, metadata) !llvm.dbg.cu = !{!0}