diff --git a/llvm/include/llvm/Analysis/AssumptionCache.h b/llvm/include/llvm/Analysis/AssumptionCache.h --- a/llvm/include/llvm/Analysis/AssumptionCache.h +++ b/llvm/include/llvm/Analysis/AssumptionCache.h @@ -26,7 +26,7 @@ namespace llvm { -class AssumeInst; +class CondGuardInst; class Function; class raw_ostream; class TargetTransformInfo; @@ -120,15 +120,15 @@ /// /// The call passed in must be an instruction within this function and must /// not already be in the cache. - void registerAssumption(AssumeInst *CI); + void registerAssumption(CondGuardInst *CI); /// Remove an \@llvm.assume intrinsic from this function's cache if it has /// been added to the cache earlier. - void unregisterAssumption(AssumeInst *CI); + void unregisterAssumption(CondGuardInst *CI); /// Update the cache of values being affected by this assumption (i.e. /// the values about which this assumption provides information). - void updateAffectedValues(AssumeInst *CI); + void updateAffectedValues(CondGuardInst *CI); /// Clear the cache of \@llvm.assume intrinsics for a function. /// diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -1513,9 +1513,20 @@ } }; +/// This represents intrinsics that guard a condition +class CondGuardInst : public IntrinsicInst { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::assume || + I->getIntrinsicID() == Intrinsic::experimental_guard; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; /// This represents the llvm.assume intrinsic. -class AssumeInst : public IntrinsicInst { +class AssumeInst : public CondGuardInst { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::assume; diff --git a/llvm/lib/Analysis/AssumeBundleQueries.cpp b/llvm/lib/Analysis/AssumeBundleQueries.cpp --- a/llvm/lib/Analysis/AssumeBundleQueries.cpp +++ b/llvm/lib/Analysis/AssumeBundleQueries.cpp @@ -162,7 +162,7 @@ return RetainedKnowledge::none(); if (AC) { for (AssumptionCache::ResultElem &Elem : AC->assumptionsFor(V)) { - auto *II = cast_or_null(Elem.Assume); + auto *II = dyn_cast_or_null(Elem.Assume); if (!II || Elem.Index == AssumptionCache::ExprResultIdx) continue; if (RetainedKnowledge RK = getKnowledgeFromBundle( diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp --- a/llvm/lib/Analysis/AssumptionCache.cpp +++ b/llvm/lib/Analysis/AssumptionCache.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// // -// This file contains a pass that keeps track of @llvm.assume intrinsics in -// the functions of a module. +// This file contains a pass that keeps track of @llvm.assume and +// @llvm.experimental.guard intrinsics in the functions of a module. // //===----------------------------------------------------------------------===// @@ -140,7 +140,7 @@ } } -void AssumptionCache::updateAffectedValues(AssumeInst *CI) { +void AssumptionCache::updateAffectedValues(CondGuardInst *CI) { SmallVector Affected; findAffectedValues(CI, TTI, Affected); @@ -153,7 +153,7 @@ } } -void AssumptionCache::unregisterAssumption(AssumeInst *CI) { +void AssumptionCache::unregisterAssumption(CondGuardInst *CI) { SmallVector Affected; findAffectedValues(CI, TTI, Affected); @@ -217,7 +217,7 @@ // to this cache. for (BasicBlock &B : F) for (Instruction &I : B) - if (isa(&I)) + if (isa(&I)) AssumeHandles.push_back({&I, ExprResultIdx}); // Mark the scan as complete. @@ -225,10 +225,10 @@ // Update affected values. for (auto &A : AssumeHandles) - updateAffectedValues(cast(A)); + updateAffectedValues(cast(A)); } -void AssumptionCache::registerAssumption(AssumeInst *CI) { +void AssumptionCache::registerAssumption(CondGuardInst *CI) { // If we haven't scanned the function yet, just drop this assumption. It will // be found when we scan later. if (!Scanned) @@ -238,9 +238,9 @@ #ifndef NDEBUG assert(CI->getParent() && - "Cannot register @llvm.assume call not in a basic block"); + "Cannot a register CondGuardInst not in a basic block"); assert(&F == CI->getParent()->getParent() && - "Cannot register @llvm.assume call not in this function"); + "Cannot a register CondGuardInst not in this function"); // We expect the number of assumptions to be small, so in an asserts build // check that we don't accumulate duplicates and that all assumptions point @@ -252,8 +252,8 @@ assert(&F == cast(VH)->getParent()->getParent() && "Cached assumption not inside this function!"); - assert(match(cast(VH), m_Intrinsic()) && - "Cached something other than a call to @llvm.assume!"); + assert(isa(VH) && + "Cached something other than CondGuardInst!"); assert(AssumptionSet.insert(VH).second && "Cache contains multiple copies of a call!"); } diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -1771,8 +1771,7 @@ // these to compute max backedge taken counts, but can still use // these to prove lack of overflow. Use this fact to avoid // doing extra work that may not pay off. - if (!isa(MaxBECount) || HasGuards || - !AC.assumptions().empty()) { + if (!isa(MaxBECount) || !AC.assumptions().empty()) { auto NewFlags = proveNoUnsignedWrapViaInduction(AR); setNoWrapFlags(const_cast(AR), NewFlags); @@ -5113,8 +5112,7 @@ // these to prove lack of overflow. Use this fact to avoid // doing extra work that may not pay off. - if (isa(MaxBECount) && !HasGuards && - AC.assumptions().empty()) + if (isa(MaxBECount) && AC.assumptions().empty()) return Result; // If the backedge is guarded by a comparison with the pre-inc value the @@ -5167,8 +5165,7 @@ // these to prove lack of overflow. Use this fact to avoid // doing extra work that may not pay off. - if (isa(MaxBECount) && !HasGuards && - AC.assumptions().empty()) + if (isa(MaxBECount) && AC.assumptions().empty()) return Result; // If the backedge is guarded by a comparison with the pre-inc value the @@ -11356,7 +11353,7 @@ ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { // No need to even try if we know the module has no guards. - if (!HasGuards) + if (AC.assumptions().empty()) return false; return any_of(*BB, [&](const Instruction &I) { @@ -11566,15 +11563,6 @@ return true; } - // Check conditions due to any @llvm.experimental.guard intrinsics. - auto *GuardDecl = F.getParent()->getFunction( - Intrinsic::getName(Intrinsic::experimental_guard)); - if (GuardDecl) - for (const auto *GU : GuardDecl->users()) - if (const auto *Guard = dyn_cast(GU)) - if (Guard->getFunction() == BB->getParent() && DT.dominates(Guard, BB)) - if (ProveViaCond(Guard->getArgOperand(0), false)) - return true; return false; } @@ -13447,25 +13435,11 @@ LoopInfo &LI) : F(F), TLI(TLI), AC(AC), DT(DT), LI(LI), CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64), - LoopDispositions(64), BlockDispositions(64) { - // To use guards for proving predicates, we need to scan every instruction in - // relevant basic blocks, and not just terminators. Doing this is a waste of - // time if the IR does not actually contain any calls to - // @llvm.experimental.guard, so do a quick check and remember this beforehand. - // - // This pessimizes the case where a pass that preserves ScalarEvolution wants - // to _add_ guards to the module when there weren't any before, and wants - // ScalarEvolution to optimize based on those guards. For now we prefer to be - // efficient in lieu of being smart in that rather obscure case. - - auto *GuardDecl = F.getParent()->getFunction( - Intrinsic::getName(Intrinsic::experimental_guard)); - HasGuards = GuardDecl && !GuardDecl->use_empty(); -} + LoopDispositions(64), BlockDispositions(64) {} ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg) - : F(Arg.F), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), - LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)), + : F(Arg.F), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), LI(Arg.LI), + CouldNotCompute(std::move(Arg.CouldNotCompute)), ValueExprMap(std::move(Arg.ValueExprMap)), PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)), PendingPhiRanges(std::move(Arg.PendingPhiRanges)), @@ -15138,16 +15112,7 @@ Terms.emplace_back(AssumeI->getOperand(0), true); } - // Second, collect information from llvm.experimental.guards dominating the loop. - auto *GuardDecl = F.getParent()->getFunction( - Intrinsic::getName(Intrinsic::experimental_guard)); - if (GuardDecl) - for (const auto *GU : GuardDecl->users()) - if (const auto *Guard = dyn_cast(GU)) - if (Guard->getFunction() == Header->getParent() && DT.dominates(Guard, Header)) - Terms.emplace_back(Guard->getArgOperand(0), true); - - // Third, collect conditions from dominating branches. Starting at the loop + // Second, collect conditions from dominating branches. Starting at the loop // predecessor, climb up the predecessor chain, as long as there are // predecessors that can be found that have unique successors leading to the // original header. diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -616,7 +616,7 @@ for (auto &AssumeVH : Q.AC->assumptionsFor(V)) { if (!AssumeVH) continue; - CallInst *I = cast(AssumeVH); + CondGuardInst *I = cast(AssumeVH); assert(I->getFunction() == Q.CxtI->getFunction() && "Got assumption for the wrong function!"); @@ -624,9 +624,6 @@ // We're running this loop for once for each value queried resulting in a // runtime of ~O(#assumes * #values). - assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && - "must be an assume intrinsic"); - Value *RHS; CmpInst::Predicate Pred; auto m_V = m_CombineOr(m_Specific(V), m_PtrToInt(m_Specific(V))); @@ -664,7 +661,7 @@ for (auto &AssumeVH : Q.AC->assumptionsFor(V)) { if (!AssumeVH) continue; - CallInst *I = cast(AssumeVH); + CondGuardInst *I = cast(AssumeVH); assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() && "Got assumption for the wrong function!"); @@ -672,9 +669,6 @@ // We're running this loop for once for each value queried resulting in a // runtime of ~O(#assumes * #values). - assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && - "must be an assume intrinsic"); - Value *Arg = I->getArgOperand(0); if (Arg == V && isValidAssumeForContext(I, Q.CxtI, Q.DT)) { @@ -7446,11 +7440,9 @@ for (auto &AssumeVH : AC->assumptionsFor(V)) { if (!AssumeVH) continue; - CallInst *I = cast(AssumeVH); + IntrinsicInst *I = cast(AssumeVH); assert(I->getParent()->getParent() == CtxI->getParent()->getParent() && "Got assumption for the wrong function!"); - assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && - "must be an assume intrinsic"); if (!isValidAssumeForContext(I, CtxI, DT)) continue; diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -1663,14 +1663,14 @@ } } - // Remove @llvm.assume calls that will be moved to the new function from the - // old function's assumption cache. + // Remove CondGuardInsts that will be moved to the new function from the old + // function's assumption cache. for (BasicBlock *Block : Blocks) { for (Instruction &I : llvm::make_early_inc_range(*Block)) { - if (auto *AI = dyn_cast(&I)) { + if (auto *CI = dyn_cast(&I)) { if (AC) - AC->unregisterAssumption(AI); - AI->eraseFromParent(); + AC->unregisterAssumption(CI); + CI->eraseFromParent(); } } } @@ -1864,7 +1864,7 @@ const Function &NewFunc, AssumptionCache *AC) { for (auto AssumeVH : AC->assumptions()) { - auto *I = dyn_cast_or_null(AssumeVH); + auto *I = dyn_cast_or_null(AssumeVH); if (!I) continue; @@ -1876,7 +1876,7 @@ // that were previously in the old function, but that have now been moved // to the new function. for (auto AffectedValVH : AC->assumptionsFor(I->getOperand(0))) { - auto *AffectedCI = dyn_cast_or_null(AffectedValVH); + auto *AffectedCI = dyn_cast_or_null(AffectedValVH); if (!AffectedCI) continue; if (AffectedCI->getFunction() != &OldFunc) diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2333,7 +2333,7 @@ for (BasicBlock &NewBlock : make_range(FirstNewBlock->getIterator(), Caller->end())) for (Instruction &I : NewBlock) - if (auto *II = dyn_cast(&I)) + if (auto *II = dyn_cast(&I)) IFI.GetAssumptionCache(*Caller).registerAssumption(II); } diff --git a/llvm/test/Analysis/AssumptionCache/basic.ll b/llvm/test/Analysis/AssumptionCache/basic.ll --- a/llvm/test/Analysis/AssumptionCache/basic.ll +++ b/llvm/test/Analysis/AssumptionCache/basic.ll @@ -3,12 +3,15 @@ target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128" declare void @llvm.assume(i1) +declare void @llvm.experimental.guard(i1, ...) define void @test1(i32 %a) { ; CHECK-LABEL: Cached assumptions for function: test1 ; CHECK-NEXT: icmp ne i32 %{{.*}}, 0 ; CHECK-NEXT: icmp slt i32 %{{.*}}, 0 ; CHECK-NEXT: icmp sgt i32 %{{.*}}, 0 +; CHECK-NEXT: icmp ult i32 %{{.*}}, 0 +; CHECK-NEXT: icmp ugt i32 %{{.*}}, 0 entry: %cond1 = icmp ne i32 %a, 0 @@ -17,6 +20,10 @@ call void @llvm.assume(i1 %cond2) %cond3 = icmp sgt i32 %a, 0 call void @llvm.assume(i1 %cond3) + %cond4 = icmp ult i32 %a, 0 + call void (i1, ...) @llvm.experimental.guard(i1 %cond4) [ "deopt"() ] + %cond5 = icmp ugt i32 %a, 0 + call void (i1, ...) @llvm.experimental.guard(i1 %cond5) [ "deopt"() ] ret void } diff --git a/llvm/test/Analysis/ScalarEvolution/guards.ll b/llvm/test/Analysis/ScalarEvolution/guards.ll --- a/llvm/test/Analysis/ScalarEvolution/guards.ll +++ b/llvm/test/Analysis/ScalarEvolution/guards.ll @@ -86,7 +86,7 @@ loop: ; CHECK: loop: ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"() ] -; CHECK: %iv.inc.cmp = icmp slt i32 %iv.inc, %len +; CHECK: %iv.inc.cmp = icmp ult i32 %iv.inc, %len ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %iv.inc.cmp) [ "deopt"() ] ; CHECK: leave: %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ] @@ -129,7 +129,7 @@ be: ; CHECK: be: -; CHECK-NEXT: %iv.cmp = icmp slt i32 %iv, %len +; CHECK-NEXT: %iv.cmp = icmp ult i32 %iv, %len ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 %iv.cmp) [ "deopt"() ] ; CHECK: leave: diff --git a/llvm/test/Transforms/EarlyCSE/guards.ll b/llvm/test/Transforms/EarlyCSE/guards.ll --- a/llvm/test/Transforms/EarlyCSE/guards.ll +++ b/llvm/test/Transforms/EarlyCSE/guards.ll @@ -83,13 +83,10 @@ ; CHECK-LABEL: @test3.unhandled( ; CHECK-NEXT: [[COND0:%.*]] = icmp slt i32 [[VAL:%.*]], 40 ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[COND0]]) [ "deopt"() ] -; CHECK-NEXT: [[COND1:%.*]] = icmp sge i32 [[VAL]], 40 -; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[COND1]]) [ "deopt"() ] +; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: ret i32 0 ; -; Demonstrates a case we do not yet handle (it is legal to fold %cond2 -; to false) %cond0 = icmp slt i32 %val, 40 call void(i1,...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ] %cond1 = icmp sge i32 %val, 40