diff --git a/llvm/include/llvm/Analysis/AssumptionCache.h b/llvm/include/llvm/Analysis/AssumptionCache.h --- a/llvm/include/llvm/Analysis/AssumptionCache.h +++ b/llvm/include/llvm/Analysis/AssumptionCache.h @@ -26,8 +26,8 @@ namespace llvm { -class AssumeInst; class Function; +class IntrinsicInst; class raw_ostream; class TargetTransformInfo; class Value; @@ -103,6 +103,8 @@ /// Scan the function for assumptions and add them to the cache. void scanFunction(); + void assertIsAssume(IntrinsicInst *II); + public: /// Construct an AssumptionCache from a function by scanning all of /// its instructions. @@ -120,15 +122,15 @@ /// /// The call passed in must be an instruction within this function and must /// not already be in the cache. - void registerAssumption(AssumeInst *CI); + void registerAssumption(IntrinsicInst *II); /// Remove an \@llvm.assume intrinsic from this function's cache if it has /// been added to the cache earlier. - void unregisterAssumption(AssumeInst *CI); + void unregisterAssumption(IntrinsicInst *II); /// Update the cache of values being affected by this assumption (i.e. /// the values about which this assumption provides information). - void updateAffectedValues(AssumeInst *CI); + void updateAffectedValues(IntrinsicInst *II); /// Clear the cache of \@llvm.assume intrinsics for a function. /// diff --git a/llvm/lib/Analysis/AssumeBundleQueries.cpp b/llvm/lib/Analysis/AssumeBundleQueries.cpp --- a/llvm/lib/Analysis/AssumeBundleQueries.cpp +++ b/llvm/lib/Analysis/AssumeBundleQueries.cpp @@ -162,7 +162,7 @@ return RetainedKnowledge::none(); if (AC) { for (AssumptionCache::ResultElem &Elem : AC->assumptionsFor(V)) { - auto *II = cast_or_null(Elem.Assume); + auto *II = dyn_cast_or_null(Elem.Assume); if (!II || Elem.Index == AssumptionCache::ExprResultIdx) continue; if (RetainedKnowledge RK = getKnowledgeFromBundle( diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp --- a/llvm/lib/Analysis/AssumptionCache.cpp +++ b/llvm/lib/Analysis/AssumptionCache.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// // -// This file contains a pass that keeps track of @llvm.assume intrinsics in -// the functions of a module. +// This file contains a pass that keeps track of @llvm.assume and +// @llvm.experimental.guard intrinsics in the functions of a module. // //===----------------------------------------------------------------------===// @@ -140,7 +140,8 @@ } } -void AssumptionCache::updateAffectedValues(AssumeInst *CI) { +void AssumptionCache::updateAffectedValues(IntrinsicInst *CI) { + assertIsAssume(CI); SmallVector Affected; findAffectedValues(CI, TTI, Affected); @@ -153,7 +154,8 @@ } } -void AssumptionCache::unregisterAssumption(AssumeInst *CI) { +void AssumptionCache::unregisterAssumption(IntrinsicInst *CI) { + assertIsAssume(CI); SmallVector Affected; findAffectedValues(CI, TTI, Affected); @@ -217,18 +219,26 @@ // to this cache. for (BasicBlock &B : F) for (Instruction &I : B) - if (isa(&I)) - AssumeHandles.push_back({&I, ExprResultIdx}); + if (IntrinsicInst *II = dyn_cast(&I)) + if (II->getIntrinsicID() == Intrinsic::assume || + II->getIntrinsicID() == Intrinsic::experimental_guard) + AssumeHandles.push_back({&I, ExprResultIdx}); // Mark the scan as complete. Scanned = true; // Update affected values. for (auto &A : AssumeHandles) - updateAffectedValues(cast(A)); + updateAffectedValues(cast(A)); } -void AssumptionCache::registerAssumption(AssumeInst *CI) { +void AssumptionCache::assertIsAssume(IntrinsicInst *CI) { + assert(CI->getIntrinsicID() == Intrinsic::assume || + CI->getIntrinsicID() == Intrinsic::experimental_guard); +} + +void AssumptionCache::registerAssumption(IntrinsicInst *CI) { + assertIsAssume(CI); // If we haven't scanned the function yet, just drop this assumption. It will // be found when we scan later. if (!Scanned) @@ -238,9 +248,11 @@ #ifndef NDEBUG assert(CI->getParent() && - "Cannot register @llvm.assume call not in a basic block"); + "Cannot register @llvm.assume or @llvm.experimental.guard call not in " + "a basic block"); assert(&F == CI->getParent()->getParent() && - "Cannot register @llvm.assume call not in this function"); + "Cannot register @llvm.assume or @llvm.experimental.guard call not in " + "this function"); // We expect the number of assumptions to be small, so in an asserts build // check that we don't accumulate duplicates and that all assumptions point @@ -252,8 +264,11 @@ assert(&F == cast(VH)->getParent()->getParent() && "Cached assumption not inside this function!"); - assert(match(cast(VH), m_Intrinsic()) && - "Cached something other than a call to @llvm.assume!"); + assert((match(cast(VH), m_Intrinsic()) || + match(cast(VH), + m_Intrinsic())) && + "Cached something other than a call to @llvm.assume or " + "@llvm.experimental.guard!"); assert(AssumptionSet.insert(VH).second && "Cache contains multiple copies of a call!"); } diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -1813,8 +1813,7 @@ // these to compute max backedge taken counts, but can still use // these to prove lack of overflow. Use this fact to avoid // doing extra work that may not pay off. - if (!isa(MaxBECount) || HasGuards || - !AC.assumptions().empty()) { + if (!isa(MaxBECount) || !AC.assumptions().empty()) { auto NewFlags = proveNoUnsignedWrapViaInduction(AR); setNoWrapFlags(const_cast(AR), NewFlags); @@ -5105,8 +5104,7 @@ // these to prove lack of overflow. Use this fact to avoid // doing extra work that may not pay off. - if (isa(MaxBECount) && !HasGuards && - AC.assumptions().empty()) + if (isa(MaxBECount) && AC.assumptions().empty()) return Result; // If the backedge is guarded by a comparison with the pre-inc value the @@ -5159,8 +5157,7 @@ // these to prove lack of overflow. Use this fact to avoid // doing extra work that may not pay off. - if (isa(MaxBECount) && !HasGuards && - AC.assumptions().empty()) + if (isa(MaxBECount) && AC.assumptions().empty()) return Result; // If the backedge is guarded by a comparison with the pre-inc value the @@ -11282,7 +11279,7 @@ ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { // No need to even try if we know the module has no guards. - if (!HasGuards) + if (AC.assumptions().empty()) return false; return any_of(*BB, [&](const Instruction &I) { @@ -11492,15 +11489,6 @@ return true; } - // Check conditions due to any @llvm.experimental.guard intrinsics. - auto *GuardDecl = F.getParent()->getFunction( - Intrinsic::getName(Intrinsic::experimental_guard)); - if (GuardDecl) - for (const auto *GU : GuardDecl->users()) - if (const auto *Guard = dyn_cast(GU)) - if (Guard->getFunction() == BB->getParent() && DT.dominates(Guard, BB)) - if (ProveViaCond(Guard->getArgOperand(0), false)) - return true; return false; } @@ -13373,25 +13361,11 @@ LoopInfo &LI) : F(F), TLI(TLI), AC(AC), DT(DT), LI(LI), CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64), - LoopDispositions(64), BlockDispositions(64) { - // To use guards for proving predicates, we need to scan every instruction in - // relevant basic blocks, and not just terminators. Doing this is a waste of - // time if the IR does not actually contain any calls to - // @llvm.experimental.guard, so do a quick check and remember this beforehand. - // - // This pessimizes the case where a pass that preserves ScalarEvolution wants - // to _add_ guards to the module when there weren't any before, and wants - // ScalarEvolution to optimize based on those guards. For now we prefer to be - // efficient in lieu of being smart in that rather obscure case. - - auto *GuardDecl = F.getParent()->getFunction( - Intrinsic::getName(Intrinsic::experimental_guard)); - HasGuards = GuardDecl && !GuardDecl->use_empty(); -} + LoopDispositions(64), BlockDispositions(64) {} ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg) - : F(Arg.F), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), - LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)), + : F(Arg.F), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), LI(Arg.LI), + CouldNotCompute(std::move(Arg.CouldNotCompute)), ValueExprMap(std::move(Arg.ValueExprMap)), PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)), PendingPhiRanges(std::move(Arg.PendingPhiRanges)), @@ -15088,16 +15062,7 @@ Terms.emplace_back(AssumeI->getOperand(0), true); } - // Second, collect information from llvm.experimental.guards dominating the loop. - auto *GuardDecl = F.getParent()->getFunction( - Intrinsic::getName(Intrinsic::experimental_guard)); - if (GuardDecl) - for (const auto *GU : GuardDecl->users()) - if (const auto *Guard = dyn_cast(GU)) - if (Guard->getFunction() == Header->getParent() && DT.dominates(Guard, Header)) - Terms.emplace_back(Guard->getArgOperand(0), true); - - // Third, collect conditions from dominating branches. Starting at the loop + // Second, collect conditions from dominating branches. Starting at the loop // predecessor, climb up the predecessor chain, as long as there are // predecessors that can be found that have unique successors leading to the // original header. diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -616,7 +616,7 @@ for (auto &AssumeVH : Q.AC->assumptionsFor(V)) { if (!AssumeVH) continue; - CallInst *I = cast(AssumeVH); + IntrinsicInst *I = cast(AssumeVH); assert(I->getFunction() == Q.CxtI->getFunction() && "Got assumption for the wrong function!"); @@ -624,8 +624,9 @@ // We're running this loop for once for each value queried resulting in a // runtime of ~O(#assumes * #values). - assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && - "must be an assume intrinsic"); + Intrinsic::ID ID = I->getCalledFunction()->getIntrinsicID(); + assert((ID == Intrinsic::assume || ID == Intrinsic::experimental_guard) && + "must be an assume or experimental.guard intrinsic"); Value *RHS; CmpInst::Predicate Pred; @@ -664,7 +665,7 @@ for (auto &AssumeVH : Q.AC->assumptionsFor(V)) { if (!AssumeVH) continue; - CallInst *I = cast(AssumeVH); + IntrinsicInst *I = cast(AssumeVH); assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() && "Got assumption for the wrong function!"); @@ -672,8 +673,9 @@ // We're running this loop for once for each value queried resulting in a // runtime of ~O(#assumes * #values). - assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && - "must be an assume intrinsic"); + Intrinsic::ID ID = I->getCalledFunction()->getIntrinsicID(); + assert((ID == Intrinsic::assume || ID == Intrinsic::experimental_guard) && + "must be an assume or experimental.guard intrinsic"); Value *Arg = I->getArgOperand(0); @@ -7443,10 +7445,11 @@ for (auto &AssumeVH : AC->assumptionsFor(V)) { if (!AssumeVH) continue; - CallInst *I = cast(AssumeVH); + IntrinsicInst *I = cast(AssumeVH); assert(I->getParent()->getParent() == CtxI->getParent()->getParent() && "Got assumption for the wrong function!"); - assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && + Intrinsic::ID ID = I->getCalledFunction()->getIntrinsicID(); + assert((ID == Intrinsic::assume || ID == Intrinsic::experimental_guard) && "must be an assume intrinsic"); if (!isValidAssumeForContext(I, CtxI, DT)) diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -1663,14 +1663,17 @@ } } - // Remove @llvm.assume calls that will be moved to the new function from the - // old function's assumption cache. + // Remove @llvm.assume and @llvm.experimental.guard calls that will be moved + // to the new function from the old function's assumption cache. for (BasicBlock *Block : Blocks) { for (Instruction &I : llvm::make_early_inc_range(*Block)) { - if (auto *AI = dyn_cast(&I)) { - if (AC) - AC->unregisterAssumption(AI); - AI->eraseFromParent(); + if (auto *II = dyn_cast(&I)) { + if (II->getIntrinsicID() == Intrinsic::assume || + II->getIntrinsicID() == Intrinsic::experimental_guard) { + if (AC) + AC->unregisterAssumption(II); + II->eraseFromParent(); + } } } } @@ -1865,7 +1868,8 @@ AssumptionCache *AC) { for (auto AssumeVH : AC->assumptions()) { auto *I = dyn_cast_or_null(AssumeVH); - if (!I) + if (!I || (I->getIntrinsicID() != Intrinsic::assume && + I->getIntrinsicID() != Intrinsic::experimental_guard)) continue; // There shouldn't be any llvm.assume intrinsics in the new function. @@ -1876,7 +1880,7 @@ // that were previously in the old function, but that have now been moved // to the new function. for (auto AffectedValVH : AC->assumptionsFor(I->getOperand(0))) { - auto *AffectedCI = dyn_cast_or_null(AffectedValVH); + auto *AffectedCI = dyn_cast_or_null(AffectedValVH); if (!AffectedCI) continue; if (AffectedCI->getFunction() != &OldFunc) diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2333,8 +2333,10 @@ for (BasicBlock &NewBlock : make_range(FirstNewBlock->getIterator(), Caller->end())) for (Instruction &I : NewBlock) - if (auto *II = dyn_cast(&I)) - IFI.GetAssumptionCache(*Caller).registerAssumption(II); + if (auto *II = dyn_cast(&I)) + if (II->getIntrinsicID() == Intrinsic::assume || + II->getIntrinsicID() == Intrinsic::experimental_guard) + IFI.GetAssumptionCache(*Caller).registerAssumption(II); } // If there are any alloca instructions in the block that used to be the entry diff --git a/llvm/test/Analysis/ScalarEvolution/guards.ll b/llvm/test/Analysis/ScalarEvolution/guards.ll --- a/llvm/test/Analysis/ScalarEvolution/guards.ll +++ b/llvm/test/Analysis/ScalarEvolution/guards.ll @@ -86,7 +86,7 @@ loop: ; CHECK: loop: ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"() ] -; CHECK: %iv.inc.cmp = icmp slt i32 %iv.inc, %len +; CHECK: %iv.inc.cmp = icmp ult i32 %iv.inc, %len ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %iv.inc.cmp) [ "deopt"() ] ; CHECK: leave: %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ] @@ -129,7 +129,7 @@ be: ; CHECK: be: -; CHECK-NEXT: %iv.cmp = icmp slt i32 %iv, %len +; CHECK-NEXT: %iv.cmp = icmp ult i32 %iv, %len ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 %iv.cmp) [ "deopt"() ] ; CHECK: leave: diff --git a/llvm/test/Transforms/EarlyCSE/guards.ll b/llvm/test/Transforms/EarlyCSE/guards.ll --- a/llvm/test/Transforms/EarlyCSE/guards.ll +++ b/llvm/test/Transforms/EarlyCSE/guards.ll @@ -83,13 +83,10 @@ ; CHECK-LABEL: @test3.unhandled( ; CHECK-NEXT: [[COND0:%.*]] = icmp slt i32 [[VAL:%.*]], 40 ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[COND0]]) [ "deopt"() ] -; CHECK-NEXT: [[COND1:%.*]] = icmp sge i32 [[VAL]], 40 -; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[COND1]]) [ "deopt"() ] +; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: ret i32 0 ; -; Demonstrates a case we do not yet handle (it is legal to fold %cond2 -; to false) %cond0 = icmp slt i32 %val, 40 call void(i1,...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ] %cond1 = icmp sge i32 %val, 40