Index: llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp =================================================================== --- llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp +++ llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp @@ -52,6 +52,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" @@ -115,6 +116,9 @@ static cl::opt SkipProfitabilityChecks("irce-skip-profitability-checks", cl::Hidden, cl::init(false)); +static cl::opt MinEstimatedIterations("irce-min-estimated-iters", + cl::Hidden, cl::init(3)); + static cl::opt AllowUnsignedLatchCondition("irce-allow-unsigned-latch", cl::Hidden, cl::init(true)); @@ -235,11 +239,15 @@ DominatorTree &DT; LoopInfo &LI; + using GetBFIFunc = + llvm::Optional >; + GetBFIFunc GetBFI; + public: InductiveRangeCheckElimination(ScalarEvolution &SE, BranchProbabilityInfo *BPI, DominatorTree &DT, - LoopInfo &LI) - : SE(SE), BPI(BPI), DT(DT), LI(LI) {} + LoopInfo &LI, GetBFIFunc GetBFI = None) + : SE(SE), BPI(BPI), DT(DT), LI(LI), GetBFI(GetBFI) {} bool run(Loop *L, function_ref LPMAddNewLoop); }; @@ -1772,15 +1780,24 @@ auto &BPI = AM.getResult(F); LoopInfo &LI = AM.getResult(F); - InductiveRangeCheckElimination IRCE(SE, &BPI, DT, LI); + // Get BFI analysis result on demand. Please note that modification of + // CFG invalidates this analysis and we should handle it. + auto getBFI = [&F, &AM ]()->BlockFrequencyInfo & { + return AM.getResult(F); + }; + InductiveRangeCheckElimination IRCE(SE, &BPI, DT, LI, { getBFI }); bool Changed = false; - + bool CFGChanged = false; for (const auto &L : LI) { - Changed |= simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, - /*PreserveLCSSA=*/false); + CFGChanged |= simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, + /*PreserveLCSSA=*/false); Changed |= formLCSSARecursively(*L, DT, &LI, &SE); } + Changed |= CFGChanged; + + if (CFGChanged && !SkipProfitabilityChecks) + AM.invalidate(F); SmallPriorityWorklist Worklist; appendLoopsToWorklist(LI, Worklist); @@ -1791,7 +1808,11 @@ while (!Worklist.empty()) { Loop *L = Worklist.pop_back_val(); - Changed |= IRCE.run(L, LPMAddNewLoop); + if (IRCE.run(L, LPMAddNewLoop)) { + Changed = true; + if (!SkipProfitabilityChecks) + AM.invalidate(F); + } } if (!Changed) @@ -1878,6 +1899,19 @@ return false; } LoopStructure LS = MaybeLoopStructure.getValue(); + // Profitability check. + if (!SkipProfitabilityChecks && GetBFI.hasValue()) { + BlockFrequencyInfo &BFI = (*GetBFI)(); + uint64_t hFreq = BFI.getBlockFreq(LS.Header).getFrequency(); + uint64_t phFreq = BFI.getBlockFreq(Preheader).getFrequency(); + if (phFreq != 0 && hFreq != 0 && + (hFreq / phFreq < MinEstimatedIterations)) { + LLVM_DEBUG(dbgs() << "irce: could not prove profit: " + << "freq estimated number iterations is " + << (hFreq / phFreq) << "\n";); + return false; + } + } const SCEVAddRecExpr *IndVar = cast(SE.getMinusSCEV(SE.getSCEV(LS.IndVarBase), SE.getSCEV(LS.IndVarStep))); Index: llvm/test/Transforms/IRCE/low-iterations.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/IRCE/low-iterations.ll @@ -0,0 +1,42 @@ +; RUN: opt -verify-loop-info -irce-print-changed-loops -passes=irce < %s 2>&1 | FileCheck %s +; + +; CHECK-NOT: constrained Loop + +define i32 @multiple_access_no_preloop( + i32* %arr_a, i32* %a_len_ptr, i32* %arr_b, i32* %b_len_ptr, i32 %n) { + + entry: + %len.a = load i32, i32* %a_len_ptr, !range !0 + %first.itr.check = icmp sgt i32 %n, 0 + br i1 %first.itr.check, label %loop, label %exit, !prof !1 + + loop: + %idx = phi i32 [ 0, %entry ] , [ %idx.next, %backedge ] + %idx.next = add i32 %idx, 1 + %abc.a = icmp slt i32 %idx, %len.a + br i1 %abc.a, label %in.bounds.a, label %exit, !prof !2 + + in.bounds.a: + %addr.a = getelementptr i32, i32* %arr_a, i32 %idx + %val = load i32, i32* %addr.a + %cond = icmp ne i32 %val, 0 +; Most probable exit from a loop. + br i1 %cond, label %found, label %backedge, !prof !3 + + backedge: + %next = icmp slt i32 %idx.next, %n + br i1 %next, label %loop, label %exit, !prof !4 + + found: + ret i32 %val + + exit: + ret i32 0 +} + +!0 = !{i32 0, i32 2147483647} +!1 = !{!"branch_weights", i32 1024, i32 1} +!2 = !{!"branch_weights", i32 512, i32 1} +!3 = !{!"branch_weights", i32 1, i32 2} +!4 = !{!"branch_weights", i32 512, i32 1}