diff --git a/llvm/include/llvm/Analysis/LoopAnalysisManager.h b/llvm/include/llvm/Analysis/LoopAnalysisManager.h --- a/llvm/include/llvm/Analysis/LoopAnalysisManager.h +++ b/llvm/include/llvm/Analysis/LoopAnalysisManager.h @@ -58,6 +58,7 @@ TargetLibraryInfo &TLI; TargetTransformInfo &TTI; BlockFrequencyInfo *BFI; + BranchProbabilityInfo *BPI; MemorySSA *MSSA; }; diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h --- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -421,10 +421,12 @@ explicit FunctionToLoopPassAdaptor(std::unique_ptr Pass, bool UseMemorySSA = false, bool UseBlockFrequencyInfo = false, + bool UseBranchProbabilityInfo = false, bool LoopNestMode = false) : Pass(std::move(Pass)), LoopCanonicalizationFPM(), UseMemorySSA(UseMemorySSA), UseBlockFrequencyInfo(UseBlockFrequencyInfo), + UseBranchProbabilityInfo(UseBranchProbabilityInfo), LoopNestMode(LoopNestMode) { LoopCanonicalizationFPM.addPass(LoopSimplifyPass()); LoopCanonicalizationFPM.addPass(LCSSAPass()); @@ -446,6 +448,7 @@ bool UseMemorySSA = false; bool UseBlockFrequencyInfo = false; + bool UseBranchProbabilityInfo = false; const bool LoopNestMode; }; @@ -457,13 +460,14 @@ inline std::enable_if_t::value, FunctionToLoopPassAdaptor> createFunctionToLoopPassAdaptor(LoopPassT &&Pass, bool UseMemorySSA = false, - bool UseBlockFrequencyInfo = false) { + bool UseBlockFrequencyInfo = false, + bool UseBranchProbabilityInfo = false) { using PassModelT = detail::PassModel; return FunctionToLoopPassAdaptor( std::make_unique(std::forward(Pass)), UseMemorySSA, - UseBlockFrequencyInfo, false); + UseBlockFrequencyInfo, UseBranchProbabilityInfo, false); } /// If \p Pass is a loop-nest pass, \p Pass will first be wrapped into a @@ -472,7 +476,8 @@ inline std::enable_if_t::value, FunctionToLoopPassAdaptor> createFunctionToLoopPassAdaptor(LoopNestPassT &&Pass, bool UseMemorySSA = false, - bool UseBlockFrequencyInfo = false) { + bool UseBlockFrequencyInfo = false, + bool UseBranchProbabilityInfo = false) { LoopPassManager LPM; LPM.addPass(std::forward(Pass)); using PassModelT = @@ -480,16 +485,17 @@ LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater &>; return FunctionToLoopPassAdaptor(std::make_unique(std::move(LPM)), - UseMemorySSA, UseBlockFrequencyInfo, true); + UseMemorySSA, UseBlockFrequencyInfo, + UseBranchProbabilityInfo, true); } /// If \p Pass is an instance of \c LoopPassManager, the returned adaptor will /// be in loop-nest mode if the pass manager contains only loop-nest passes. template <> inline FunctionToLoopPassAdaptor -createFunctionToLoopPassAdaptor(LoopPassManager &&LPM, - bool UseMemorySSA, - bool UseBlockFrequencyInfo) { +createFunctionToLoopPassAdaptor( + LoopPassManager &&LPM, bool UseMemorySSA, bool UseBlockFrequencyInfo, + bool UseBranchProbabilityInfo) { // Check if LPM contains any loop pass and if it does not, returns an adaptor // in loop-nest mode. using PassModelT = @@ -499,7 +505,7 @@ bool LoopNestMode = (LPM.getNumLoopPasses() == 0); return FunctionToLoopPassAdaptor(std::make_unique(std::move(LPM)), UseMemorySSA, UseBlockFrequencyInfo, - LoopNestMode); + UseBranchProbabilityInfo, LoopNestMode); } /// Pass for printing a loop's contents as textual IR. diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1302,8 +1302,11 @@ bool UseMemorySSA = (Name == "loop-mssa"); bool UseBFI = llvm::any_of( InnerPipeline, [](auto Pipeline) { return Pipeline.Name == "licm"; }); + bool UseBPI = llvm::any_of(InnerPipeline, [](auto Pipeline) { + return Pipeline.Name == "loop-predication"; + }); FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA, - UseBFI)); + UseBFI, UseBPI)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { diff --git a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp --- a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp +++ b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp @@ -1057,8 +1057,8 @@ auto &LAM = AM.getResult(F).getManager(); std::function GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }; diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp --- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp +++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp @@ -721,8 +721,8 @@ auto &LAM = AM.getResult(F).getManager(); bool Changed = eliminateLoadsAcrossLoops( F, LI, DT, BFI, PSI, &SE, &AC, [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }); diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp --- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp @@ -10,6 +10,7 @@ #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" @@ -207,6 +208,10 @@ BlockFrequencyInfo *BFI = UseBlockFrequencyInfo && F.hasProfileData() ? (&AM.getResult(F)) : nullptr; + BranchProbabilityInfo *BPI = + UseBranchProbabilityInfo && F.hasProfileData() + ? (&AM.getResult(F)) + : nullptr; LoopStandardAnalysisResults LAR = {AM.getResult(F), AM.getResult(F), AM.getResult(F), @@ -215,6 +220,7 @@ AM.getResult(F), AM.getResult(F), BFI, + BPI, MSSA}; // Setup the loop analysis manager from its proxy. It is important that @@ -335,6 +341,8 @@ PA.preserve(); if (UseBlockFrequencyInfo && F.hasProfileData()) PA.preserve(); + if (UseBranchProbabilityInfo && F.hasProfileData()) + PA.preserve(); if (UseMemorySSA) PA.preserve(); return PA; diff --git a/llvm/lib/Transforms/Scalar/LoopPredication.cpp b/llvm/lib/Transforms/Scalar/LoopPredication.cpp --- a/llvm/lib/Transforms/Scalar/LoopPredication.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPredication.cpp @@ -367,14 +367,10 @@ LoopStandardAnalysisResults &AR, LPMUpdater &U) { Function *F = L.getHeader()->getParent(); - // For the new PM, we also can't use BranchProbabilityInfo as an analysis - // pass. Function analyses need to be preserved across loop transformations - // but BPI is not preserved, hence a newly built one is needed. - BranchProbabilityInfo BPI(*F, AR.LI, &AR.TLI, &AR.DT, nullptr); std::unique_ptr MSSAU; if (AR.MSSA) MSSAU = std::make_unique(AR.MSSA); - LoopPredication LP(&AR.AA, &AR.DT, &AR.SE, &AR.LI, &BPI, + LoopPredication LP(&AR.AA, &AR.DT, &AR.SE, &AR.LI, AR.BPI, MSSAU ? MSSAU.get() : nullptr); if (!LP.runOnLoop(&L)) return PreservedAnalyses::all(); diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp --- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp +++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp @@ -357,8 +357,8 @@ auto &LAM = AM.getResult(F).getManager(); auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -10557,8 +10557,8 @@ auto &LAM = AM.getResult(F).getManager(); std::function GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }; auto &MAMProxy = AM.getResult(F); diff --git a/llvm/test/Transforms/LoopPredication/preserve-bpi.ll b/llvm/test/Transforms/LoopPredication/preserve-bpi.ll --- a/llvm/test/Transforms/LoopPredication/preserve-bpi.ll +++ b/llvm/test/Transforms/LoopPredication/preserve-bpi.ll @@ -1,24 +1,19 @@ -; RUN: opt -mtriple=x86_64 -passes='require,require,loop-mssa(loop-predication,licm,simple-loop-unswitch,loop-simplifycfg)' -debug-pass-manager -debug-only=branch-prob -S < %s 2>&1 | FileCheck %s +; RUN: opt -mtriple=x86_64 -passes='loop-mssa(loop-predication,licm,simple-loop-unswitch,loop-simplifycfg)' -debug-pass-manager -debug-only=branch-prob -S < %s 2>&1 | FileCheck %s ; REQUIRES: asserts +; This test is to solely check that we do not run BPI every single time loop +; predication is invoked (since BPI is preserved as part of +; LoopStandardAnalysisResults). declare void @llvm.experimental.guard(i1, ...) -; CHECK: Running pass: RequireAnalysisPass<{{.*}}BranchProbabilityAnalysis -; CHECK-NEXT: Running analysis: BranchProbabilityAnalysis on unsigned_loop_0_to_n_ult_check -; CHECK-NEXT: Running analysis: PostDominatorTreeAnalysis on unsigned_loop_0_to_n_ult_check -; CHECK-NEXT: ---- Branch Probability Info : unsigned_loop_0_to_n_ult_check ---- -; CHECK: Running pass: LoopSimplifyPass on unsigned_loop_0_to_n_ult_check ; CHECK: Running pass: LoopPredicationPass on Loop at depth 1 -; CHECK-NEXT: ---- Branch Probability Info : unsigned_loop_0_to_n_ult_check ---- -; CHECK: Running pass: LICMPass on Loop at depth 1 +; CHECK-NEXT: Running pass: LICMPass on Loop at depth 1 ; CHECK-NEXT: Running pass: SimpleLoopUnswitchPass on Loop at depth 1 ; CHECK-NEXT: Running pass: LoopPredicationPass on Loop at depth 1 -; CHECK-NEXT: ---- Branch Probability Info : unsigned_loop_0_to_n_ult_check ---- -; CHECK: Running pass: LICMPass on Loop at depth 1 +; CHECK-NEXT: Running pass: LICMPass on Loop at depth 1 ; CHECK-NEXT: Running pass: SimpleLoopUnswitchPass on Loop at depth 1 ; CHECK-NEXT: Running pass: LoopSimplifyCFGPass on Loop at depth 1 -; CHECK-NEXT: Invalidating analysis: BranchProbabilityAnalysis on unsigned_loop_0_to_n_ult_check define i32 @unsigned_loop_0_to_n_ult_check(i32* %array, i32 %length, i32 %n) {