diff --git a/llvm/include/llvm/Analysis/LoopAnalysisManager.h b/llvm/include/llvm/Analysis/LoopAnalysisManager.h --- a/llvm/include/llvm/Analysis/LoopAnalysisManager.h +++ b/llvm/include/llvm/Analysis/LoopAnalysisManager.h @@ -58,6 +58,7 @@ TargetLibraryInfo &TLI; TargetTransformInfo &TTI; BlockFrequencyInfo *BFI; + BranchProbabilityInfo *BPI; MemorySSA *MSSA; }; diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h --- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -421,10 +421,12 @@ explicit FunctionToLoopPassAdaptor(std::unique_ptr Pass, bool UseMemorySSA = false, bool UseBlockFrequencyInfo = false, + bool UseBranchProbabilityInfo = false, bool LoopNestMode = false) : Pass(std::move(Pass)), LoopCanonicalizationFPM(), UseMemorySSA(UseMemorySSA), UseBlockFrequencyInfo(UseBlockFrequencyInfo), + UseBranchProbabilityInfo(UseBranchProbabilityInfo), LoopNestMode(LoopNestMode) { LoopCanonicalizationFPM.addPass(LoopSimplifyPass()); LoopCanonicalizationFPM.addPass(LCSSAPass()); @@ -446,6 +448,7 @@ bool UseMemorySSA = false; bool UseBlockFrequencyInfo = false; + bool UseBranchProbabilityInfo = false; const bool LoopNestMode; }; @@ -457,13 +460,14 @@ inline std::enable_if_t::value, FunctionToLoopPassAdaptor> createFunctionToLoopPassAdaptor(LoopPassT &&Pass, bool UseMemorySSA = false, - bool UseBlockFrequencyInfo = false) { + bool UseBlockFrequencyInfo = false, + bool UseBranchProbabilityInfo = false) { using PassModelT = detail::PassModel; return FunctionToLoopPassAdaptor( std::make_unique(std::forward(Pass)), UseMemorySSA, - UseBlockFrequencyInfo, false); + UseBlockFrequencyInfo, UseBranchProbabilityInfo, false); } /// If \p Pass is a loop-nest pass, \p Pass will first be wrapped into a @@ -472,7 +476,8 @@ inline std::enable_if_t::value, FunctionToLoopPassAdaptor> createFunctionToLoopPassAdaptor(LoopNestPassT &&Pass, bool UseMemorySSA = false, - bool UseBlockFrequencyInfo = false) { + bool UseBlockFrequencyInfo = false, + bool UseBranchProbabilityInfo = false) { LoopPassManager LPM; LPM.addPass(std::forward(Pass)); using PassModelT = @@ -480,16 +485,17 @@ LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater &>; return FunctionToLoopPassAdaptor(std::make_unique(std::move(LPM)), - UseMemorySSA, UseBlockFrequencyInfo, true); + UseMemorySSA, UseBlockFrequencyInfo, + UseBranchProbabilityInfo, true); } /// If \p Pass is an instance of \c LoopPassManager, the returned adaptor will /// be in loop-nest mode if the pass manager contains only loop-nest passes. template <> inline FunctionToLoopPassAdaptor -createFunctionToLoopPassAdaptor(LoopPassManager &&LPM, - bool UseMemorySSA, - bool UseBlockFrequencyInfo) { +createFunctionToLoopPassAdaptor( + LoopPassManager &&LPM, bool UseMemorySSA, bool UseBlockFrequencyInfo, + bool UseBranchProbabilityInfo) { // Check if LPM contains any loop pass and if it does not, returns an adaptor // in loop-nest mode. using PassModelT = @@ -499,7 +505,7 @@ bool LoopNestMode = (LPM.getNumLoopPasses() == 0); return FunctionToLoopPassAdaptor(std::make_unique(std::move(LPM)), UseMemorySSA, UseBlockFrequencyInfo, - LoopNestMode); + UseBranchProbabilityInfo, LoopNestMode); } /// Pass for printing a loop's contents as textual IR. diff --git a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp --- a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp +++ b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp @@ -1057,8 +1057,8 @@ auto &LAM = AM.getResult(F).getManager(); std::function GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }; diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp --- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp +++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp @@ -721,8 +721,8 @@ auto &LAM = AM.getResult(F).getManager(); bool Changed = eliminateLoadsAcrossLoops( F, LI, DT, BFI, PSI, &SE, &AC, [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }); diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp --- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp @@ -10,6 +10,7 @@ #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" @@ -207,6 +208,10 @@ BlockFrequencyInfo *BFI = UseBlockFrequencyInfo && F.hasProfileData() ? (&AM.getResult(F)) : nullptr; + BranchProbabilityInfo *BPI = + UseBranchProbabilityInfo && F.hasProfileData() + ? (&AM.getResult(F)) + : nullptr; LoopStandardAnalysisResults LAR = {AM.getResult(F), AM.getResult(F), AM.getResult(F), @@ -215,6 +220,7 @@ AM.getResult(F), AM.getResult(F), BFI, + BPI, MSSA}; // Setup the loop analysis manager from its proxy. It is important that @@ -335,6 +341,8 @@ PA.preserve(); if (UseBlockFrequencyInfo && F.hasProfileData()) PA.preserve(); + if (UseBranchProbabilityInfo && F.hasProfileData()) + PA.preserve(); if (UseMemorySSA) PA.preserve(); return PA; diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp --- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp +++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp @@ -357,8 +357,8 @@ auto &LAM = AM.getResult(F).getManager(); auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -10557,8 +10557,8 @@ auto &LAM = AM.getResult(F).getManager(); std::function GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, - TLI, TTI, nullptr, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr, nullptr}; return LAM.getResult(L, AR); }; auto &MAMProxy = AM.getResult(F);