Index: clang/test/CodeGen/thinlto-distributed-newpm.ll =================================================================== --- clang/test/CodeGen/thinlto-distributed-newpm.ll +++ clang/test/CodeGen/thinlto-distributed-newpm.ll @@ -112,6 +112,9 @@ ; CHECK-O: Running analysis: LoopAnalysis on main ; CHECK-O: Running pass: LCSSAPass on main ; CHECK-O: Finished {{.*}}Function pass manager run. +; CHECK-O: Running analysis: BlockFrequencyAnalysis on main +; CHECK-O: Running analysis: BranchProbabilityAnalysis on main +; CHECK-O: Running analysis: PostDominatorTreeAnalysis on main ; CHECK-O: Running pass: SimplifyCFGPass on main ; CHECK-O: Running pass: InstCombinePass on main ; CHECK-O: Starting {{.*}}Function pass manager run. @@ -136,7 +139,6 @@ ; CHECK-O: Running pass: LCSSAPass on main ; CHECK-O: Finished {{.*}}Function pass manager run. ; CHECK-O: Running pass: ADCEPass on main -; CHECK-O: Running analysis: PostDominatorTreeAnalysis on main ; CHECK-O: Running pass: SimplifyCFGPass on main ; CHECK-O: Running pass: InstCombinePass on main ; CHECK-O: Finished {{.*}}Function pass manager run. @@ -146,10 +148,10 @@ ; CHECK-O: Invalidating analysis: AAManager on main ; CHECK-O: Invalidating analysis: MemorySSAAnalysis on main ; CHECK-O: Invalidating analysis: LoopAnalysis on main +; CHECK-O: Invalidating analysis: PostDominatorTreeAnalysis on main ; CHECK-O: Invalidating analysis: PhiValuesAnalysis on main ; CHECK-O: Invalidating analysis: MemoryDependenceAnalysis on main ; CHECK-O: Invalidating analysis: DemandedBitsAnalysis on main -; CHECK-O: Invalidating analysis: PostDominatorTreeAnalysis on main ; CHECK-O: Invalidating analysis: CallGraphAnalysis ; CHECK-O: Finished {{.*}}Module pass manager run. ; CHECK-O: Starting {{.*}}Module pass manager run. Index: llvm/include/llvm/Analysis/BlockFrequencyInfo.h =================================================================== --- llvm/include/llvm/Analysis/BlockFrequencyInfo.h +++ llvm/include/llvm/Analysis/BlockFrequencyInfo.h @@ -15,10 +15,12 @@ #include "llvm/ADT/Optional.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" #include "llvm/Support/BlockFrequency.h" #include #include +#include namespace llvm { @@ -37,7 +39,20 @@ class BlockFrequencyInfo { using ImplType = BlockFrequencyInfoImpl; - std::unique_ptr BFI; + class BFICallbackVH final : public CallbackVH { + std::weak_ptr BFI; + const BasicBlock *BB; + + public: + BFICallbackVH(std::shared_ptr BFI, const BasicBlock *BB); + virtual ~BFICallbackVH() = default; + + virtual void deleted() override; + virtual void allUsesReplacedWith(Value *V) override; + }; + std::vector BlockWatchers; + + std::shared_ptr BFI; public: BlockFrequencyInfo(); Index: llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h =================================================================== --- llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -847,10 +847,13 @@ /// (Running this until fixed point would "solve" the geometric /// series by simulation.) template class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase { +public: + using BlockT = typename bfi_detail::TypeMap::BlockT; + +private: // This is part of a workaround for a GCC 4.7 crash on lambdas. friend struct bfi_detail::BlockEdgesAdder; - using BlockT = typename bfi_detail::TypeMap::BlockT; using BlockKeyT = typename bfi_detail::TypeMap::BlockKeyT; using FunctionT = typename bfi_detail::TypeMap::FunctionT; using BranchProbabilityInfoT = @@ -1015,6 +1018,13 @@ const BranchProbabilityInfoT &getBPI() const { return *BPI; } + /// Removes \p BB from internal datastructures. + /// \return false if the block did not exist. + bool removeBlock(const BlockT *BB); + + /// \return all blocks known to this datastructure. + DenseSet getRegisteredBlocks() const; + /// Print the frequencies for the current function. /// /// Prints the frequencies for the blocks in the current function. @@ -1121,6 +1131,20 @@ } } +template +bool BlockFrequencyInfoImpl::removeBlock(const BlockT *BB) { + return Nodes.erase(BB); +} + +template +DenseSet::BlockT *> +BlockFrequencyInfoImpl::getRegisteredBlocks() const { + DenseSet Ret; + for (auto &BN : Nodes) + Ret.insert(BN.first); + return Ret; +} + template void BlockFrequencyInfoImpl::initializeRPOT() { const BlockT *Entry = &F->front(); RPOT.reserve(F->size()); Index: llvm/include/llvm/Analysis/LoopAnalysisManager.h =================================================================== --- llvm/include/llvm/Analysis/LoopAnalysisManager.h +++ llvm/include/llvm/Analysis/LoopAnalysisManager.h @@ -57,6 +57,7 @@ ScalarEvolution &SE; TargetLibraryInfo &TLI; TargetTransformInfo &TTI; + BlockFrequencyInfo *BFI; MemorySSA *MSSA; }; Index: llvm/include/llvm/Transforms/Scalar/LoopPassManager.h =================================================================== --- llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -41,6 +41,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BasicAliasAnalysis.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" @@ -233,9 +234,11 @@ : public PassInfoMixin> { public: explicit FunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false, + bool UseBlockFrequencyInfo = false, bool DebugLogging = false) : Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging), - UseMemorySSA(UseMemorySSA) { + UseMemorySSA(UseMemorySSA), + UseBlockFrequencyInfo(UseBlockFrequencyInfo) { LoopCanonicalizationFPM.addPass(LoopSimplifyPass()); LoopCanonicalizationFPM.addPass(LCSSAPass()); } @@ -267,6 +270,9 @@ MemorySSA *MSSA = UseMemorySSA ? (&AM.getResult(F).getMSSA()) : nullptr; + BlockFrequencyInfo *BFI = UseBlockFrequencyInfo + ? (&AM.getResult(F)) + : nullptr; LoopStandardAnalysisResults LAR = {AM.getResult(F), AM.getResult(F), AM.getResult(F), @@ -274,6 +280,7 @@ AM.getResult(F), AM.getResult(F), AM.getResult(F), + BFI, MSSA}; // Setup the loop analysis manager from its proxy. It is important that @@ -370,6 +377,8 @@ PA.preserve(); PA.preserve(); PA.preserve(); + if (UseBlockFrequencyInfo) + PA.preserve(); if (UseMemorySSA) PA.preserve(); // FIXME: What we really want to do here is preserve an AA category, but @@ -389,6 +398,7 @@ FunctionPassManager LoopCanonicalizationFPM; bool UseMemorySSA = false; + bool UseBlockFrequencyInfo = false; }; /// A function to deduce a loop pass type and wrap it in the templated @@ -396,9 +406,10 @@ template FunctionToLoopPassAdaptor createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false, + bool UseBlockFrequencyInfo = false, bool DebugLogging = false) { - return FunctionToLoopPassAdaptor(std::move(Pass), UseMemorySSA, - DebugLogging); + return FunctionToLoopPassAdaptor( + std::move(Pass), UseMemorySSA, UseBlockFrequencyInfo, DebugLogging); } /// Pass for printing a loop's contents as textual IR. Index: llvm/lib/Analysis/BlockFrequencyInfo.cpp =================================================================== --- llvm/lib/Analysis/BlockFrequencyInfo.cpp +++ llvm/lib/Analysis/BlockFrequencyInfo.cpp @@ -181,12 +181,36 @@ PAC.preservedSet()); } +BlockFrequencyInfo::BFICallbackVH::BFICallbackVH( + std::shared_ptr> BFI, + const BasicBlock *BB) + : CallbackVH(const_cast(BB)), BFI(BFI), BB(BB) {} + +void BlockFrequencyInfo::BFICallbackVH::deleted() { + if (auto BFII = BFI.lock()) + BFII->removeBlock(BB); + CallbackVH::deleted(); +} + +void BlockFrequencyInfo::BFICallbackVH::allUsesReplacedWith(Value *V) { + if (auto BFII = BFI.lock()) + BFII->removeBlock(BB); + CallbackVH::allUsesReplacedWith(V); +} + void BlockFrequencyInfo::calculate(const Function &F, const BranchProbabilityInfo &BPI, const LoopInfo &LI) { if (!BFI) BFI.reset(new ImplType); BFI->calculate(F, BPI, LI); + + BlockWatchers.clear(); + auto RegisteredBlocks = BFI->getRegisteredBlocks(); + BlockWatchers.reserve(RegisteredBlocks.size()); + for (const auto *BB : RegisteredBlocks) + BlockWatchers.emplace_back(BFICallbackVH(BFI, BB)); + if (ViewBlockFreqPropagationDAG != GVDT_None && (ViewBlockFreqFuncName.empty() || F.getName().equals(ViewBlockFreqFuncName))) { Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -512,13 +512,15 @@ FPM.addPass( RequireAnalysisPass()); FPM.addPass(createFunctionToLoopPassAdaptor( - std::move(LPM1), EnableMSSALoopDependency, DebugLogging)); + std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, + DebugLogging)); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA. // *All* loop passes must preserve it, in order to be able to use it. FPM.addPass(createFunctionToLoopPassAdaptor( - std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging)); + std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false, + DebugLogging)); // Delete small array after loop unroll. FPM.addPass(SROA()); @@ -669,14 +671,16 @@ FPM.addPass( RequireAnalysisPass()); FPM.addPass(createFunctionToLoopPassAdaptor( - std::move(LPM1), EnableMSSALoopDependency, DebugLogging)); + std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, + DebugLogging)); FPM.addPass(SimplifyCFGPass()); FPM.addPass(InstCombinePass()); // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass, // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA. // *All* loop passes must preserve it, in order to be able to use it. FPM.addPass(createFunctionToLoopPassAdaptor( - std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging)); + std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false, + DebugLogging)); // Delete small array after loop unroll. FPM.addPass(SROA()); @@ -713,7 +717,7 @@ FPM.addPass(DSEPass()); FPM.addPass(createFunctionToLoopPassAdaptor( LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap), - EnableMSSALoopDependency, DebugLogging)); + EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging)); if (PTO.Coroutines) FPM.addPass(CoroElidePass()); @@ -791,7 +795,8 @@ FunctionPassManager FPM; FPM.addPass(createFunctionToLoopPassAdaptor( - LoopRotatePass(), EnableMSSALoopDependency, DebugLogging)); + LoopRotatePass(), EnableMSSALoopDependency, + /*UseBlockFrequencyInfo=*/false, DebugLogging)); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); // Add the profile lowering pass. @@ -1115,7 +1120,8 @@ // First rotate loops that may have been un-rotated by prior passes. OptimizePM.addPass(createFunctionToLoopPassAdaptor( - LoopRotatePass(), EnableMSSALoopDependency, DebugLogging)); + LoopRotatePass(), EnableMSSALoopDependency, + /*UseBlockFrequencyInfo=*/false, DebugLogging)); // Distribute loops to allow partial vectorization. I.e. isolate dependences // into separate loop that would otherwise inhibit vectorization. This is @@ -1179,7 +1185,7 @@ OptimizePM.addPass(RequireAnalysisPass()); OptimizePM.addPass(createFunctionToLoopPassAdaptor( LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap), - EnableMSSALoopDependency, DebugLogging)); + EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging)); // Now that we've vectorized and unrolled loops, we may have more refined // alignment information, try to re-derive it here. @@ -2244,8 +2250,9 @@ } #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ - MPM.addPass(createModuleToFunctionPassAdaptor( \ - createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging))); \ + MPM.addPass( \ + createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ + CREATE_PASS, false, false, DebugLogging))); \ return Error::success(); \ } #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ @@ -2255,7 +2262,7 @@ return Params.takeError(); \ MPM.addPass( \ createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ - CREATE_PASS(Params.get()), false, DebugLogging))); \ + CREATE_PASS(Params.get()), false, false, DebugLogging))); \ return Error::success(); \ } #include "PassRegistry.def" @@ -2356,8 +2363,9 @@ } #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ - CGPM.addPass(createCGSCCToFunctionPassAdaptor( \ - createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging))); \ + CGPM.addPass( \ + createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ + CREATE_PASS, false, false, DebugLogging))); \ return Error::success(); \ } #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ @@ -2367,7 +2375,7 @@ return Params.takeError(); \ CGPM.addPass( \ createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \ - CREATE_PASS(Params.get()), false, DebugLogging))); \ + CREATE_PASS(Params.get()), false, false, DebugLogging))); \ return Error::success(); \ } #include "PassRegistry.def" @@ -2404,8 +2412,9 @@ return Err; // Add the nested pass manager with the appropriate adaptor. bool UseMemorySSA = (Name == "loop-mssa"); - FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA, - DebugLogging)); + FPM.addPass(createFunctionToLoopPassAdaptor( + std::move(LPM), UseMemorySSA, /*UseBlockFrequencyInfo=*/false, + DebugLogging)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { @@ -2459,8 +2468,8 @@ // The risk is that it may become obsolete if we're not careful. #define LOOP_PASS(NAME, CREATE_PASS) \ if (Name == NAME) { \ - FPM.addPass( \ - createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging)); \ + FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false, \ + DebugLogging)); \ return Error::success(); \ } #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ @@ -2469,7 +2478,7 @@ if (!Params) \ return Params.takeError(); \ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \ - false, DebugLogging)); \ + false, false, DebugLogging)); \ return Error::success(); \ } #include "PassRegistry.def" Index: llvm/lib/Transforms/Scalar/LICM.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LICM.cpp +++ llvm/lib/Transforms/Scalar/LICM.cpp @@ -39,6 +39,7 @@ #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/GuardUtils.h" +#include "llvm/Analysis/LazyBlockFrequencyInfo.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopIterator.h" @@ -171,8 +172,8 @@ namespace { struct LoopInvariantCodeMotion { bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT, - TargetLibraryInfo *TLI, TargetTransformInfo *TTI, - ScalarEvolution *SE, MemorySSA *MSSA, + BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, + TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE); LoopInvariantCodeMotion(unsigned LicmMssaOptCap, @@ -209,13 +210,14 @@ ? (&getAnalysis().getMSSA()) : nullptr; // For the old PM, we can't use OptimizationRemarkEmitter as an analysis - // pass. Function analyses need to be preserved across loop transformations + // pass. Function analyses need to be preserved across loop transformations // but ORE cannot be preserved (see comment before the pass definition). OptimizationRemarkEmitter ORE(L->getHeader()->getParent()); return LICM.runOnLoop(L, &getAnalysis().getAAResults(), &getAnalysis().getLoopInfo(), &getAnalysis().getDomTree(), + &getAnalysis().getBFI(), &getAnalysis().getTLI( *L->getHeader()->getParent()), &getAnalysis().getTTI( @@ -236,6 +238,7 @@ } AU.addRequired(); getLoopAnalysisUsage(AU); + LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); } private: @@ -251,8 +254,8 @@ OptimizationRemarkEmitter ORE(L.getHeader()->getParent()); LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap); - if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, &AR.TLI, &AR.TTI, &AR.SE, - AR.MSSA, &ORE)) + if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI, + &AR.SE, AR.MSSA, &ORE)) return PreservedAnalyses::all(); auto PA = getLoopPassPreservedAnalyses(); @@ -272,6 +275,7 @@ INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass) +INITIALIZE_PASS_DEPENDENCY(LazyBFIPass) INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false, false) @@ -286,8 +290,8 @@ /// times on one loop. bool LoopInvariantCodeMotion::runOnLoop( Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT, - TargetLibraryInfo *TLI, TargetTransformInfo *TTI, ScalarEvolution *SE, - MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) { + BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI, + ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) { bool Changed = false; assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form."); Index: llvm/lib/Transforms/Scalar/LoopDistribute.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopDistribute.cpp +++ llvm/lib/Transforms/Scalar/LoopDistribute.cpp @@ -1058,7 +1058,8 @@ auto &LAM = AM.getResult(F).getManager(); std::function GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, nullptr}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, nullptr}; return LAM.getResult(L, AR); }; Index: llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp +++ llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp @@ -705,7 +705,8 @@ auto &LAM = AM.getResult(F).getManager(); bool Changed = eliminateLoadsAcrossLoops( F, LI, DT, BFI, PSI, [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, MSSA}; return LAM.getResult(L, AR); }); Index: llvm/lib/Transforms/Scalar/LoopUnswitch.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopUnswitch.cpp +++ llvm/lib/Transforms/Scalar/LoopUnswitch.cpp @@ -32,6 +32,7 @@ #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CodeMetrics.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/LazyBlockFrequencyInfo.h" #include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopIterator.h" @@ -217,6 +218,10 @@ /// loop preheaders be inserted into the CFG. /// void getAnalysisUsage(AnalysisUsage &AU) const override { + // Lazy BFI and BPI are marked as preserved here so Loop Unswitching + // can remain part of the same loop pass as LICM + AU.addPreserved(); + AU.addPreserved(); AU.addRequired(); AU.addRequired(); if (EnableMSSALoopDependency) { Index: llvm/lib/Transforms/Utils/LoopVersioning.cpp =================================================================== --- llvm/lib/Transforms/Utils/LoopVersioning.cpp +++ llvm/lib/Transforms/Utils/LoopVersioning.cpp @@ -357,7 +357,8 @@ auto &LAM = AM.getResult(F).getManager(); auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, MSSA}; return LAM.getResult(L, AR); }; Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8558,7 +8558,8 @@ auto &LAM = AM.getResult(F).getManager(); std::function GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA}; + LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, + TLI, TTI, nullptr, MSSA}; return LAM.getResult(L, AR); }; auto &MAMProxy = AM.getResult(F); Index: llvm/test/Other/new-pm-defaults.ll =================================================================== --- llvm/test/Other/new-pm-defaults.ll +++ llvm/test/Other/new-pm-defaults.ll @@ -164,6 +164,9 @@ ; CHECK-O-NEXT: Running analysis: LoopAnalysis ; CHECK-O-NEXT: Running pass: LCSSAPass ; CHECK-O-NEXT: Finished llvm::Function pass manager run. +; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis +; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis +; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Starting Loop pass manager run. @@ -212,7 +215,6 @@ ; CHECK-O23SZ-NEXT: Running pass: LICMPass ; CHECK-EP-SCALAR-LATE-NEXT: Running pass: NoOpFunctionPass ; CHECK-O-NEXT: Running pass: ADCEPass -; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass ; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass @@ -240,8 +242,6 @@ ; CHECK-O-NEXT: Running pass: LoopDistributePass ; CHECK-O-NEXT: Running pass: InjectTLIMappings ; CHECK-O-NEXT: Running pass: LoopVectorizePass -; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis -; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis ; CHECK-O-NEXT: Running pass: LoopLoadEliminationPass ; CHECK-O-NEXT: Running analysis: LoopAccessAnalysis ; CHECK-O-NEXT: Running pass: InstCombinePass Index: llvm/test/Other/new-pm-thinlto-defaults.ll =================================================================== --- llvm/test/Other/new-pm-thinlto-defaults.ll +++ llvm/test/Other/new-pm-thinlto-defaults.ll @@ -128,6 +128,9 @@ ; CHECK-O-NEXT: Running analysis: LoopAnalysis ; CHECK-O-NEXT: Running pass: LCSSAPass ; CHECK-O-NEXT: Finished llvm::Function pass manager run +; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis +; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis +; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Starting Loop pass manager run. @@ -184,7 +187,6 @@ ; CHECK-O23SZ-NEXT: Finished llvm::Function pass manager run ; CHECK-O23SZ-NEXT: Running pass: LICMPass on Loop at depth 1 containing: %loop ; CHECK-O-NEXT: Running pass: ADCEPass -; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: InstCombinePass ; CHECK-O-NEXT: Finished llvm::Function pass manager run. @@ -210,8 +212,6 @@ ; CHECK-POSTLINK-O-NEXT: Running pass: LoopDistributePass ; CHECK-POSTLINK-O-NEXT: Running pass: InjectTLIMappings ; CHECK-POSTLINK-O-NEXT: Running pass: LoopVectorizePass -; CHECK-POSTLINK-O-NEXT: Running analysis: BlockFrequencyAnalysis -; CHECK-POSTLINK-O-NEXT: Running analysis: BranchProbabilityAnalysis ; CHECK-POSTLINK-O-NEXT: Running pass: LoopLoadEliminationPass ; CHECK-POSTLINK-O-NEXT: Running analysis: LoopAccessAnalysis ; CHECK-POSTLINK-O-NEXT: Running pass: InstCombinePass Index: llvm/test/Other/opt-O2-pipeline.ll =================================================================== --- llvm/test/Other/opt-O2-pipeline.ll +++ llvm/test/Other/opt-O2-pipeline.ll @@ -111,6 +111,8 @@ ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Rotate Loops ; CHECK-NEXT: Memory SSA +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Unswitch loops @@ -168,6 +170,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Post-Dominator Tree Construction @@ -270,6 +274,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Lazy Branch Probability Analysis Index: llvm/test/Other/opt-O3-pipeline-enable-matrix.ll =================================================================== --- llvm/test/Other/opt-O3-pipeline-enable-matrix.ll +++ llvm/test/Other/opt-O3-pipeline-enable-matrix.ll @@ -116,6 +116,8 @@ ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Rotate Loops ; CHECK-NEXT: Memory SSA +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Unswitch loops @@ -173,6 +175,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Post-Dominator Tree Construction @@ -282,6 +286,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Lazy Branch Probability Analysis Index: llvm/test/Other/opt-O3-pipeline.ll =================================================================== --- llvm/test/Other/opt-O3-pipeline.ll +++ llvm/test/Other/opt-O3-pipeline.ll @@ -116,6 +116,8 @@ ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Rotate Loops ; CHECK-NEXT: Memory SSA +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Unswitch loops @@ -173,6 +175,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Post-Dominator Tree Construction @@ -275,6 +279,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Lazy Branch Probability Analysis Index: llvm/test/Other/opt-Os-pipeline.ll =================================================================== --- llvm/test/Other/opt-Os-pipeline.ll +++ llvm/test/Other/opt-Os-pipeline.ll @@ -97,6 +97,8 @@ ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Rotate Loops ; CHECK-NEXT: Memory SSA +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Unswitch loops @@ -154,6 +156,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Post-Dominator Tree Construction @@ -256,6 +260,8 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Loop Invariant Code Motion ; CHECK-NEXT: Lazy Branch Probability Analysis Index: llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp =================================================================== --- llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp +++ llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp @@ -9,7 +9,10 @@ #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/MemorySSA.h" +#include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -294,6 +297,9 @@ // those. FAM.registerPass([&] { return AAManager(); }); FAM.registerPass([&] { return AssumptionAnalysis(); }); + FAM.registerPass([&] { return BlockFrequencyAnalysis(); }); + FAM.registerPass([&] { return BranchProbabilityAnalysis(); }); + FAM.registerPass([&] { return PostDominatorTreeAnalysis(); }); FAM.registerPass([&] { return MemorySSAAnalysis(); }); FAM.registerPass([&] { return ScalarEvolutionAnalysis(); }); FAM.registerPass([&] { return TargetLibraryAnalysis(); });