Index: include/llvm/Passes/PassBuilder.h =================================================================== --- include/llvm/Passes/PassBuilder.h +++ include/llvm/Passes/PassBuilder.h @@ -71,6 +71,25 @@ std::vector InnerPipeline; }; + /// \brief LLVM optimization phase. + /// + /// This enumerates the LLVM high0levle optimization phases. + enum OptimizationPhase { + /// Regular optimization phase, which generated final assembly/object file. + Regular, + + /// Optimization phase invoked by ThinLTO pre-link phase. + /// For PGO, it invokes profile instrumentation/use at this phase. + /// For SamplePGO, it invokes the first profile annotation at this phase. + /// ICP should not be invoked in this phase. + PreLinkThinLTO, + + /// Optimization phase invoked by ThinLTO post-link phase. + /// For PGO, it invokes ICP in this phase. + /// For SamplePGO, it invokes the 2nd profile annotation and then ICP. + PostLinkThinLTO + }; + /// \brief LLVM-provided high-level optimization levels. /// /// This enumerates the LLVM-provided high-level optimization levels. Each @@ -214,13 +233,11 @@ /// require some transformations for semantic reasons, they should explicitly /// build them. /// - /// \p PrepareForThinLTO indicates whether this is invoked in - /// PrepareForThinLTO phase. Special handling is needed for sample PGO to - /// ensure profile accurate in the backend profile annotation phase. + /// \p Phase indicates the current optimization phase. FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging = false, - bool PrepareForThinLTO = false); + OptimizationPhase Phase, + bool DebugLogging = false); /// Construct the core LLVM module canonicalization and simplification /// pipeline. @@ -236,13 +253,11 @@ /// require some transformations for semantic reasons, they should explicitly /// build them. /// - /// \p PrepareForThinLTO indicates whether this is invoked in - /// PrepareForThinLTO phase. Special handling is needed for sample PGO to - /// ensure profile accurate in the backend profile annotation phase. + /// \p Phase indicates the current optimization phase. ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging = false, - bool PrepareForThinLTO = false); + OptimizationPhase Phase, + bool DebugLogging = false); /// Construct the core LLVM module optimization pipeline. /// Index: lib/Passes/PassBuilder.cpp =================================================================== --- lib/Passes/PassBuilder.cpp +++ lib/Passes/PassBuilder.cpp @@ -325,8 +325,8 @@ FunctionPassManager PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging, - bool PrepareForThinLTO) { + OptimizationPhase Phase, + bool DebugLogging) { assert(Level != O0 && "Must request optimizations!"); FunctionPassManager FPM(DebugLogging); @@ -389,10 +389,10 @@ C(LPM2, Level); LPM2.addPass(LoopDeletionPass()); - // Do not enable unrolling in PrepareForThinLTO phase during sample PGO + // Do not enable unrolling in PreLinkThinLTO phase during sample PGO // because it changes IR to makes profile annotation in back compile // inaccurate. - if (!PrepareForThinLTO || !PGOOpt || PGOOpt->SampleProfileFile.empty()) + if (Phase != PreLinkThinLTO || !PGOOpt || PGOOpt->SampleProfileFile.empty()) LPM2.addPass(LoopUnrollPass::createFull(Level)); for (auto &C : LoopOptimizerEndEPCallbacks) @@ -524,8 +524,8 @@ ModulePassManager PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, - bool DebugLogging, - bool PrepareForThinLTO) { + OptimizationPhase Phase, + bool DebugLogging) { ModulePassManager MPM(DebugLogging); // Do basic inference of function attributes from known properties of system @@ -581,9 +581,9 @@ MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile)); // Indirect call promotion that promotes intra-module targes only. - // Do not enable it in PrepareForThinLTO phase during sample PGO because + // Do not enable it in PreLinkThinLTO phase during sample PGO because // it changes IR to makes profile annotation in back compile inaccurate. - if ((!PrepareForThinLTO && !PGOOpt->SampleProfileFile.empty()) + if ((Phase != PreLinkThinLTO && !PGOOpt->SampleProfileFile.empty()) || !PGOOpt->ProfileUseFile.empty()) MPM.addPass(PGOIndirectCallPromotion( false, PGOOpt && !PGOOpt->SampleProfileFile.empty())); @@ -611,10 +611,10 @@ // Run the inliner first. The theory is that we are walking bottom-up and so // the callees have already been fully optimized, and we want to inline them // into the callers so that our optimizations can reflect that. - // For PrepareForThinLTO pass, we disable hot-caller heuristic for sample PGO + // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO // because it makes profile annotation in the backend inaccurate. InlineParams IP = getInlineParamsFromOptLevel(Level); - if (PrepareForThinLTO && PGOOpt && !PGOOpt->SampleProfileFile.empty()) + if (Phase == PreLinkThinLTO && PGOOpt && !PGOOpt->SampleProfileFile.empty()) IP.HotCallSiteThreshold = 0; MainCGPipeline.addPass(InlinerPass(IP)); @@ -629,8 +629,7 @@ // Lastly, add the core function simplification pipeline nested inside the // CGSCC walk. MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( - buildFunctionSimplificationPipeline(Level, DebugLogging, - PrepareForThinLTO))); + buildFunctionSimplificationPipeline(Level, Phase, DebugLogging))); for (auto &C : CGSCCOptimizerLateEPCallbacks) C(MainCGPipeline, Level); @@ -782,8 +781,7 @@ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging, - /*PrepareForThinLTO=*/false)); + MPM.addPass(buildModuleSimplificationPipeline(Level, Regular, DebugLogging)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); @@ -807,8 +805,8 @@ // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we // can. - MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging, - /*PrepareForThinLTO=*/true)); + MPM.addPass(buildModuleSimplificationPipeline(Level, PreLinkThinLTO, + DebugLogging)); // Run partial inlining pass to partially inline functions that have // large bodies. @@ -846,8 +844,8 @@ !PGOOpt->ProfileUseFile.empty())); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, DebugLogging, - /*PrepareForThinLTO=*/false)); + MPM.addPass(buildModuleSimplificationPipeline(Level, PostLinkThinLTO, + DebugLogging)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));