diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -555,6 +555,34 @@ return PIC; } + // Invoke the callbacks registered for the various extension points. + // Custom pipelines should use these to invoke the callbacks registered + // by TargetMachines and other clients. + void invokePeepholeEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level); + void invokeLateLoopOptimizationsEPCallbacks(LoopPassManager &LPM, + OptimizationLevel Level); + void invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM, + OptimizationLevel Level); + void invokeScalarOptimizerLateEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level); + void invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM, + OptimizationLevel Level); + void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level); + void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokeFullLinkTimeOptimizationEarlyEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokeFullLinkTimeOptimizationLastEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokePipelineStartEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokePipelineEarlySimplificationEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + private: // O1 pass pipeline FunctionPassManager @@ -589,7 +617,6 @@ std::string ProfileRemappingFile, ThinOrFullLTOPhase LTOPhase, IntrusiveRefCntPtr FS); - void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); // Extension Point callbacks SmallVector, 2> diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -297,8 +297,75 @@ void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM, OptimizationLevel Level) { - for (auto &C : PeepholeEPCallbacks) + for (auto &C : PeepholeEPCallbacks) { C(FPM, Level); + } +} +void PassBuilder::invokeLateLoopOptimizationsEPCallbacks( + LoopPassManager &LPM, OptimizationLevel Level) { + for (auto &C : LateLoopOptimizationsEPCallbacks) { + C(LPM, Level); + } +} +void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM, + OptimizationLevel Level) { + for (auto &C : LoopOptimizerEndEPCallbacks) { + C(LPM, Level); + } +} +void PassBuilder::invokeScalarOptimizerLateEPCallbacks( + FunctionPassManager &FPM, OptimizationLevel Level) { + for (auto &C : ScalarOptimizerLateEPCallbacks) { + C(FPM, Level); + } +} +void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM, + OptimizationLevel Level) { + for (auto &C : CGSCCOptimizerLateEPCallbacks) { + C(CGPM, Level); + } +} +void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level) { + for (auto &C : VectorizerStartEPCallbacks) { + C(FPM, Level); + } +} +void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level) { + for (auto &C : OptimizerEarlyEPCallbacks) { + C(MPM, Level); + } +} +void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level) { + for (auto &C : OptimizerLastEPCallbacks) { + C(MPM, Level); + } +} +void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks( + ModulePassManager &MPM, OptimizationLevel Level) { + for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks) { + C(MPM, Level); + } +} +void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks( + ModulePassManager &MPM, OptimizationLevel Level) { + for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) { + C(MPM, Level); + } +} +void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level) { + for (auto &C : PipelineStartEPCallbacks) { + C(MPM, Level); + } +} +void PassBuilder::invokePipelineEarlySimplificationEPCallbacks( + ModulePassManager &MPM, OptimizationLevel Level) { + for (auto &C : PipelineEarlySimplificationEPCallbacks) { + C(MPM, Level); + } } // Helper to add AnnotationRemarksPass. @@ -384,8 +451,7 @@ LPM2.addPass(LoopIdiomRecognizePass()); LPM2.addPass(IndVarSimplifyPass()); - for (auto &C : LateLoopOptimizationsEPCallbacks) - C(LPM2, Level); + invokeLateLoopOptimizationsEPCallbacks(LPM2, Level); LPM2.addPass(LoopDeletionPass()); @@ -403,8 +469,7 @@ /* OnlyWhenForced= */ !PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll)); - for (auto &C : LoopOptimizerEndEPCallbacks) - C(LPM2, Level); + invokeLoopOptimizerEndEPCallbacks(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. @@ -445,8 +510,7 @@ FPM.addPass(CoroElidePass()); - for (auto &C : ScalarOptimizerLateEPCallbacks) - C(FPM, Level); + invokeScalarOptimizerLateEPCallbacks(FPM, Level); // Finally, do an expensive DCE pass to catch all the dead code exposed by // the simplifications and basic cleanup after all the simplifications. @@ -570,8 +634,7 @@ LPM2.addPass(LoopIdiomRecognizePass()); LPM2.addPass(IndVarSimplifyPass()); - for (auto &C : LateLoopOptimizationsEPCallbacks) - C(LPM2, Level); + invokeLateLoopOptimizationsEPCallbacks(LPM2, Level); LPM2.addPass(LoopDeletionPass()); @@ -589,8 +652,7 @@ /* OnlyWhenForced= */ !PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll)); - for (auto &C : LoopOptimizerEndEPCallbacks) - C(LPM2, Level); + invokeLoopOptimizerEndEPCallbacks(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. @@ -662,8 +724,7 @@ FPM.addPass(CoroElidePass()); - for (auto &C : ScalarOptimizerLateEPCallbacks) - C(FPM, Level); + invokeScalarOptimizerLateEPCallbacks(FPM, Level); FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions() .convertSwitchRangeToICmp(true) @@ -849,8 +910,7 @@ if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3) MainCGPipeline.addPass(OpenMPOptCGSCCPass()); - for (auto &C : CGSCCOptimizerLateEPCallbacks) - C(MainCGPipeline, Level); + invokeCGSCCOptimizerLateEPCallbacks(MainCGPipeline, Level); // Add the core function simplification pipeline nested inside the // CGSCC walk. @@ -1007,8 +1067,7 @@ if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink) MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); - for (auto &C : PipelineEarlySimplificationEPCallbacks) - C(MPM, Level); + invokePipelineEarlySimplificationEPCallbacks(MPM, Level); // Interprocedural constant propagation now that basic cleanup has occurred // and prior to optimizing globals. @@ -1279,8 +1338,7 @@ // memory operations. MPM.addPass(RecomputeGlobalsAAPass()); - for (auto &C : OptimizerEarlyEPCallbacks) - C(MPM, Level); + invokeOptimizerEarlyEPCallbacks(MPM, Level); FunctionPassManager OptimizePM; OptimizePM.addPass(Float2IntPass()); @@ -1303,8 +1361,7 @@ // rather than on each loop in an inside-out manner, and so they are actually // function passes. - for (auto &C : VectorizerStartEPCallbacks) - C(OptimizePM, Level); + invokeVectorizerStartEPCallbacks(OptimizePM, Level); LoopPassManager LPM; // First rotate loops that may have been un-rotated by prior passes. @@ -1356,8 +1413,7 @@ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM), PTO.EagerlyInvalidateAnalyses)); - for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level); // Split out cold code. Splitting is done late to avoid hiding context from // other optimizations and inadvertently regressing performance. The tradeoff @@ -1413,8 +1469,7 @@ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Apply module pipeline start EP callback. - for (auto &C : PipelineStartEPCallbacks) - C(MPM, Level); + invokePipelineStartEPCallbacks(MPM, Level); const ThinOrFullLTOPhase LTOPhase = LTOPreLink ? ThinOrFullLTOPhase::FullLTOPreLink @@ -1455,8 +1510,7 @@ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Apply module pipeline start EP callback. - for (auto &C : PipelineStartEPCallbacks) - C(MPM, Level); + invokePipelineStartEPCallbacks(MPM, Level); // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we @@ -1481,10 +1535,8 @@ // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual // optimization is going to be done in PostLink stage, but clang can't add // callbacks there in case of in-process ThinLTO called by linker. - for (auto &C : OptimizerEarlyEPCallbacks) - C(MPM, Level); - for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); + invokeOptimizerEarlyEPCallbacks(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1559,8 +1611,7 @@ ModuleSummaryIndex *ExportSummary) { ModulePassManager MPM; - for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level); // Create a function that performs CFI checks for cross-DSO calls with targets // in the current module. @@ -1575,8 +1626,7 @@ // in ICP. MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); - for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1657,8 +1707,7 @@ // pipeline). MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); - for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1867,8 +1916,7 @@ if (PTO.CallGraphProfile) MPM.addPass(CGProfilePass()); - for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1898,14 +1946,12 @@ /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, PGOOpt->FS); - for (auto &C : PipelineStartEPCallbacks) - C(MPM, Level); + invokePipelineStartEPCallbacks(MPM, Level); if (PGOOpt && PGOOpt->DebugInfoForProfiling) MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); - for (auto &C : PipelineEarlySimplificationEPCallbacks) - C(MPM, Level); + invokePipelineEarlySimplificationEPCallbacks(MPM, Level); // Build a minimal pipeline based on the semantics required by LLVM, // which is just that always inlining occurs. Further, disable generating @@ -1923,15 +1969,13 @@ if (!CGSCCOptimizerLateEPCallbacks.empty()) { CGSCCPassManager CGPM; - for (auto &C : CGSCCOptimizerLateEPCallbacks) - C(CGPM, Level); + invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level); if (!CGPM.isEmpty()) MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); } if (!LateLoopOptimizationsEPCallbacks.empty()) { LoopPassManager LPM; - for (auto &C : LateLoopOptimizationsEPCallbacks) - C(LPM, Level); + invokeLateLoopOptimizationsEPCallbacks(LPM, Level); if (!LPM.isEmpty()) { MPM.addPass(createModuleToFunctionPassAdaptor( createFunctionToLoopPassAdaptor(std::move(LPM)))); @@ -1939,8 +1983,7 @@ } if (!LoopOptimizerEndEPCallbacks.empty()) { LoopPassManager LPM; - for (auto &C : LoopOptimizerEndEPCallbacks) - C(LPM, Level); + invokeLoopOptimizerEndEPCallbacks(LPM, Level); if (!LPM.isEmpty()) { MPM.addPass(createModuleToFunctionPassAdaptor( createFunctionToLoopPassAdaptor(std::move(LPM)))); @@ -1948,19 +1991,16 @@ } if (!ScalarOptimizerLateEPCallbacks.empty()) { FunctionPassManager FPM; - for (auto &C : ScalarOptimizerLateEPCallbacks) - C(FPM, Level); + invokeScalarOptimizerLateEPCallbacks(FPM, Level); if (!FPM.isEmpty()) MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - for (auto &C : OptimizerEarlyEPCallbacks) - C(MPM, Level); + invokeOptimizerEarlyEPCallbacks(MPM, Level); if (!VectorizerStartEPCallbacks.empty()) { FunctionPassManager FPM; - for (auto &C : VectorizerStartEPCallbacks) - C(FPM, Level); + invokeVectorizerStartEPCallbacks(FPM, Level); if (!FPM.isEmpty()) MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } @@ -1974,8 +2014,7 @@ CoroPM.addPass(GlobalDCEPass()); MPM.addPass(CoroConditionalWrapper(std::move(CoroPM))); - for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level); if (LTOPreLink) addRequiredLTOPreLinkPasses(MPM);