diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -350,8 +350,8 @@ /// Construct the module pipeline that performs inlining as well as /// the inlining-driven cleanups. - ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, - ThinOrFullLTOPhase Phase); + ModulePassManager buildInlinerPipeline(OptimizationLevel Level, + ThinOrFullLTOPhase Phase); /// Construct the core LLVM module optimization pipeline. /// diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h --- a/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -110,39 +110,24 @@ const bool OnlyMandatory; }; -/// Module pass, wrapping the inliner pass. This works in conjunction with the +/// Create a ModulePassManager that revolves around the inliner. The +/// NoInlinerCGPM should contain all passes that run after the inliner /// InlineAdvisorAnalysis to facilitate inlining decisions taking into account -/// module-wide state, that need to keep track of inter-inliner pass runs, for +/// module-wide state, that need to keep track of inter-inliner pass runs for /// a given module. An InlineAdvisor is configured and kept alive for the /// duration of the ModuleInlinerWrapperPass::run. -class ModuleInlinerWrapperPass - : public PassInfoMixin { -public: - ModuleInlinerWrapperPass( - InlineParams Params = getInlineParams(), bool Debugging = false, - bool MandatoryFirst = true, - InliningAdvisorMode Mode = InliningAdvisorMode::Default, - unsigned MaxDevirtIterations = 0); - ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default; - - PreservedAnalyses run(Module &, ModuleAnalysisManager &); - - /// Allow adding more CGSCC passes, besides inlining. This should be called - /// before run is called, as part of pass pipeline building. - CGSCCPassManager &getPM() { return PM; } +ModulePassManager +createInlinerPipeline(CGSCCPassManager NoInlinerCGPM, + InlineParams Params = getInlineParams(), + bool DebugLogging = false, bool MandatoryFirst = true, + InliningAdvisorMode Mode = InliningAdvisorMode::Default, + unsigned MaxDevirtIterations = 0); +ModulePassManager +createInlinerPipeline(InlineParams Params = getInlineParams(), + bool DebugLogging = false, bool MandatoryFirst = true, + InliningAdvisorMode Mode = InliningAdvisorMode::Default, + unsigned MaxDevirtIterations = 0); - /// Allow adding module-level analyses benefiting the contained CGSCC passes. - template void addRequiredModuleAnalysis() { - MPM.addPass(RequireAnalysisPass()); - } - -private: - const InlineParams Params; - const InliningAdvisorMode Mode; - const unsigned MaxDevirtIterations; - CGSCCPassManager PM; - ModulePassManager MPM; -}; } // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_INLINER_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -880,8 +880,7 @@ // performance testing. // FIXME: this comment is cargo culted from the old pass manager, revisit). IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325; - ModuleInlinerWrapperPass MIWP(IP, DebugLogging); - CGSCCPassManager &CGPipeline = MIWP.getPM(); + CGSCCPassManager CGPipeline(DebugLogging); FunctionPassManager FPM; FPM.addPass(SROA()); @@ -892,7 +891,7 @@ CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); - MPM.addPass(std::move(MIWP)); + MPM.addPass(createInlinerPipeline(std::move(CGPipeline), IP, DebugLogging)); // Delete anything that is now dead to make sure that we don't instrument // dead code. Instrumentation can end up keeping dead code around and @@ -959,9 +958,8 @@ return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel()); } -ModuleInlinerWrapperPass -PassBuilder::buildInlinerPipeline(OptimizationLevel Level, - ThinOrFullLTOPhase Phase) { +ModulePassManager PassBuilder::buildInlinerPipeline(OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { InlineParams IP = getInlineParamsFromOptLevel(Level); if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) @@ -970,23 +968,21 @@ if (PGOOpt) IP.EnableDeferral = EnablePGOInlineDeferral; - ModuleInlinerWrapperPass MIWP(IP, DebugLogging, - PerformMandatoryInliningsFirst, - UseInlineAdvisor, MaxDevirtIterations); + ModulePassManager MPM(DebugLogging); // Require the GlobalsAA analysis for the module so we can query it within // the CGSCC pipeline. - MIWP.addRequiredModuleAnalysis(); + MPM.addPass(RequireAnalysisPass()); // Require the ProfileSummaryAnalysis for the module so we can query it within // the inliner pass. - MIWP.addRequiredModuleAnalysis(); + MPM.addPass(RequireAnalysisPass()); // Now begin the main postorder CGSCC pipeline. // FIXME: The current CGSCC pipeline has its origins in the legacy pass // manager and trying to emulate its precise behavior. Much of this doesn't // make a lot of sense and we should revisit the core CGSCC structure. - CGSCCPassManager &MainCGPipeline = MIWP.getPM(); + CGSCCPassManager MainCGPipeline(DebugLogging); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is @@ -1020,7 +1016,11 @@ MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( buildFunctionSimplificationPipeline(Level, Phase))); - return MIWP; + MPM.addPass(createInlinerPipeline(std::move(MainCGPipeline), IP, DebugLogging, + PerformMandatoryInliningsFirst, + UseInlineAdvisor, MaxDevirtIterations)); + + return MPM; } ModulePassManager @@ -1725,8 +1725,8 @@ // valuable as the inliner doesn't currently care whether it is inlining an // invoke or a call. // Run the inliner now. - MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level), - DebugLogging)); + MPM.addPass(createInlinerPipeline( + CGSCCPassManager(), getInlineParamsFromOptLevel(Level), DebugLogging)); // Optimize globals again after we ran the inliner. MPM.addPass(GlobalOptPass()); diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -62,8 +62,8 @@ MODULE_PASS("hwasan", HWAddressSanitizerPass(false, false)) MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true)) MODULE_PASS("inferattrs", InferFunctionAttrsPass()) -MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass()) -MODULE_PASS("inliner-wrapper-no-mandatory-first", ModuleInlinerWrapperPass( +MODULE_PASS("inliner-wrapper", createInlinerPipeline()) +MODULE_PASS("inliner-wrapper-no-mandatory-first", createInlinerPipeline( getInlineParams(), DebugLogging, false)) diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -997,32 +997,48 @@ return PA; } -ModuleInlinerWrapperPass::ModuleInlinerWrapperPass(InlineParams Params, - bool Debugging, - bool MandatoryFirst, - InliningAdvisorMode Mode, - unsigned MaxDevirtIterations) - : Params(Params), Mode(Mode), MaxDevirtIterations(MaxDevirtIterations), - PM(Debugging), MPM(Debugging) { - // Run the inliner first. The theory is that we are walking bottom-up and so - // the callees have already been fully optimized, and we want to inline them - // into the callers so that our optimizations can reflect that. - // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO - // because it makes profile annotation in the backend inaccurate. - if (MandatoryFirst) - PM.addPass(InlinerPass(/*OnlyMandatory*/ true)); - PM.addPass(InlinerPass()); -} +struct CreateInlineAdvisorPass : PassInfoMixin { + CreateInlineAdvisorPass(InlineParams Params, InliningAdvisorMode Mode) + : Params(Params), Mode(Mode) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) { + auto &IAA = AM.getResult(M); + if (!IAA.tryCreate(Params, Mode, CGSCCInlineReplayFile)) { + M.getContext().emitError( + "Could not setup Inlining Advisor for the requested " + "mode and/or options"); + } + return PreservedAnalyses::all(); + } + +private: + InlineParams Params; + InliningAdvisorMode Mode; +}; -PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M, - ModuleAnalysisManager &MAM) { - auto &IAA = MAM.getResult(M); - if (!IAA.tryCreate(Params, Mode, CGSCCInlineReplayFile)) { - M.getContext().emitError( - "Could not setup Inlining Advisor for the requested " - "mode and/or options"); +struct ClearInlineAdvisorPass : PassInfoMixin { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) { + auto *IAA = AM.getCachedResult(M); + assert(IAA && "InlineAdvisorAnalysis should not be invalidated"); + IAA->clear(); return PreservedAnalyses::all(); } +}; + +ModulePassManager llvm::createInlinerPipeline(CGSCCPassManager NoInlinerCGPM, + InlineParams Params, + bool DebugLogging, + bool MandatoryFirst, + InliningAdvisorMode Mode, + unsigned MaxDevirtIterations) { + ModulePassManager MPM(DebugLogging); + + MPM.addPass(CreateInlineAdvisorPass(Params, Mode)); + + CGSCCPassManager CGPM(DebugLogging); + if (MandatoryFirst) + CGPM.addPass(InlinerPass(/*OnlyMandatory*/ true)); + CGPM.addPass(InlinerPass()); + CGPM.addPass(std::move(NoInlinerCGPM)); // We wrap the CGSCC pipeline in a devirtualization repeater. This will try // to detect when we devirtualize indirect calls and iterate the SCC passes @@ -1032,12 +1048,22 @@ // If MaxDevirtIterations is 0, we just don't use the devirtualization // wrapper. if (MaxDevirtIterations == 0) - MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(PM))); + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); else MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( - createDevirtSCCRepeatedPass(std::move(PM), MaxDevirtIterations))); - auto Ret = MPM.run(M, MAM); + createDevirtSCCRepeatedPass(std::move(CGPM), MaxDevirtIterations))); + + MPM.addPass(ClearInlineAdvisorPass()); + + return MPM; +} - IAA.clear(); - return Ret; +ModulePassManager llvm::createInlinerPipeline(InlineParams Params, + bool DebugLogging, + bool MandatoryFirst, + InliningAdvisorMode Mode, + unsigned MaxDevirtIterations) { + return createInlinerPipeline(CGSCCPassManager(DebugLogging), Params, + DebugLogging, MandatoryFirst, Mode, + MaxDevirtIterations); }