diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h --- a/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -477,11 +477,12 @@ public: using PassConceptT = detail::PassConcept; - explicit CGSCCToFunctionPassAdaptor(std::unique_ptr Pass) - : Pass(std::move(Pass)) {} + explicit CGSCCToFunctionPassAdaptor(std::unique_ptr Pass, + bool EagerlyInvalidate) + : Pass(std::move(Pass)), EagerlyInvalidate(EagerlyInvalidate) {} CGSCCToFunctionPassAdaptor(CGSCCToFunctionPassAdaptor &&Arg) - : Pass(std::move(Arg.Pass)) {} + : Pass(std::move(Arg.Pass)), EagerlyInvalidate(Arg.EagerlyInvalidate) {} friend void swap(CGSCCToFunctionPassAdaptor &LHS, CGSCCToFunctionPassAdaptor &RHS) { @@ -499,7 +500,10 @@ void printPipeline(raw_ostream &OS, function_ref MapClassName2PassName) { - OS << "function("; + OS << "function"; + if (EagerlyInvalidate) + OS << ""; + OS << "("; Pass->printPipeline(OS, MapClassName2PassName); OS << ")"; } @@ -508,13 +512,15 @@ private: std::unique_ptr Pass; + bool EagerlyInvalidate; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. template CGSCCToFunctionPassAdaptor -createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass) { +createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass, + bool EagerlyInvalidate = false) { using PassModelT = detail::PassModel; @@ -522,7 +528,8 @@ // causing terrible compile times. return CGSCCToFunctionPassAdaptor( std::unique_ptr( - new PassModelT(std::forward(Pass)))); + new PassModelT(std::forward(Pass))), + EagerlyInvalidate); } /// A helper that repeats an SCC pass each time an indirect call is refined to diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -1204,8 +1204,9 @@ public: using PassConceptT = detail::PassConcept; - explicit ModuleToFunctionPassAdaptor(std::unique_ptr Pass) - : Pass(std::move(Pass)) {} + explicit ModuleToFunctionPassAdaptor(std::unique_ptr Pass, + bool EagerlyInvalidate) + : Pass(std::move(Pass)), EagerlyInvalidate(EagerlyInvalidate) {} /// Runs the function pass across every function in the module. PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); @@ -1216,13 +1217,15 @@ private: std::unique_ptr Pass; + bool EagerlyInvalidate; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. template ModuleToFunctionPassAdaptor -createModuleToFunctionPassAdaptor(FunctionPassT &&Pass) { +createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, + bool EagerlyInvalidate = false) { using PassModelT = detail::PassModel; @@ -1230,7 +1233,8 @@ // causing terrible compile times. return ModuleToFunctionPassAdaptor( std::unique_ptr( - new PassModelT(std::forward(Pass)))); + new PassModelT(std::forward(Pass))), + EagerlyInvalidate); } /// A utility pass template to force an analysis result to be available. diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -73,6 +73,15 @@ /// Tuning option to enable/disable function merging. Its default value is /// false. bool MergeFunctions; + + // Experimental option to eagerly invalidate more analyses. This has the + // potential to decrease max memory usage in exchange for more compile time. + // This may affect codegen due to either passes using analyses only when + // cached, or invalidating and recalculating an analysis that was + // stale/imprecise but still valid. Currently this invalidates all function + // analyses after various module->function or cgscc->function adaptors in the + // default pipelines. + bool EagerlyInvalidateAnalyses; }; /// This class provides access to building LLVM's passes. diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp --- a/llvm/lib/Analysis/CGSCCPassManager.cpp +++ b/llvm/lib/Analysis/CGSCCPassManager.cpp @@ -38,8 +38,6 @@ // Explicit template instantiations and specialization definitions for core // template typedefs. namespace llvm { -extern cl::opt EagerlyInvalidateAnalyses; - static cl::opt AbortOnMaxDevirtIterationsReached( "abort-on-max-devirt-iterations-reached", cl::desc("Abort when the max iterations for devirtualization CGSCC repeat " @@ -557,8 +555,7 @@ // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. - FAM.invalidate(F, EagerlyInvalidateAnalyses ? PreservedAnalyses::none() - : PassPA); + FAM.invalidate(F, EagerlyInvalidate ? PreservedAnalyses::none() : PassPA); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp --- a/llvm/lib/IR/PassManager.cpp +++ b/llvm/lib/IR/PassManager.cpp @@ -15,17 +15,6 @@ using namespace llvm; namespace llvm { -// Experimental option to eagerly invalidate more analyses. This has the -// potential to decrease max memory usage in exchange for more compile time. -// This may affect codegen due to either passes using analyses only when -// cached, or invalidating and recalculating an analysis that was -// stale/imprecise but still valid. Currently this invalidates all function -// analyses after a module->function or cgscc->function adaptor. -// TODO: make this a PipelineTuningOption. -cl::opt EagerlyInvalidateAnalyses( - "eagerly-invalidate-analyses", cl::init(false), cl::Hidden, - cl::desc("Eagerly invalidate more analyses in default pipelines")); - // Explicit template instantiations and specialization defininitions for core // template typedefs. template class AllAnalysesOn; @@ -105,7 +94,10 @@ void ModuleToFunctionPassAdaptor::printPipeline( raw_ostream &OS, function_ref MapClassName2PassName) { - OS << "function("; + OS << "function"; + if (EagerlyInvalidate) + OS << ""; + OS << "("; Pass->printPipeline(OS, MapClassName2PassName); OS << ")"; } @@ -141,8 +133,7 @@ // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. - FAM.invalidate(F, EagerlyInvalidateAnalyses ? PreservedAnalyses::none() - : PassPA); + FAM.invalidate(F, EagerlyInvalidate ? PreservedAnalyses::none() : PassPA); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -832,7 +832,7 @@ return true; if (Name == "cgscc") return true; - if (Name == "function") + if (Name == "function" || Name == "function") return true; // Explicitly handle custom-parsed pass names. @@ -858,7 +858,7 @@ // Explicitly handle pass manager names. if (Name == "cgscc") return true; - if (Name == "function") + if (Name == "function" || Name == "function") return true; // Explicitly handle custom-parsed pass names. @@ -884,7 +884,7 @@ template static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. - if (Name == "function") + if (Name == "function" || Name == "function") return true; if (Name == "loop" || Name == "loop-mssa") return true; @@ -1013,11 +1013,12 @@ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); return Error::success(); } - if (Name == "function") { + if (Name == "function" || Name == "function") { FunctionPassManager FPM; if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline)) return Err; - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), + Name != "function")); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { @@ -1180,12 +1181,13 @@ CGPM.addPass(std::move(NestedCGPM)); return Error::success(); } - if (Name == "function") { + if (Name == "function" || Name == "function") { FunctionPassManager FPM; if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline)) return Err; // Add the nested pass manager with the appropriate adaptor. - CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); + CGPM.addPass( + createCGSCCToFunctionPassAdaptor(std::move(FPM), Name != "function")); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -162,6 +162,10 @@ "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3")); +static cl::opt EnableEagerlyInvalidateAnalyses( + "eagerly-invalidate-analyses", cl::init(false), cl::Hidden, + cl::desc("Eagerly invalidate more analyses in default pipelines")); + PipelineTuningOptions::PipelineTuningOptions() { LoopInterleaving = true; LoopVectorization = true; @@ -172,6 +176,7 @@ LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap; CallGraphProfile = true; MergeFunctions = false; + EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses; } namespace llvm { @@ -596,7 +601,8 @@ FPM.addPass(InstCombinePass()); // Combine silly sequences. invokePeepholeEPCallbacks(FPM, Level); - CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); + CGPipeline.addPass(createCGSCCToFunctionPassAdaptor( + std::move(FPM), PTO.EagerlyInvalidateAnalyses)); MPM.addPass(std::move(MIWP)); @@ -623,7 +629,8 @@ FPM.addPass(createFunctionToLoopPassAdaptor( LoopRotatePass(Level != OptimizationLevel::Oz), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false)); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), + PTO.EagerlyInvalidateAnalyses)); // Add the profile lowering pass. InstrProfOptions Options; @@ -723,7 +730,8 @@ // Lastly, add the core function simplification pipeline nested inside the // CGSCC walk. MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( - buildFunctionSimplificationPipeline(Level, Phase))); + buildFunctionSimplificationPipeline(Level, Phase), + PTO.EagerlyInvalidateAnalyses)); MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0)); @@ -792,7 +800,8 @@ // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured. if (LoadSampleProfile) EarlyFPM.addPass(InstCombinePass()); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM), + PTO.EagerlyInvalidateAnalyses)); if (LoadSampleProfile) { // Annotate sample profile right after early FPM to ensure freshness of @@ -866,7 +875,8 @@ invokePeepholeEPCallbacks(GlobalCleanupPM, Level); GlobalCleanupPM.addPass(SimplifyCFGPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM), + PTO.EagerlyInvalidateAnalyses)); // Add all the requested passes for instrumentation PGO, if requested. if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink && @@ -1154,7 +1164,8 @@ OptimizePM.addPass(CoroCleanupPass()); // Add the core optimizing pipeline. - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM), + PTO.EagerlyInvalidateAnalyses)); for (auto &C : OptimizerLastEPCallbacks) C(MPM, Level); @@ -1397,7 +1408,8 @@ if (Level.getSpeedupLevel() > 1) { FunctionPassManager EarlyFPM; EarlyFPM.addPass(CallSiteSplittingPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor( + std::move(EarlyFPM), PTO.EagerlyInvalidateAnalyses)); // Indirect call promotion. This should promote all the targets that are // left by the earlier promotion pass that promotes intra-module targets. @@ -1473,7 +1485,8 @@ PeepholeFPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(PeepholeFPM, Level); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM), + PTO.EagerlyInvalidateAnalyses)); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is @@ -1520,7 +1533,8 @@ FPM.addPass(TailCallElimPass()); // Run a few AA driver optimizations here and now to cleanup the code. - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), + PTO.EagerlyInvalidateAnalyses)); MPM.addPass( createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass())); @@ -1577,7 +1591,8 @@ invokePeepholeEPCallbacks(MainFPM, Level); MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true)); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM), + PTO.EagerlyInvalidateAnalyses)); // Lower type metadata and the type.test intrinsic. This pass supports // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs diff --git a/llvm/test/Other/new-pm-eager-invalidate.ll b/llvm/test/Other/new-pm-eager-invalidate.ll --- a/llvm/test/Other/new-pm-eager-invalidate.ll +++ b/llvm/test/Other/new-pm-eager-invalidate.ll @@ -1,8 +1,27 @@ -; RUN: opt -disable-verify -debug-pass-manager -passes='function(require)' -disable-output -eagerly-invalidate-analyses %s 2>&1 | FileCheck %s -; RUN: opt -disable-verify -debug-pass-manager -passes='cgscc(function(require))' -disable-output -eagerly-invalidate-analyses %s 2>&1 | FileCheck %s +; RUN: opt -disable-verify -debug-pass-manager -passes='function(require)' -disable-output %s 2>&1 | FileCheck %s --check-prefix=NORMAL +; RUN: opt -disable-verify -debug-pass-manager -passes='cgscc(function(require))' -disable-output %s 2>&1 | FileCheck %s --check-prefix=NORMAL +; RUN: opt -disable-verify -debug-pass-manager -passes='function(require)' -disable-output %s 2>&1 | FileCheck %s --check-prefix=EAGER +; RUN: opt -disable-verify -debug-pass-manager -passes='cgscc(function(require))' -disable-output %s 2>&1 | FileCheck %s --check-prefix=EAGER -; CHECK: Invalidating analysis: NoOpFunctionAnalysis +; RUN: opt -disable-verify -debug-pass-manager -passes='default' -disable-output %s 2>&1 | FileCheck %s --check-prefix=PIPELINE +; RUN: opt -disable-verify -debug-pass-manager -passes='default' -eagerly-invalidate-analyses -disable-output %s 2>&1 | FileCheck %s --check-prefix=PIPELINE-EAGER -define void @foo() { - unreachable +; NORMAL-NOT: Invalidating analysis: NoOpFunctionAnalysis +; EAGER: Invalidating analysis: NoOpFunctionAnalysis +; PIPELINE-NOT: Invalidating analysis: DominatorTreeAnalysis +; PIPELINE-EAGER: Invalidating analysis: DominatorTreeAnalysis + +declare void @bar() local_unnamed_addr + +define void @foo(i32 %n) local_unnamed_addr { +entry: + br label %loop +loop: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] + %iv.next = add i32 %iv, 1 + tail call void @bar() + %cmp = icmp eq i32 %iv, %n + br i1 %cmp, label %exit, label %loop +exit: + ret void } diff --git a/llvm/test/Other/new-pm-print-pipeline.ll b/llvm/test/Other/new-pm-print-pipeline.ll --- a/llvm/test/Other/new-pm-print-pipeline.ll +++ b/llvm/test/Other/new-pm-print-pipeline.ll @@ -66,3 +66,6 @@ ; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='scc-oz-module-inliner' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-21 ; CHECK-21: require,function(invalidate),require,cgscc(devirt<4>(inline,inline,{{.*}},instcombine{{.*}})) + +; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='cgscc(function(no-op-function)),function(no-op-function)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-22 +; CHECK-22: cgscc(function(no-op-function)),function(no-op-function)