Index: llvm/include/llvm/Passes/PassBuilder.h =================================================================== --- llvm/include/llvm/Passes/PassBuilder.h +++ llvm/include/llvm/Passes/PassBuilder.h @@ -228,11 +228,12 @@ /// Build a per-module default optimization pipeline. /// /// This provides a good default optimization pipeline for per-module - /// optimization and code generation without any link-time optimization. It - /// typically correspond to frontend "-O[123]" options for optimization - /// levels \c O1, \c O2 and \c O3 resp. - ModulePassManager buildPerModuleDefaultPipeline(OptimizationLevel Level, - bool LTOPreLink = false); + /// optimization and code generation, and can also be configured for + /// pre-link optimization. It typically correspond to frontend "-O[123]" + /// options for optimization levels \c O1, \c O2 and \c O3 resp. + ModulePassManager buildPerModuleDefaultPipeline( + OptimizationLevel Level, + ThinOrFullLTOPhase Phase = ThinOrFullLTOPhase::None); /// Build a pre-link, ThinLTO-targeting default optimization pipeline to /// a pass manager. Index: llvm/lib/Passes/PassBuilderPipelines.cpp =================================================================== --- llvm/lib/Passes/PassBuilderPipelines.cpp +++ llvm/lib/Passes/PassBuilderPipelines.cpp @@ -1227,7 +1227,7 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase LTOPhase) { - const bool LTOPreLink = isLTOPreLink(LTOPhase); + assert(!isLTOPreLink(LTOPhase) && "Should not run during LTO pre-link"); ModulePassManager MPM; // Run partial inlining pass to partially inline functions that have @@ -1236,16 +1236,14 @@ MPM.addPass(PartialInlinerPass()); // Remove avail extern fns and globals definitions since we aren't compiling - // an object file for later LTO. For LTO we want to preserve these so they - // are eligible for inlining at link-time. Note if they are unreferenced they + // an object file for later LTO. Note if they are unreferenced they // will be removed by GlobalDCE later, so this only impacts referenced // available externally globals. Eventually they will be suppressed during // codegen, but eliminating here enables more opportunity for GlobalDCE as it // may make globals referenced by available external functions dead and saves // running remaining passes on the eliminated functions. These should be // preserved during prelinking for link-time inlining decisions. - if (!LTOPreLink) - MPM.addPass(EliminateAvailableExternallyPass()); + MPM.addPass(EliminateAvailableExternallyPass()); if (EnableOrderFileInstrumentation) MPM.addPass(InstrOrderFilePass()); @@ -1256,10 +1254,8 @@ MPM.addPass(ReversePostOrderFunctionAttrsPass()); // Do a post inline PGO instrumentation and use pass. This is a context - // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as - // cross-module inline has not been done yet. The context sensitive - // instrumentation is after all the inlines are done. - if (!LTOPreLink && PGOOpt) { + // sensitive PGO pass. + if (PGOOpt) { if (PGOOpt->CSAction == PGOOptions::CSIRInstr) addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ true, PGOOpt->CSProfileGenFile, @@ -1309,7 +1305,7 @@ LoopPassManager LPM; // First rotate loops that may have been un-rotated by prior passes. // Disable header duplication at -Oz. - LPM.addPass(LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink)); + LPM.addPass(LoopRotatePass(Level != OptimizationLevel::Oz)); // Some loops may have become dead by now. Try to delete them. // FIXME: see discussion in https://reviews.llvm.org/D112851, // this may need to be revisited once we run GVN before loop deletion @@ -1362,7 +1358,7 @@ // Split out cold code. Splitting is done late to avoid hiding context from // other optimizations and inadvertently regressing performance. The tradeoff // is that this has a higher code size cost than splitting early. - if (EnableHotColdSplit && !LTOPreLink) + if (EnableHotColdSplit) MPM.addPass(HotColdSplittingPass()); // Search the code for similar regions of code. If enough similar regions can @@ -1383,23 +1379,19 @@ MPM.addPass(GlobalDCEPass()); MPM.addPass(ConstantMergePass()); - if (PTO.CallGraphProfile && !LTOPreLink) + if (PTO.CallGraphProfile) MPM.addPass(CGProfilePass()); - // TODO: Relative look table converter pass caused an issue when full lto is - // enabled. See https://reviews.llvm.org/D94355 for more details. - // Until the issue fixed, disable this pass during pre-linking phase. - if (!LTOPreLink) - MPM.addPass(RelLookupTableConverterPass()); + MPM.addPass(RelLookupTableConverterPass()); return MPM; } ModulePassManager PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, - bool LTOPreLink) { + ThinOrFullLTOPhase Phase) { if (Level == OptimizationLevel::O0) - return buildO0DefaultPipeline(Level, LTOPreLink); + return buildO0DefaultPipeline(Level, isLTOPreLink(Phase)); ModulePassManager MPM; @@ -1416,14 +1408,32 @@ for (auto &C : PipelineStartEPCallbacks) C(MPM, Level); - const ThinOrFullLTOPhase LTOPhase = LTOPreLink - ? ThinOrFullLTOPhase::FullLTOPreLink - : ThinOrFullLTOPhase::None; // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, LTOPhase)); + MPM.addPass(buildModuleSimplificationPipeline(Level, Phase)); - // Now add the optimization pipeline. - MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPhase)); + // If we are planning to perform LTO later, don't bloat the code with + // unrolling/vectorization/... now. + if (!isLTOPreLink(Phase)) { + MPM.addPass(buildModuleOptimizationPipeline(Level, Phase)); + } else { + // Run partial inlining pass to partially inline functions that have + // large bodies. + // FIXME: It isn't clear whether this is really the right place to run this + // in ThinLTO. Because there is another canonicalization and simplification + // phase that will run after the thin link, running this here ends up with + // less information than will be available later and it may grow functions + // in ways that aren't beneficial. + if (RunPartialInlining) + MPM.addPass(PartialInlinerPass()); + + // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual + // optimization is going to be done in PostLink stage, but clang can't add + // callbacks there in case of in-process LTO called by linker. + for (auto &C : OptimizerEarlyEPCallbacks) + C(MPM, Level); + for (auto &C : OptimizerLastEPCallbacks) + C(MPM, Level); + } if (PGOOpt && PGOOpt->PseudoProbeForProfiling && PGOOpt->Action == PGOOptions::SampleUse) @@ -1432,7 +1442,7 @@ // Emit annotation remarks. addAnnotationRemarksPass(MPM); - if (LTOPreLink) + if (isLTOPreLink(Phase)) addRequiredLTOPreLinkPasses(MPM); return MPM; @@ -1440,58 +1450,8 @@ ModulePassManager PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) { - if (Level == OptimizationLevel::O0) - return buildO0DefaultPipeline(Level, /*LTOPreLink*/true); - - ModulePassManager MPM; - - // Convert @llvm.global.annotations to !annotation metadata. - MPM.addPass(Annotation2MetadataPass()); - - // Force any function attributes we want the rest of the pipeline to observe. - MPM.addPass(ForceFunctionAttrsPass()); - - if (PGOOpt && PGOOpt->DebugInfoForProfiling) - MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); - - // Apply module pipeline start EP callback. - for (auto &C : PipelineStartEPCallbacks) - C(MPM, Level); - - // If we are planning to perform ThinLTO later, we don't bloat the code with - // unrolling/vectorization/... now. Just simplify the module as much as we - // can. - MPM.addPass(buildModuleSimplificationPipeline( - Level, ThinOrFullLTOPhase::ThinLTOPreLink)); - - // Run partial inlining pass to partially inline functions that have - // large bodies. - // FIXME: It isn't clear whether this is really the right place to run this - // in ThinLTO. Because there is another canonicalization and simplification - // phase that will run after the thin link, running this here ends up with - // less information than will be available later and it may grow functions in - // ways that aren't beneficial. - if (RunPartialInlining) - MPM.addPass(PartialInlinerPass()); - - if (PGOOpt && PGOOpt->PseudoProbeForProfiling && - PGOOpt->Action == PGOOptions::SampleUse) - MPM.addPass(PseudoProbeUpdatePass()); - - // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual - // optimization is going to be done in PostLink stage, but clang can't add - // callbacks there in case of in-process ThinLTO called by linker. - for (auto &C : OptimizerEarlyEPCallbacks) - C(MPM, Level); - for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); - - // Emit annotation remarks. - addAnnotationRemarksPass(MPM); - - addRequiredLTOPreLinkPasses(MPM); - - return MPM; + return buildPerModuleDefaultPipeline(Level, + ThinOrFullLTOPhase::ThinLTOPreLink); } ModulePassManager PassBuilder::buildThinLTODefaultPipeline( @@ -1549,9 +1509,8 @@ ModulePassManager PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) { - // FIXME: We should use a customized pre-link pipeline! return buildPerModuleDefaultPipeline(Level, - /* LTOPreLink */ true); + ThinOrFullLTOPhase::FullLTOPreLink); } ModulePassManager Index: llvm/test/Other/new-pm-defaults.ll =================================================================== --- llvm/test/Other/new-pm-defaults.ll +++ llvm/test/Other/new-pm-defaults.ll @@ -22,9 +22,6 @@ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ ; RUN: -passes='default' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-NO-FUNC-SPEC,CHECK-Oz,CHECK-O23SZ,%llvmcheckext -; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ -; RUN: -passes='lto-pre-link' -S %s 2>&1 \ -; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-LTO,CHECK-NO-FUNC-SPEC,CHECK-O2,CHECK-O23SZ,%llvmcheckext ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ ; RUN: -passes-ep-peephole='no-op-function' \ @@ -59,10 +56,6 @@ ; RUN: -passes='default' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-FUNC-SPEC,CHECK-O3,%llvmcheckext,CHECK-EP-PIPELINE-EARLY-SIMPLIFICATION,CHECK-O23SZ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ -; RUN: -passes-ep-pipeline-start='no-op-module' \ -; RUN: -passes='lto-pre-link' -S %s 2>&1 \ -; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-LTO,CHECK-NO-FUNC-SPEC,CHECK-O3,%llvmcheckext,CHECK-EP-PIPELINE-START,CHECK-O23SZ -; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ ; RUN: -passes-ep-optimizer-early='no-op-module' \ ; RUN: -passes='default' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-DEFAULT,CHECK-FUNC-SPEC,CHECK-O3,%llvmcheckext,CHECK-EP-OPTIMIZER-EARLY,CHECK-O23SZ @@ -226,7 +219,6 @@ ; CHECK-O-NEXT: Running pass: GlobalOptPass ; CHECK-O-NEXT: Running pass: GlobalDCEPass ; CHECK-DEFAULT-NEXT: Running pass: EliminateAvailableExternallyPass -; CHECK-LTO-NOT: Running pass: EliminateAvailableExternallyPass ; CHECK-O-NEXT: Running pass: ReversePostOrderFunctionAttrsPass ; CHECK-O-NEXT: Running pass: RecomputeGlobalsAAPass ; CHECK-EP-OPTIMIZER-EARLY: Running pass: NoOpModulePass @@ -277,10 +269,7 @@ ; CHECK-O-NEXT: Running pass: ConstantMergePass ; CHECK-DEFAULT-NEXT: Running pass: CGProfilePass ; CHECK-DEFAULT-NEXT: Running pass: RelLookupTableConverterPass -; CHECK-LTO-NOT: Running pass: RelLookupTableConverterPass ; CHECK-O-NEXT: Running pass: AnnotationRemarksPass on foo -; CHECK-LTO-NEXT: Running pass: CanonicalizeAliasesPass -; CHECK-LTO-NEXT: Running pass: NameAnonGlobalPass ; CHECK-O-NEXT: Running pass: PrintModulePass ; ; Make sure we get the IR back out without changes when we print the module. Index: llvm/test/Other/new-pm-lto-prelink-defaults.ll =================================================================== --- llvm/test/Other/new-pm-lto-prelink-defaults.ll +++ llvm/test/Other/new-pm-lto-prelink-defaults.ll @@ -32,7 +32,22 @@ ; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager -debug-info-for-profiling \ ; RUN: -passes='thinlto-pre-link' -S %s 2>&1 \ ; RUN: | FileCheck %s --check-prefixes=CHECK-DIS,CHECK-O,CHECK-O2,CHECK-O23SZ -; + +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes='lto-pre-link' -S %s 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O1,CHECK-O-NODIS,CHECK-FULL +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes='lto-pre-link' -S %s 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-O-NODIS,CHECK-FULL +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes='lto-pre-link' -S %s 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O23SZ,CHECK-O-NODIS,CHECK-FULL +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager \ +; RUN: -passes='lto-pre-link' -S %s 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-O,CHECK-O23SZ,CHECK-O-NODIS,CHECK-FULL +; RUN: opt -disable-verify -verify-analysis-invalidation=0 -eagerly-invalidate-analyses=0 -debug-pass-manager -debug-info-for-profiling \ +; RUN: -passes='lto-pre-link' -S %s 2>&1 \ +; RUN: | FileCheck %s --check-prefixes=CHECK-DIS,CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-FULL ; Suppress FileCheck --allow-unused-prefixes=false diagnostics. ; CHECK-NOEXT: {{^}} @@ -163,6 +178,7 @@ ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass ; CHECK-O-NEXT: Running pass: CoroCleanupPass ; CHECK-O-NEXT: Running pass: GlobalOptPass +; CHECK-FULL-NEXT: Running pass: GlobalDCEPass ; CHECK-EXT: Running pass: {{.*}}::Bye ; CHECK-EP-OPT-EARLY-NEXT: Running pass: NoOpModulePass ; CHECK-EP-OPT-LAST-NEXT: Running pass: NoOpModulePass