Index: llvm/include/llvm/Pass.h =================================================================== --- llvm/include/llvm/Pass.h +++ llvm/include/llvm/Pass.h @@ -69,6 +69,22 @@ PT_PassManager }; +/// LTO or ThinLTO phases. +/// +/// This enumerates the LLVM LTO or ThinLTO optimization phases. +enum class PhaseInAllLTO { + /// No LTO/ThinLTO behavior needed. + None, + /// ThinLTO prelink (summary) phase. + ThinLTOPreLink, + /// ThinLTO postlink (backend compile) phase. + ThinLTOPostLink, + /// Full LTO prelink phase. + LTOPreLink, + /// Full LTO postlink (backend compile) phase. + LTOPostLink +}; + //===----------------------------------------------------------------------===// /// Pass interface - Implemented by all 'passes'. Subclass this if you are an /// interprocedural optimization or you do not fit into any of the more Index: llvm/include/llvm/Passes/PassBuilder.h =================================================================== --- llvm/include/llvm/Passes/PassBuilder.h +++ llvm/include/llvm/Passes/PassBuilder.h @@ -159,18 +159,6 @@ std::vector InnerPipeline; }; - /// ThinLTO phase. - /// - /// This enumerates the LLVM ThinLTO optimization phases. - enum class ThinLTOPhase { - /// No ThinLTO behavior needed. - None, - /// ThinLTO prelink (summary) phase. - PreLink, - /// ThinLTO postlink (backend compile) phase. - PostLink - }; - /// LLVM-provided high-level optimization levels. /// /// This enumerates the LLVM-provided high-level optimization levels. Each @@ -342,7 +330,7 @@ /// \p Phase indicates the current ThinLTO phase. FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase); + PhaseInAllLTO Phase); /// Construct the core LLVM module canonicalization and simplification /// pipeline. @@ -360,12 +348,12 @@ /// /// \p Phase indicates the current ThinLTO phase. ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase); + PhaseInAllLTO Phase); /// Construct the module pipeline that performs inlining as well as /// the inlining-driven cleanups. ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, - ThinLTOPhase Phase, + PhaseInAllLTO Phase, bool MandatoryOnly); /// Construct the core LLVM module optimization pipeline. @@ -718,7 +706,7 @@ // O1 pass pipeline FunctionPassManager buildO1FunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase); + PhaseInAllLTO Phase); void addRequiredLTOPreLinkPasses(ModulePassManager &MPM); Index: llvm/include/llvm/Transforms/IPO/SampleProfile.h =================================================================== --- llvm/include/llvm/Transforms/IPO/SampleProfile.h +++ llvm/include/llvm/Transforms/IPO/SampleProfile.h @@ -25,16 +25,16 @@ class SampleProfileLoaderPass : public PassInfoMixin { public: SampleProfileLoaderPass(std::string File = "", std::string RemappingFile = "", - bool IsThinLTOPreLink = false) + PhaseInAllLTO LTOPhase = PhaseInAllLTO::None) : ProfileFileName(File), ProfileRemappingFileName(RemappingFile), - IsThinLTOPreLink(IsThinLTOPreLink) {} + LTOPhase(LTOPhase) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; std::string ProfileRemappingFileName; - bool IsThinLTOPreLink; + PhaseInAllLTO LTOPhase; }; } // end namespace llvm Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -522,7 +522,7 @@ // TODO: Investigate the cost/benefit of tail call elimination on debugging. FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase) { + PhaseInAllLTO Phase) { FunctionPassManager FPM(DebugLogging); @@ -584,7 +584,7 @@ // inaccurate. The normal unroller doesn't pay attention to forced full unroll // attributes so we need to make sure and allow the full unroll pass to pay // attention to it. - if (Phase != ThinLTOPhase::PreLink || !PGOOpt || + if (Phase != PhaseInAllLTO::ThinLTOPreLink || !PGOOpt || PGOOpt->Action != PGOOptions::SampleUse) LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), /* OnlyWhenForced= */ !PTO.LoopUnrolling, @@ -648,7 +648,7 @@ FunctionPassManager PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase) { + PhaseInAllLTO Phase) { assert(Level != OptimizationLevel::O0 && "Must request optimizations!"); // The O1 pipeline has a separate pipeline creation function to simplify @@ -745,7 +745,7 @@ // inaccurate. The normal unroller doesn't pay attention to forced full unroll // attributes so we need to make sure and allow the full unroll pass to pay // attention to it. - if (Phase != ThinLTOPhase::PreLink || !PGOOpt || + if (Phase != PhaseInAllLTO::ThinLTOPreLink || !PGOOpt || PGOOpt->Action != PGOOptions::SampleUse) LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), /* OnlyWhenForced= */ !PTO.LoopUnrolling, @@ -932,10 +932,10 @@ } ModuleInlinerWrapperPass -PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase, +PassBuilder::buildInlinerPipeline(OptimizationLevel Level, PhaseInAllLTO Phase, bool MandatoryOnly) { InlineParams IP = getInlineParamsFromOptLevel(Level); - if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt && + if (Phase == PhaseInAllLTO::ThinLTOPreLink && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) IP.HotCallSiteThreshold = 0; @@ -1001,7 +1001,7 @@ ModulePassManager PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase) { + PhaseInAllLTO Phase) { ModulePassManager MPM(DebugLogging); // Add UniqueInternalLinkageNames Pass which renames internal linkage @@ -1012,7 +1012,7 @@ // Place pseudo probe instrumentation as the first pass of the pipeline to // minimize the impact of optimization changes. if (PGOOpt && PGOOpt->PseudoProbeForProfiling && - Phase != ThinLTOPhase::PostLink) + Phase != PhaseInAllLTO::ThinLTOPostLink) MPM.addPass(SampleProfileProbePass(TM)); bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse); @@ -1022,7 +1022,7 @@ // no need to load the profile again in PostLink. bool LoadSampleProfile = HasSampleProfile && - !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink); + !(FlattenedProfileUsed && Phase == PhaseInAllLTO::ThinLTOPostLink); // During the ThinLTO backend phase we perform early indirect call promotion // here, before globalopt. Otherwise imported available_externally functions @@ -1038,7 +1038,7 @@ // command line. E.g. for flattened profiles where we will not be reloading // the sample profile in the ThinLTO backend, we ideally shouldn't have to // provide the sample profile file. - if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile) + if (Phase == PhaseInAllLTO::ThinLTOPostLink && !LoadSampleProfile) MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile)); // Do basic inference of function attributes from known properties of system @@ -1071,20 +1071,19 @@ // Annotate sample profile right after early FPM to ensure freshness of // the debug info. MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, - PGOOpt->ProfileRemappingFile, - Phase == ThinLTOPhase::PreLink)); + PGOOpt->ProfileRemappingFile, Phase)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard // for the profile annotation to be accurate in the ThinLTO backend. - if (Phase != ThinLTOPhase::PreLink) + if (Phase != PhaseInAllLTO::ThinLTOPreLink) // We perform early indirect call promotion here, before globalopt. // This is important for the ThinLTO backend phase because otherwise // imported available_externally functions look unreferenced and are // removed. - MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink, - true /* SamplePGO */)); + MPM.addPass(PGOIndirectCallPromotion( + Phase == PhaseInAllLTO::ThinLTOPostLink, true /* SamplePGO */)); } if (AttributorRun & AttributorRunOption::MODULE) @@ -1093,7 +1092,7 @@ // Lower type metadata and the type.test intrinsic in the ThinLTO // post link pipeline after ICP. This is to enable usage of the type // tests in ICP sequences. - if (Phase == ThinLTOPhase::PostLink) + if (Phase == PhaseInAllLTO::ThinLTOPostLink) MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); for (auto &C : PipelineEarlySimplificationEPCallbacks) @@ -1133,7 +1132,7 @@ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); // Add all the requested passes for instrumentation PGO, if requested. - if (PGOOpt && Phase != ThinLTOPhase::PostLink && + if (PGOOpt && Phase != PhaseInAllLTO::ThinLTOPostLink && (PGOOpt->Action == PGOOptions::IRInstr || PGOOpt->Action == PGOOptions::IRUse)) { addPGOInstrPasses(MPM, Level, @@ -1142,7 +1141,7 @@ PGOOpt->ProfileRemappingFile); MPM.addPass(PGOIndirectCallPromotion(false, false)); } - if (PGOOpt && Phase != ThinLTOPhase::PostLink && + if (PGOOpt && Phase != PhaseInAllLTO::ThinLTOPostLink && PGOOpt->CSAction == PGOOptions::CSIRInstr) MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile)); @@ -1154,7 +1153,7 @@ MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/true)); MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/false)); - if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) { + if (EnableMemProfiler && Phase != PhaseInAllLTO::ThinLTOPreLink) { MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); MPM.addPass(ModuleMemProfilerPass()); } @@ -1400,7 +1399,8 @@ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None)); + MPM.addPass(buildModuleSimplificationPipeline( + Level, LTOPreLink ? PhaseInAllLTO::LTOPreLink : PhaseInAllLTO::None)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink)); @@ -1437,7 +1437,8 @@ // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we // can. - MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink)); + MPM.addPass( + buildModuleSimplificationPipeline(Level, PhaseInAllLTO::ThinLTOPreLink)); // Run partial inlining pass to partially inline functions that have // large bodies. @@ -1500,7 +1501,8 @@ MPM.addPass(ForceFunctionAttrsPass()); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink)); + MPM.addPass( + buildModuleSimplificationPipeline(Level, PhaseInAllLTO::ThinLTOPostLink)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level)); @@ -1547,7 +1549,7 @@ // Load sample profile before running the LTO optimization pipeline. MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, - false /* ThinLTOPhase::PreLink */)); + PhaseInAllLTO::LTOPostLink)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); Index: llvm/lib/Passes/PassRegistry.def =================================================================== --- llvm/lib/Passes/PassRegistry.def +++ llvm/lib/Passes/PassRegistry.def @@ -100,7 +100,7 @@ MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass()) MODULE_PASS("sample-profile", SampleProfileLoaderPass()) MODULE_PASS("scc-oz-module-inliner", - buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None, + buildInlinerPipeline(OptimizationLevel::Oz, PhaseInAllLTO::None, /*MandatoryOnly=*/false)) MODULE_PASS("loop-extract-single", LoopExtractorPass(1)) MODULE_PASS("oz-module-optimizer", Index: llvm/lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- llvm/lib/Transforms/IPO/SampleProfile.cpp +++ llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -321,15 +321,14 @@ class SampleProfileLoader { public: SampleProfileLoader( - StringRef Name, StringRef RemapName, bool IsThinLTOPreLink, + StringRef Name, StringRef RemapName, PhaseInAllLTO LTOPhase, std::function GetAssumptionCache, std::function GetTargetTransformInfo, std::function GetTLI) : GetAC(std::move(GetAssumptionCache)), GetTTI(std::move(GetTargetTransformInfo)), GetTLI(std::move(GetTLI)), CoverageTracker(*this), Filename(std::string(Name)), - RemappingFilename(std::string(RemapName)), - IsThinLTOPreLink(IsThinLTOPreLink) {} + RemappingFilename(std::string(RemapName)), LTOPhase(LTOPhase) {} bool doInitialization(Module &M, FunctionAnalysisManager *FAM = nullptr); bool runOnModule(Module &M, ModuleAnalysisManager *AM, @@ -448,11 +447,12 @@ /// Flag indicating whether input profile is context-sensitive bool ProfileIsCS = false; - /// Flag indicating if the pass is invoked in ThinLTO compile phase. + /// Flag indicating which LTO/ThinLTO phase the pass is invoked in. /// - /// In this phase, in annotation, we should not promote indirect calls. - /// Instead, we will mark GUIDs that needs to be annotated to the function. - bool IsThinLTOPreLink; + /// We need to know the LTO phase because for example in ThinLTOPrelink + /// phase, in annotation, we should not promote indirect calls. Instead, + /// we will mark GUIDs that needs to be annotated to the function. + PhaseInAllLTO LTOPhase; /// Profile Summary Info computed from sample profile. ProfileSummaryInfo *PSI = nullptr; @@ -506,9 +506,9 @@ static char ID; SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile, - bool IsThinLTOPreLink = false) + PhaseInAllLTO LTOPhase = PhaseInAllLTO::None) : ModulePass(ID), SampleLoader( - Name, SampleProfileRemappingFile, IsThinLTOPreLink, + Name, SampleProfileRemappingFile, LTOPhase, [&](Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); }, @@ -1120,7 +1120,7 @@ continue; uint64_t Sum; for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) { - if (IsThinLTOPreLink) { + if (LTOPhase == PhaseInAllLTO::ThinLTOPreLink) { FS->findInlinedFunctions(InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); continue; @@ -1173,7 +1173,7 @@ LocalChanged = true; ++NumCSInlined; } - } else if (IsThinLTOPreLink) { + } else if (LTOPhase == PhaseInAllLTO::ThinLTOPreLink) { findCalleeFunctionSamples(*I)->findInlinedFunctions( InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); } @@ -2151,7 +2151,7 @@ ProfileFileName.empty() ? SampleProfileFile : ProfileFileName, ProfileRemappingFileName.empty() ? SampleProfileRemappingFile : ProfileRemappingFileName, - IsThinLTOPreLink, GetAssumptionCache, GetTTI, GetTLI); + LTOPhase, GetAssumptionCache, GetTTI, GetTLI); if (!SampleLoader.doInitialization(M, &FAM)) return PreservedAnalyses::all();