diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h --- a/llvm/include/llvm/Pass.h +++ b/llvm/include/llvm/Pass.h @@ -69,6 +69,20 @@ PT_PassManager }; +/// This enumerates the LLVM full LTO or ThinLTO optimization phases. +enum class ThinOrFullLTOPhase { + /// No LTO/ThinLTO behavior needed. + None, + /// ThinLTO prelink (summary) phase. + ThinLTOPreLink, + /// ThinLTO postlink (backend compile) phase. + ThinLTOPostLink, + /// Full LTO prelink phase. + FullLTOPreLink, + /// Full LTO postlink (backend compile) phase. + FullLTOPostLink +}; + //===----------------------------------------------------------------------===// /// Pass interface - Implemented by all 'passes'. Subclass this if you are an /// interprocedural optimization or you do not fit into any of the more diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -159,18 +159,6 @@ std::vector InnerPipeline; }; - /// ThinLTO phase. - /// - /// This enumerates the LLVM ThinLTO optimization phases. - enum class ThinLTOPhase { - /// No ThinLTO behavior needed. - None, - /// ThinLTO prelink (summary) phase. - PreLink, - /// ThinLTO postlink (backend compile) phase. - PostLink - }; - /// LLVM-provided high-level optimization levels. /// /// This enumerates the LLVM-provided high-level optimization levels. Each @@ -342,7 +330,7 @@ /// \p Phase indicates the current ThinLTO phase. FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase); + ThinOrFullLTOPhase Phase); /// Construct the core LLVM module canonicalization and simplification /// pipeline. @@ -360,12 +348,12 @@ /// /// \p Phase indicates the current ThinLTO phase. ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase); + ThinOrFullLTOPhase Phase); /// Construct the module pipeline that performs inlining as well as /// the inlining-driven cleanups. ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, - ThinLTOPhase Phase, + ThinOrFullLTOPhase Phase, bool MandatoryOnly); /// Construct the core LLVM module optimization pipeline. @@ -718,7 +706,7 @@ // O1 pass pipeline FunctionPassManager buildO1FunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase); + ThinOrFullLTOPhase Phase); void addRequiredLTOPreLinkPasses(ModulePassManager &MPM); diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfile.h b/llvm/include/llvm/Transforms/IPO/SampleProfile.h --- a/llvm/include/llvm/Transforms/IPO/SampleProfile.h +++ b/llvm/include/llvm/Transforms/IPO/SampleProfile.h @@ -24,17 +24,18 @@ /// The sample profiler data loader pass. class SampleProfileLoaderPass : public PassInfoMixin { public: - SampleProfileLoaderPass(std::string File = "", std::string RemappingFile = "", - bool IsThinLTOPreLink = false) + SampleProfileLoaderPass( + std::string File = "", std::string RemappingFile = "", + ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None) : ProfileFileName(File), ProfileRemappingFileName(RemappingFile), - IsThinLTOPreLink(IsThinLTOPreLink) {} + LTOPhase(LTOPhase) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; std::string ProfileRemappingFileName; - bool IsThinLTOPreLink; + ThinOrFullLTOPhase LTOPhase; }; } // end namespace llvm diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -514,7 +514,7 @@ // TODO: Investigate the cost/benefit of tail call elimination on debugging. FunctionPassManager PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase) { + ThinOrFullLTOPhase Phase) { FunctionPassManager FPM(DebugLogging); @@ -576,7 +576,7 @@ // inaccurate. The normal unroller doesn't pay attention to forced full unroll // attributes so we need to make sure and allow the full unroll pass to pay // attention to it. - if (Phase != ThinLTOPhase::PreLink || !PGOOpt || + if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt || PGOOpt->Action != PGOOptions::SampleUse) LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), /* OnlyWhenForced= */ !PTO.LoopUnrolling, @@ -640,7 +640,7 @@ FunctionPassManager PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase) { + ThinOrFullLTOPhase Phase) { assert(Level != OptimizationLevel::O0 && "Must request optimizations!"); // The O1 pipeline has a separate pipeline creation function to simplify @@ -738,7 +738,7 @@ // inaccurate. The normal unroller doesn't pay attention to forced full unroll // attributes so we need to make sure and allow the full unroll pass to pay // attention to it. - if (Phase != ThinLTOPhase::PreLink || !PGOOpt || + if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt || PGOOpt->Action != PGOOptions::SampleUse) LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(), /* OnlyWhenForced= */ !PTO.LoopUnrolling, @@ -924,11 +924,10 @@ return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel()); } -ModuleInlinerWrapperPass -PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase, - bool MandatoryOnly) { +ModuleInlinerWrapperPass PassBuilder::buildInlinerPipeline( + OptimizationLevel Level, ThinOrFullLTOPhase Phase, bool MandatoryOnly) { InlineParams IP = getInlineParamsFromOptLevel(Level); - if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt && + if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) IP.HotCallSiteThreshold = 0; @@ -994,7 +993,7 @@ ModulePassManager PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, - ThinLTOPhase Phase) { + ThinOrFullLTOPhase Phase) { ModulePassManager MPM(DebugLogging); // Add UniqueInternalLinkageNames Pass which renames internal linkage @@ -1005,7 +1004,7 @@ // Place pseudo probe instrumentation as the first pass of the pipeline to // minimize the impact of optimization changes. if (PGOOpt && PGOOpt->PseudoProbeForProfiling && - Phase != ThinLTOPhase::PostLink) + Phase != ThinOrFullLTOPhase::ThinLTOPostLink) MPM.addPass(SampleProfileProbePass(TM)); bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse); @@ -1015,7 +1014,7 @@ // no need to load the profile again in PostLink. bool LoadSampleProfile = HasSampleProfile && - !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink); + !(FlattenedProfileUsed && Phase == ThinOrFullLTOPhase::ThinLTOPostLink); // During the ThinLTO backend phase we perform early indirect call promotion // here, before globalopt. Otherwise imported available_externally functions @@ -1031,7 +1030,7 @@ // command line. E.g. for flattened profiles where we will not be reloading // the sample profile in the ThinLTO backend, we ideally shouldn't have to // provide the sample profile file. - if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile) + if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile) MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile)); // Do basic inference of function attributes from known properties of system @@ -1064,20 +1063,19 @@ // Annotate sample profile right after early FPM to ensure freshness of // the debug info. MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, - PGOOpt->ProfileRemappingFile, - Phase == ThinLTOPhase::PreLink)); + PGOOpt->ProfileRemappingFile, Phase)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard // for the profile annotation to be accurate in the ThinLTO backend. - if (Phase != ThinLTOPhase::PreLink) + if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink) // We perform early indirect call promotion here, before globalopt. // This is important for the ThinLTO backend phase because otherwise // imported available_externally functions look unreferenced and are // removed. - MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink, - true /* SamplePGO */)); + MPM.addPass(PGOIndirectCallPromotion( + Phase == ThinOrFullLTOPhase::ThinLTOPostLink, true /* SamplePGO */)); } if (AttributorRun & AttributorRunOption::MODULE) @@ -1086,7 +1084,7 @@ // Lower type metadata and the type.test intrinsic in the ThinLTO // post link pipeline after ICP. This is to enable usage of the type // tests in ICP sequences. - if (Phase == ThinLTOPhase::PostLink) + if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink) MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); for (auto &C : PipelineEarlySimplificationEPCallbacks) @@ -1126,7 +1124,7 @@ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); // Add all the requested passes for instrumentation PGO, if requested. - if (PGOOpt && Phase != ThinLTOPhase::PostLink && + if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink && (PGOOpt->Action == PGOOptions::IRInstr || PGOOpt->Action == PGOOptions::IRUse)) { addPGOInstrPasses(MPM, Level, @@ -1135,7 +1133,7 @@ PGOOpt->ProfileRemappingFile); MPM.addPass(PGOIndirectCallPromotion(false, false)); } - if (PGOOpt && Phase != ThinLTOPhase::PostLink && + if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink && PGOOpt->CSAction == PGOOptions::CSIRInstr) MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile)); @@ -1147,7 +1145,7 @@ MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/true)); MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/false)); - if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) { + if (EnableMemProfiler && Phase != ThinOrFullLTOPhase::ThinLTOPreLink) { MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); MPM.addPass(ModuleMemProfilerPass()); } @@ -1393,7 +1391,9 @@ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None)); + MPM.addPass(buildModuleSimplificationPipeline( + Level, LTOPreLink ? ThinOrFullLTOPhase::FullLTOPreLink + : ThinOrFullLTOPhase::None)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink)); @@ -1430,7 +1430,8 @@ // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we // can. - MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink)); + MPM.addPass(buildModuleSimplificationPipeline( + Level, ThinOrFullLTOPhase::ThinLTOPreLink)); // Run partial inlining pass to partially inline functions that have // large bodies. @@ -1493,7 +1494,8 @@ MPM.addPass(ForceFunctionAttrsPass()); // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink)); + MPM.addPass(buildModuleSimplificationPipeline( + Level, ThinOrFullLTOPhase::ThinLTOPostLink)); // Now add the optimization pipeline. MPM.addPass(buildModuleOptimizationPipeline(Level)); @@ -1540,7 +1542,7 @@ // Load sample profile before running the LTO optimization pipeline. MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, - false /* ThinLTOPhase::PreLink */)); + ThinOrFullLTOPhase::FullLTOPostLink)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -98,7 +98,7 @@ MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass()) MODULE_PASS("sample-profile", SampleProfileLoaderPass()) MODULE_PASS("scc-oz-module-inliner", - buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None, + buildInlinerPipeline(OptimizationLevel::Oz, ThinOrFullLTOPhase::None, /*MandatoryOnly=*/false)) MODULE_PASS("loop-extract-single", LoopExtractorPass(1)) MODULE_PASS("oz-module-optimizer", diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -321,15 +321,14 @@ class SampleProfileLoader { public: SampleProfileLoader( - StringRef Name, StringRef RemapName, bool IsThinLTOPreLink, + StringRef Name, StringRef RemapName, ThinOrFullLTOPhase LTOPhase, std::function GetAssumptionCache, std::function GetTargetTransformInfo, std::function GetTLI) : GetAC(std::move(GetAssumptionCache)), GetTTI(std::move(GetTargetTransformInfo)), GetTLI(std::move(GetTLI)), CoverageTracker(*this), Filename(std::string(Name)), - RemappingFilename(std::string(RemapName)), - IsThinLTOPreLink(IsThinLTOPreLink) {} + RemappingFilename(std::string(RemapName)), LTOPhase(LTOPhase) {} bool doInitialization(Module &M, FunctionAnalysisManager *FAM = nullptr); bool runOnModule(Module &M, ModuleAnalysisManager *AM, @@ -448,11 +447,12 @@ /// Flag indicating whether input profile is context-sensitive bool ProfileIsCS = false; - /// Flag indicating if the pass is invoked in ThinLTO compile phase. + /// Flag indicating which LTO/ThinLTO phase the pass is invoked in. /// - /// In this phase, in annotation, we should not promote indirect calls. - /// Instead, we will mark GUIDs that needs to be annotated to the function. - bool IsThinLTOPreLink; + /// We need to know the LTO phase because for example in ThinLTOPrelink + /// phase, in annotation, we should not promote indirect calls. Instead, + /// we will mark GUIDs that needs to be annotated to the function. + ThinOrFullLTOPhase LTOPhase; /// Profile Summary Info computed from sample profile. ProfileSummaryInfo *PSI = nullptr; @@ -505,10 +505,11 @@ // Class identification, replacement for typeinfo static char ID; - SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile, - bool IsThinLTOPreLink = false) + SampleProfileLoaderLegacyPass( + StringRef Name = SampleProfileFile, + ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None) : ModulePass(ID), SampleLoader( - Name, SampleProfileRemappingFile, IsThinLTOPreLink, + Name, SampleProfileRemappingFile, LTOPhase, [&](Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); }, @@ -1120,7 +1121,7 @@ continue; uint64_t Sum; for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) { - if (IsThinLTOPreLink) { + if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { FS->findInlinedFunctions(InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); continue; @@ -1173,7 +1174,7 @@ LocalChanged = true; ++NumCSInlined; } - } else if (IsThinLTOPreLink) { + } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { findCalleeFunctionSamples(*I)->findInlinedFunctions( InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); } @@ -2151,7 +2152,7 @@ ProfileFileName.empty() ? SampleProfileFile : ProfileFileName, ProfileRemappingFileName.empty() ? SampleProfileRemappingFile : ProfileRemappingFileName, - IsThinLTOPreLink, GetAssumptionCache, GetTTI, GetTLI); + LTOPhase, GetAssumptionCache, GetTTI, GetTLI); if (!SampleLoader.doInitialization(M, &FAM)) return PreservedAnalyses::all(); diff --git a/polly/lib/Support/RegisterPasses.cpp b/polly/lib/Support/RegisterPasses.cpp --- a/polly/lib/Support/RegisterPasses.cpp +++ b/polly/lib/Support/RegisterPasses.cpp @@ -524,7 +524,7 @@ PM.addPass(CodePreparationPass()); PM.addPass(createFunctionToScopPassAdaptor(std::move(SPM))); PM.addPass(PB.buildFunctionSimplificationPipeline( - Level, PassBuilder::ThinLTOPhase::None)); // Cleanup + Level, ThinOrFullLTOPhase::None)); // Cleanup assert(!DumpAfter && "This option is not implemented"); assert(DumpAfterFile.empty() && "This option is not implemented");