[CSSPGO] Switches to disable pre-link inlining

[hoy]

Summary

Adding switches to disable pre-link FDO inlining and cgscc inlining so that we can a chance to collect cross-module inline candidate's GUID to help cross module importing.

[hoy]

The best of csspgo thinlto performance comes from disabling prelink sample inlining and cgscc inlining. This gives a more centralized global inlining. This still doesn't achieve the fulllto level inlining. We will rely on the llvm-profgen preinliner to improve that.

[hoy]

Sending out this patch for @wmi to experiment with csspgo for now. Some work may be needed to finalize if we'd like to have it landed.

[wenlei]

fyi, we tried to upstream this, and there were some discussions: https://reviews.llvm.org/D96197

[wmi]

Thanks Hongtao, I will try it.

llvm/include/llvm/Transforms/IPO/Inliner.h

 /// a given module. An InlineAdvisor is configured and kept alive for the
 /// duration of the ModuleInlinerWrapperPass::run.
 class ModuleInlinerWrapperPass
     : public PassInfoMixin<ModuleInlinerWrapperPass> {
 public:
   ModuleInlinerWrapperPass(
       InlineParams Params = getInlineParams(), bool MandatoryFirst = true,
       InliningAdvisorMode Mode = InliningAdvisorMode::Default,
-      unsigned MaxDevirtIterations = 0);
+      unsigned MaxDevirtIterations = 0, bool skipInline = false);
   ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default;
 
   PreservedAnalyses run(Module &, ModuleAnalysisManager &);
 
   /// Allow adding more CGSCC passes, besides inlining. This should be called
   /// before run is called, as part of pass pipeline building.
   CGSCCPassManager &getPM() { return PM; }
 

llvm/lib/Passes/PassBuilder.cpp

 static const Regex DefaultAliasRegex(
     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
 
 /// Flag to enable inline deferral during PGO.
 static cl::opt<bool>
     EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
                             cl::Hidden,
                             cl::desc("Enable inline deferral during PGO"));
+static cl::opt<bool> EnableCGSCCInline("enable-cgscc-inline", cl::init(true),
+                                       cl::Hidden,
+                                       cl::desc("Enable CGSCC Inlining"));
 
 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
                                        cl::Hidden, cl::ZeroOrMore,
                                        cl::desc("Enable memory profiler"));
 
 static cl::opt<bool> PerformMandatoryInliningsFirst(
     "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
     cl::desc("Perform mandatory inlinings module-wide, before performing "
PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
   if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
       PGOOpt->Action == PGOOptions::SampleUse)
     IP.HotCallSiteThreshold = 0;
 
   if (PGOOpt)
     IP.EnableDeferral = EnablePGOInlineDeferral;
 
   ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
-                                UseInlineAdvisor, MaxDevirtIterations);
+                                UseInlineAdvisor, MaxDevirtIterations,
+                                !EnableCGSCCInline);
 
   // Require the GlobalsAA analysis for the module so we can query it within
   // the CGSCC pipeline.
   MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());
   // Invalidate AAManager so it can be recreated and pick up the newly available
   // GlobalsAA.
   MIWP.addModulePass(
       createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
 
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
 
   // Note: historically, the PruneEH pass was run first to deduce nounwind and
   // generally clean up exception handling overhead. It isn't clear this is
   // valuable as the inliner doesn't currently care whether it is inlining an
   // invoke or a call.
   // Run the inliner now.
-  MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level)));
+  MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level), true,
+                                       InliningAdvisorMode::Default, 0,
+                                       !EnableCGSCCInline));
 
   // Optimize globals again after we ran the inliner.
   MPM.addPass(GlobalOptPass());
 
   // Garbage collect dead functions.
   // FIXME: Add ArgumentPromotion pass after once it's ported.
   MPM.addPass(GlobalDCEPass());
 

llvm/lib/Transforms/IPO/Inliner.cpp

PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
   PreservedAnalyses PA;
   PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
   return PA;
 }
 
 ModuleInlinerWrapperPass::ModuleInlinerWrapperPass(InlineParams Params,
                                                    bool MandatoryFirst,
                                                    InliningAdvisorMode Mode,
-                                                   unsigned MaxDevirtIterations)
+                                                   unsigned MaxDevirtIterations,
+                                                   bool skipInline)
     : Params(Params), Mode(Mode), MaxDevirtIterations(MaxDevirtIterations),
       PM(), MPM() {
   // Run the inliner first. The theory is that we are walking bottom-up and so
   // the callees have already been fully optimized, and we want to inline them
   // into the callers so that our optimizations can reflect that.
   // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
   // because it makes profile annotation in the backend inaccurate.
   if (MandatoryFirst)
     PM.addPass(InlinerPass(/*OnlyMandatory*/ true));
+  if (!skipInline)
     PM.addPass(InlinerPass());
 }
 
 PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
                                                 ModuleAnalysisManager &MAM) {
   auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
   if (!IAA.tryCreate(Params, Mode, CGSCCInlineReplayFile)) {
     M.getContext().emitError(
         "Could not setup Inlining Advisor for the requested "

llvm/lib/Transforms/IPO/SampleProfile.cpp

cl::desc(
         "call promotion in proirity-based sample profile loader inlining."));
 
 static cl::opt<bool> CallsitePrioritizedInline(
     "sample-profile-prioritized-inline", cl::Hidden, cl::ZeroOrMore,
     cl::init(false),
     cl::desc("Use call site prioritized inlining for sample profile loader."
              "Currently only CSSPGO is supported."));
 
+static cl::opt<bool> EnableSampleProfileInline(
+    "enable-sample-profile-inline", cl::Hidden, cl::init(true),
+    cl::desc("Enable Inlining from sample profile loader."));
+
 static cl::opt<std::string> ProfileInlineReplayFile(
     "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"),
     cl::desc(
         "Optimization remarks file containing inline remarks to be replayed "
         "by inlining from sample profile loader."),
     cl::Hidden);
 
 static cl::opt<unsigned>
if (ProfileMergeInlinee) {
       pair.first->second.entryCount += FS->getEntrySamples();
     }
   }
   return Changed;
 }
 
 bool SampleProfileLoader::tryInlineCandidate(
     InlineCandidate &Candidate, SmallVector<CallBase *, 8> *InlinedCallSites) {
+  // Disable only the inlining part of sample profile loader, and still
+  // perform profile annotation. This can be useful for pre-LTO pipeline
+  // tuning. Note that to preserve context profile from previous inline,
+  // ProfileMergeInlinee and ProfileTopDownLoad will also be needed.
+  if (!EnableSampleProfileInline)
+    return false;
 
   CallBase &CB = *Candidate.CallInstr;
   Function *CalledFunction = CB.getCalledFunction();
   assert(CalledFunction && "Expect a callee with definition");
   DebugLoc DLoc = CB.getDebugLoc();
   BasicBlock *BB = CB.getParent();
 
   InlineCost Cost = shouldInlineCandidate(Candidate);
bool SampleProfileLoader::inlineHotFunctionsWithPriority(
   // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
   // Profile symbol list is ignored when profile-sample-accurate is on.
   assert((!ProfAccForSymsInList ||
           (!ProfileSampleAccurate &&
            !F.hasFnAttribute("profile-sample-accurate"))) &&
          "ProfAccForSymsInList should be false when profile-sample-accurate "
          "is enabled");
 
+  // Disable only the inlining part of sample profile loader, and still
+  // perform profile annotation. This can be useful for pre-LTO pipeline
+  // tuning. Note that to preserve context profile from previous inline,
+  // ProfileMergeInlinee and ProfileTopDownLoad will also be needed.
+  // For ThinLTO PreLink, we needed to collect InlinedGUIDs, so don't
+  // bail out just yet.
+  if (!EnableSampleProfileInline &&
+      !(LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink))
+    return false;
+
   // Populating worklist with initial call sites from root inliner, along
   // with call site weights.
   CandidateQueue CQueue;
   InlineCandidate NewCandidate;
   for (auto &BB : F) {
     for (auto &I : BB.getInstList()) {
       auto *CB = dyn_cast<CallBase>(&I);
       if (!CB)