diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -199,7 +199,7 @@ /// inlining should not be attempted. Optional shouldInline(CallBase &CB, function_ref GetInlineCost, - OptimizationRemarkEmitter &ORE); + OptimizationRemarkEmitter &ORE, bool EnableDeferral); /// Emit ORE message. void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -182,6 +182,9 @@ /// Compute inline cost even when the cost has exceeded the threshold. Optional ComputeFullInlineCost; + + /// Indicate whether we should allow inline deferral. + Optional EnableDeferral = true; }; /// Generate the parameters to tune the inline cost analysis based only on the diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -118,7 +118,9 @@ return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI, GetBFI, PSI, RemarksEnabled ? &ORE : nullptr); }; - auto OIC = llvm::shouldInline(CB, GetInlineCost, ORE); + auto OIC = llvm::shouldInline(CB, GetInlineCost, ORE, + Params.EnableDeferral.hasValue() && + Params.EnableDeferral.getValue()); return std::make_unique(this, CB, OIC, ORE); } @@ -298,7 +300,7 @@ Optional llvm::shouldInline(CallBase &CB, function_ref GetInlineCost, - OptimizationRemarkEmitter &ORE) { + OptimizationRemarkEmitter &ORE, bool EnableDeferral) { using namespace ore; InlineCost IC = GetInlineCost(CB); @@ -335,7 +337,8 @@ } int TotalSecondaryCost = 0; - if (shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) { + if (EnableDeferral && + shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) { LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB << " Cost = " << IC.getCost() << ", outer Cost = " << TotalSecondaryCost << '\n'); diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -2250,7 +2250,8 @@ /*HotCallSiteThreshold*/ {}, /*LocallyHotCallSiteThreshold*/ {}, /*ColdCallSiteThreshold*/ {}, - /* ComputeFullInlineCost*/ true}; + /*ComputeFullInlineCost*/ true, + /*EnableDeferral*/ true}; InlineCostCallAnalyzer CA(*Call.getCalledFunction(), Call, Params, CalleeTTI, GetAssumptionCache, GetBFI, PSI, ORE, true, diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -254,6 +254,12 @@ "enable-npm-call-graph-profile", cl::init(true), cl::Hidden, cl::desc("Enable call graph profile pass for the new PM (default = on)")); +/// Flag to enable inline deferral during PGO. +static cl::opt + EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true), + cl::Hidden, + cl::desc("Enable inline deferral during PGO")); + PipelineTuningOptions::PipelineTuningOptions() { LoopInterleaving = true; LoopVectorization = true; @@ -712,6 +718,9 @@ PGOOpt->Action == PGOOptions::SampleUse) IP.HotCallSiteThreshold = 0; + if (PGOOpt) + IP.EnableDeferral = EnablePGOInlineDeferral; + ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor, MaxDevirtIterations); diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -438,7 +438,7 @@ // just become a regular analysis dependency. OptimizationRemarkEmitter ORE(Caller); - auto OIC = shouldInline(CB, GetInlineCost, ORE); + auto OIC = shouldInline(CB, GetInlineCost, ORE, true); // If the policy determines that we should inline this function, // delete the call instead. if (!OIC)