diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -213,6 +213,9 @@ /// Indicate whether we should allow inline deferral. Optional EnableDeferral = true; + + /// Indicate whether we allow inlining for recursive call. + Optional AllowRecursiveCall = false; }; /// Generate the parameters to tune the inline cost analysis based only on the diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -333,6 +333,10 @@ /// whenever we simplify away the stores that would otherwise cause them to be /// loads. bool EnableLoadElimination; + + /// Whether we allow inlining for recursive call. + bool AllowRecursiveCall; + SmallPtrSet LoadAddrSet; AllocaInst *getSROAArgForValueOrNull(Value *V) const { @@ -421,7 +425,8 @@ OptimizationRemarkEmitter *ORE = nullptr) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), GetBFI(GetBFI), PSI(PSI), F(Callee), DL(F.getParent()->getDataLayout()), ORE(ORE), - CandidateCall(Call), EnableLoadElimination(true) {} + CandidateCall(Call), EnableLoadElimination(true), + AllowRecursiveCall(false) {} InlineResult analyze(); @@ -930,7 +935,9 @@ Params(Params), Threshold(Params.DefaultThreshold), BoostIndirectCalls(BoostIndirect), IgnoreThreshold(IgnoreThreshold), CostBenefitAnalysisEnabled(isCostBenefitAnalysisEnabled()), - Writer(this) {} + Writer(this) { + AllowRecursiveCall = Params.AllowRecursiveCall.getValue(); + } /// Annotation Writer for instruction details InlineCostAnnotationWriter Writer; @@ -2098,7 +2105,8 @@ // This flag will fully abort the analysis, so don't bother with anything // else. IsRecursiveCall = true; - return false; + if (!AllowRecursiveCall) + return false; } if (TTI.isLoweredToCall(F)) { @@ -2336,7 +2344,7 @@ using namespace ore; // If the visit this instruction detected an uninlinable pattern, abort. InlineResult IR = InlineResult::success(); - if (IsRecursiveCall) + if (IsRecursiveCall && !AllowRecursiveCall) IR = InlineResult::failure("recursive"); else if (ExposesReturnsTwice) IR = InlineResult::failure("exposes returns twice"); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -219,6 +219,11 @@ cl::init(false), cl::desc("Use the preinliner decisions stored in profile context.")); +static cl::opt AllowRecursiveInline( + "sample-profile-recursive-inline", cl::Hidden, cl::ZeroOrMore, + cl::init(false), + cl::desc("Allow sample loader inliner to inline recursive calls.")); + static cl::opt ProfileInlineReplayFile( "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc( @@ -1283,7 +1288,9 @@ assert(Callee && "Expect a definition for inline candidate of direct call"); InlineParams Params = getInlineParams(); + // We will ignore the threshold from inline cost, so always get full cost. Params.ComputeFullInlineCost = true; + Params.AllowRecursiveCall = AllowRecursiveInline; // Checks if there is anything in the reachable portion of the callee at // this callsite that makes this inlining potentially illegal. Need to // set ComputeFullInlineCost, otherwise getInlineCost may return early @@ -1840,6 +1847,10 @@ if (!UsePreInlinerDecision.getNumOccurrences()) UsePreInlinerDecision = true; + // For CSSPGO, we also allow recursive inline to best use context profile. + if (!AllowRecursiveInline.getNumOccurrences()) + AllowRecursiveInline = true; + // Enable iterative-BFI by default for CSSPGO. if (!UseIterativeBFIInference.getNumOccurrences()) UseIterativeBFIInference = true;