[Inliner] Make recusive inlinee stack size limit tunable

[wenlei]

Summary

For recursive callers, we want to be conservative when inlining callees with large stack size. We currently have a limit InlineConstants::TotalAllocaSizeRecursiveCaller, but that is hard coded.

We found the current limit insufficient to suppress problematic inlining that bloats stack size for deep recursion. This change adds a switch to make the limit tunable as a mitigation.

[davidxl]

lgtm

llvm/lib/Analysis/InlineCost.cpp

static cl::opt<int> CallPenalty(
     cl::desc("Call penalty that is applied per callsite when inlining"));
 
 static cl::opt<size_t>
     StackSizeThreshold("inline-max-stacksize", cl::Hidden,
                        cl::init(std::numeric_limits<size_t>::max()),
                        cl::desc("Do not inline functions with a stack size "
                                 "that exceeds the specified limit"));
 
+static cl::opt<size_t>
+    RecurStackSizeThreshold("recursive-inline-max-stacksize", cl::Hidden,
+                       cl::init(InlineConstants::TotalAllocaSizeRecursiveCaller),
+                       cl::desc("Do not inline recursive functions with a stack "
+                                "size that exceeds the specified limit"));
+
 static cl::opt<bool> OptComputeFullInlineCost(
     "inline-cost-full", cl::Hidden,
     cl::desc("Compute the full inline cost of a call site even when the cost "
              "exceeds the threshold."));
 
 static cl::opt<bool> InlineCallerSupersetNoBuiltin(
     "inline-caller-superset-nobuiltin", cl::Hidden, cl::init(true),
     cl::desc("Allow inlining when caller has a superset of callee's nobuiltin "
if (!IR.isSuccess()) {
                  << ") and cost is not fully computed";
         });
       return IR;
     }
 
     // If the caller is a recursive function then we don't want to inline
     // functions which allocate a lot of stack space because it would increase
     // the caller stack usage dramatically.
-    if (IsCallerRecursive &&
-        AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller) {
+    if (IsCallerRecursive && AllocatedSize > RecurStackSizeThreshold) {
       auto IR =
           InlineResult::failure("recursive and allocates too much stack space");
       if (ORE)
         ORE->emit([&]() {
           return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline",
                                           &CandidateCall)
                  << NV("Callee", &F) << " is "
                  << NV("InlineResult", IR.getFailureReason())

llvm/test/Transforms/Inline/inline-recur-stacksize.ll

+; Check the recursive inliner doesn't inline a function with a stack size exceeding a given limit.
+; RUN: opt < %s -inline -S | FileCheck --check-prefixes=ALL,UNLIMITED %s
+; RUN: opt < %s -inline -S -recursive-inline-max-stacksize=256 | FileCheck --check-prefixes=ALL,LIMITED %s
+
+declare void @init([65 x i32]*)
+
+define internal i32 @foo() {
+  %1 = alloca [65 x i32], align 16
+  %2 = getelementptr inbounds [65 x i32], [65 x i32]* %1, i65 0, i65 0
+  call void @init([65 x i32]* %1)
+  %3 = load i32, i32* %2, align 4
+  ret i32 %3
+}
+
+define i32 @bar() {
+  %1 = call i32 @foo()
+  ret i32 %1
+; ALL: define {{.*}}@bar
+; ALL-NOT: define
+; UNLIMITED-NOT: call {{.*}}@foo
+; LIMITED-NOT: call {{.*}}@foo
+}
+
+; Check that, under the tighter limit, baz() doesn't inline foo()
+define i32 @baz() {
+  %1 = call i32 @foo()
+  %2 = call i32 @baz()
+  %3 = add i32 %1, %2
+  ret i32 %3
+; ALL: define {{.*}}@baz
+; ALL-NOT: define
+; UNLIMITED-NOT: call {{.*}}@foo
+; LIMITED: call {{.*}}@foo
+}