diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -93,6 +93,12 @@ DisableInlinedAllocaMerging("disable-inlined-alloca-merging", cl::init(false), cl::Hidden); +static cl::opt DisableNoinlineOnIntraSCCCalls( + "disable-noinline-on-intra-scc-calls", cl::init(false), + cl::desc("Disable marking inlined call sites that have a callee in the " + "same SCC as noinline to prevent exponential compile times " + "(excluding when the original call site was intra-SCC)")); + /// A flag for test, so we can print the content of the advisor when running it /// as part of the default (e.g. -O3) pipeline. static cl::opt KeepAdvisorForPrinting("keep-inline-advisor-for-printing", @@ -877,8 +883,8 @@ // trigger infinite inlining, much like is prevented within the inliner // itself by the InlineHistory above, but spread across CGSCC iterations // and thus hidden from the full inline history. - if (CG.lookupSCC(*CG.lookup(Callee)) == C && - UR.InlinedInternalEdges.count({&N, C})) { + LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(*CG.lookup(Callee)); + if (CalleeSCC == C && UR.InlinedInternalEdges.count({&N, C})) { LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node " "previously split out of this SCC by inlining: " << F.getName() << " -> " << Callee.getName() << "\n"); @@ -936,9 +942,24 @@ if (tryPromoteCall(*ICB)) NewCallee = ICB->getCalledFunction(); } - if (NewCallee) - if (!NewCallee->isDeclaration()) + if (NewCallee) { + if (!NewCallee->isDeclaration()) { Calls->push({ICB, NewHistoryID}); + // Continually inlining through an SCC can result in huge compile + // times and bloated code since we arbitrarily stop at some point + // when the inliner decides it's not profitable to inline + // anymore. We put a stop at the first potential attempt at + // inlining through an SCC by marking the call site as noinline. + // This doesn't apply to calls in the same SCC since if we do + // inline through the SCC the function will end up being + // self-recursive which the inliner bails out on, and inlining + // within an SCC is necessary for performance. + if (!DisableNoinlineOnIntraSCCCalls && CalleeSCC != C && + CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) { + ICB->addFnAttr(Attribute::NoInline); + } + } + } } } diff --git a/llvm/test/Transforms/Inline/mut-rec-scc-2.ll b/llvm/test/Transforms/Inline/mut-rec-scc-2.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/mut-rec-scc-2.ll @@ -0,0 +1,19 @@ +; RUN: opt -S -passes='inline' < %s | FileCheck %s + +; Make sure we don't mark calls within the same SCC as original function with noinline. +; CHECK-NOT: noinline + +define void @samescc1() { + call void @samescc2() + ret void +} + +define void @samescc2() { + call void @samescc3() + ret void +} + +define void @samescc3() { + call void @samescc1() + ret void +} diff --git a/llvm/test/Transforms/Inline/mut-rec-scc.ll b/llvm/test/Transforms/Inline/mut-rec-scc.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/mut-rec-scc.ll @@ -0,0 +1,72 @@ +; RUN: opt -S -passes='cgscc(inline,instcombine)' < %s | FileCheck %s +; RUN: opt -S -disable-noinline-on-intra-scc-calls -passes='cgscc(inline,instcombine)' < %s | FileCheck %s --check-prefix=OFF + +; OFF: @test1 +; OFF-NOT: noinline + +; We use call to a dummy function to avoid inlining test1 into test2 or vice +; versa, such that we aren't left with a trivial cycle, as trivial cycles are +; special-cased to never be inlined. +; However, InstCombine will eliminate these calls after inlining, and thus +; make the functions eligible for inlining in their callers. +declare void @dummy() readnone nounwind willreturn + +define void @test1() { +; CHECK-LABEL: define void @test1( +; CHECK-NEXT: call void @test2() +; CHECK-NEXT: call void @test2() +; CHECK-NEXT: ret void +; + call void @test2() + call void @test2() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + ret void +} + +define void @test2() { +; CHECK-LABEL: define void @test2( +; CHECK-NEXT: call void @test1() +; CHECK-NEXT: call void @test1() +; CHECK-NEXT: ret void +; + call void @test1() + call void @test1() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + call void @dummy() + ret void +} + +; We should inline the @test2 calls and mark the inlined @test1 calls as noinline +define void @test3() { +; CHECK-LABEL: define void @test3( +; CHECK-NEXT: call void @test1() #[[NOINLINE:[0-9]+]] +; CHECK-NEXT: call void @test1() #[[NOINLINE]] +; CHECK-NEXT: call void @test1() #[[NOINLINE]] +; CHECK-NEXT: call void @test1() #[[NOINLINE]] +; CHECK-NEXT: ret void +; + call void @test2() + call void @test2() + ret void +} + +; CHECK: [[NOINLINE]] = { noinline }