Index: include/llvm/IR/CallSite.h =================================================================== --- include/llvm/IR/CallSite.h +++ include/llvm/IR/CallSite.h @@ -31,6 +31,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" namespace llvm { @@ -103,6 +104,15 @@ *getCallee() = V; } + FunTy *getMetaCallDestination() const { + FunTy *LiteralDest = getCalledFunction(); + if (LiteralDest && + LiteralDest->getIntrinsicID() == Intrinsic::experimental_gc_statepoint) + return dyn_cast(getArgument(2)); + + return LiteralDest; + } + /// isCallee - Determine whether the passed iterator points to the /// callee operand's Use. bool isCallee(Value::const_user_iterator UI) const { Index: lib/Analysis/IPA/InlineCost.cpp =================================================================== --- lib/Analysis/IPA/InlineCost.cpp +++ lib/Analysis/IPA/InlineCost.cpp @@ -1315,8 +1315,8 @@ InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee, int Threshold) { - // Cannot inline indirect calls. - if (!Callee) + // Cannot inline indirect calls or calls through intrinsics. + if (!Callee || Callee->isIntrinsic()) return llvm::InlineCost::getNever(); // Calls to functions with always-inline attributes should be inlined Index: lib/Transforms/IPO/InlineAlways.cpp =================================================================== --- lib/Transforms/IPO/InlineAlways.cpp +++ lib/Transforms/IPO/InlineAlways.cpp @@ -93,7 +93,7 @@ /// small functions which have the explicit attribute to force inlining, it is /// likely not worth it in practice. InlineCost AlwaysInliner::getInlineCost(CallSite CS) { - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getMetaCallDestination(); // Only inline direct calls to functions with always-inline attributes // that are viable for inlining. FIXME: We shouldn't even get here for Index: lib/Transforms/IPO/Inliner.cpp =================================================================== --- lib/Transforms/IPO/Inliner.cpp +++ lib/Transforms/IPO/Inliner.cpp @@ -129,12 +129,22 @@ static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime) { - Function *Callee = CS.getCalledFunction(); Function *Caller = CS.getCaller(); + Function *Callee = nullptr; + bool InlineResult = false; + // Try to inline the function. Get the list of static allocas that were // inlined. - if (!InlineFunction(CS, IFI, InsertLifetime)) + if (Statepoint SP = Statepoint(CS)) { + Callee = cast(SP.getActualCallee()); + InlineResult = InlineFunction(SP, IFI, InsertLifetime); + } else { + Callee = CS.getCalledFunction(); + InlineResult = InlineFunction(CS, IFI, InsertLifetime); + } + + if (!InlineResult) return false; AdjustCallerSSPLevel(Caller, Callee); @@ -444,11 +454,29 @@ DEBUG(dbgs() << " " << (F ? F->getName() : "INDIRECTNODE")); } + class InlineTask { + Function *Callee; + Function *Caller; + CallSite CS; + int InlineHistoryID; + + public: + explicit InlineTask(Function *Callee, Function *Caller, CallSite CS, + int InlineHistoryID = -1) + : Callee(Callee), Caller(Caller), CS(CS), + InlineHistoryID(InlineHistoryID) {} + + Function *getCalledFunction() const { return Callee; } + Function *getCaller() const { return Caller; } + CallSite getCallSite() const { return CS; } + int getInlineHistoryID() const { return InlineHistoryID; } + }; + // Scan through and identify all call sites ahead of time so that we only // inline call sites in the original functions, not call sites that result // from inlining other functions. - SmallVector, 16> CallSites; - + SmallVector InlineTaskList; + // When inlining a callee produces new call sites, we want to keep track of // the fact that they were inlined from the callee. This allows us to avoid // infinite inlining in some obscure cases. To represent this, we use an @@ -462,36 +490,41 @@ for (BasicBlock &BB : *F) for (Instruction &I : BB) { CallSite CS(cast(&I)); - // If this isn't a call, or it is a call to an intrinsic, it can - // never be inlined. - if (!CS || isa(I)) + // If this isn't a call never be inlined. + if (!CS) continue; - + + // We can inline through gc_statepoint calls. Calls to other intrinsics + // cannot be inlined. + if (isa(I) && !isStatepoint(I)) + continue; + + Function *CalledFunction = CS.getMetaCallDestination(); + // If this is a direct call to an external function, we can never inline // it. If it is an indirect call, inlining may resolve it to be a // direct call, so we keep it. - if (CS.getCalledFunction() && CS.getCalledFunction()->isDeclaration()) + if (CalledFunction && CalledFunction->isDeclaration()) continue; - - CallSites.push_back(std::make_pair(CS, -1)); + + InlineTaskList.emplace_back(CalledFunction, CS.getCaller(), CS); } } - DEBUG(dbgs() << ": " << CallSites.size() << " call sites.\n"); + DEBUG(dbgs() << ": " << InlineTaskList.size() << " call sites.\n"); // If there are no calls in this function, exit early. - if (CallSites.empty()) + if (InlineTaskList.empty()) return false; // Now that we have all of the call sites, move the ones to functions in the // current SCC to the end of the list. - unsigned FirstCallInSCC = CallSites.size(); + unsigned FirstCallInSCC = InlineTaskList.size(); for (unsigned i = 0; i < FirstCallInSCC; ++i) - if (Function *F = CallSites[i].first.getCalledFunction()) + if (Function *F = InlineTaskList[i].getCalledFunction()) if (SCCFunctions.count(F)) - std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); + std::swap(InlineTaskList[i--], InlineTaskList[--FirstCallInSCC]); - InlinedArrayAllocasTy InlinedArrayAllocas; InlineFunctionInfo InlineInfo(&CG, AA, ACT); @@ -503,12 +536,12 @@ LocalChange = false; // Iterate over the outer loop because inlining functions can cause indirect // calls to become direct calls. - // CallSites may be modified inside so ranged for loop can not be used. - for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) { - CallSite CS = CallSites[CSi].first; - + // InlineTaskList may be modified inside so ranged for loop can not be used. + for (unsigned CSi = 0; CSi != InlineTaskList.size(); ++CSi) { + CallSite CS = InlineTaskList[CSi].getCallSite(); + Function *Caller = CS.getCaller(); - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getMetaCallDestination(); // If this call site is dead and it is to a readonly function, we should // just delete the call instead of trying to inline it, regardless of @@ -530,7 +563,7 @@ // itself. If so, we'd be recursively inlining the same function, // which would provide the same callsites, which would cause us to // infinitely inline. - int InlineHistoryID = CallSites[CSi].second; + int InlineHistoryID = InlineTaskList[CSi].getInlineHistoryID(); if (InlineHistoryID != -1 && InlineHistoryIncludes(Callee, InlineHistoryID, InlineHistory)) continue; @@ -574,8 +607,11 @@ int NewHistoryID = InlineHistory.size(); InlineHistory.push_back(std::make_pair(Callee, InlineHistoryID)); - for (Value *Ptr : InlineInfo.InlinedCalls) - CallSites.push_back(std::make_pair(CallSite(Ptr), NewHistoryID)); + for (Value *Ptr : InlineInfo.InlinedCalls) { + CallSite CS(Ptr); + InlineTaskList.emplace_back(CS.getMetaCallDestination(), + CS.getCaller(), CS, NewHistoryID); + } } } @@ -606,10 +642,10 @@ // move a call site to a function in this SCC before the // 'FirstCallInSCC' barrier. if (SCC.isSingular()) { - CallSites[CSi] = CallSites.back(); - CallSites.pop_back(); + InlineTaskList[CSi] = InlineTaskList.back(); + InlineTaskList.pop_back(); } else { - CallSites.erase(CallSites.begin()+CSi); + InlineTaskList.erase(InlineTaskList.begin() + CSi); } --CSi; Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -162,6 +162,13 @@ SP.gc_transition_args_begin() != SP.gc_transition_args_end()) return false; + Function *Callee = cast(SP.getActualCallee()); + if (!Callee->hasGC()) + return false; + + if (StringRef(Callee->getGC()) != SP.getCallSite().getCaller()->getGC()) + return false; + Value *ReturnValOut = nullptr; bool InlinedMustTailCalls = false; if (!InlineFunctionImpl(InlineSite(SP), IFI, InsertLifetime, ReturnValOut, Index: test/Transforms/Inline/statepoints-basic.ll =================================================================== --- /dev/null +++ test/Transforms/Inline/statepoints-basic.ll @@ -0,0 +1,69 @@ +; RUN: opt -S -always-inline < %s | FileCheck %s + +declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) + +@A = global i32 7 + +declare i32* @fake_personality_function() + +define void @callee() gc "statepoint-example" { + store i32 123, i32* @A + ret void +} + +define void @callee_nogc() { + store i32 123, i32* @A + ret void +} + +define i8 addrspace(1)* @test_inline_call(i8 addrspace(1)* %obj) gc "statepoint-example" { +; CHECK-LABEL: test_inline_call( +entry: +; CHECK: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +} + +define i8 addrspace(1)* @test_inline_invoke(i8 addrspace(1)* %obj) gc "statepoint-example" personality i32* ()* @fake_personality_function { +; CHECK-LABEL: test_inline_invoke( +entry: +; CHECK: store i32 123, i32* @A + invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline to label %normal unwind label %exc + + normal: + ret i8 addrspace(1)* %obj + + exc: + %lpad = landingpad { i8*, i32 } cleanup + ret i8 addrspace(1)* null +} + +define i8 addrspace(1)* @test_no_inline_1(i8 addrspace(1)* %obj) gc "statepoint-example" { +; We don't inline function bodies that have a mismatching GC type. + +; CHECK-LABEL: test_no_inline_1( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +} + +define i8 addrspace(1)* @test_no_inline_2(i8 addrspace(1)* %obj) gc "statepoint-example" { +; Inlining through a statepoint that has gc arguments is not currently supported + +; CHECK-LABEL: test_no_inline_2( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj) alwaysinline + ret i8 addrspace(1)* null +} + +define i8 addrspace(1)* @test_no_inline_3(i8 addrspace(1)* %obj) gc "statepoint-example" { +; Inlining through a statepoint that has deopt arguments is not currenlty supported + +; CHECK-LABEL: test_no_inline_3( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 1, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +}