Index: include/llvm/IR/CallSite.h =================================================================== --- include/llvm/IR/CallSite.h +++ include/llvm/IR/CallSite.h @@ -31,6 +31,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" namespace llvm { @@ -103,6 +104,19 @@ *getCallee() = V; } + /// getTargetFunction - Return the final destination this call or invoke will + /// dispatch to if it is a function literal, with consideration to the fact + /// that the control flow may be carried out indirectly via gc_statepoint. + /// + FunTy *getTargetFunction() const { + FunTy *LiteralDest = getCalledFunction(); + if (LiteralDest && + LiteralDest->getIntrinsicID() == Intrinsic::experimental_gc_statepoint) + return dyn_cast(getArgument(2)); + + return LiteralDest; + } + /// isCallee - Determine whether the passed iterator points to the /// callee operand's Use. bool isCallee(Value::const_user_iterator UI) const { Index: lib/Analysis/IPA/InlineCost.cpp =================================================================== --- lib/Analysis/IPA/InlineCost.cpp +++ lib/Analysis/IPA/InlineCost.cpp @@ -1315,8 +1315,8 @@ InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee, int Threshold) { - // Cannot inline indirect calls. - if (!Callee) + // Cannot inline indirect calls or calls through intrinsics. + if (!Callee || Callee->isIntrinsic()) return llvm::InlineCost::getNever(); // Calls to functions with always-inline attributes should be inlined Index: lib/Transforms/IPO/InlineAlways.cpp =================================================================== --- lib/Transforms/IPO/InlineAlways.cpp +++ lib/Transforms/IPO/InlineAlways.cpp @@ -93,7 +93,7 @@ /// small functions which have the explicit attribute to force inlining, it is /// likely not worth it in practice. InlineCost AlwaysInliner::getInlineCost(CallSite CS) { - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); // Only inline direct calls to functions with always-inline attributes // that are viable for inlining. FIXME: We shouldn't even get here for Index: lib/Transforms/IPO/Inliner.cpp =================================================================== --- lib/Transforms/IPO/Inliner.cpp +++ lib/Transforms/IPO/Inliner.cpp @@ -129,7 +129,7 @@ static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime) { - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); Function *Caller = CS.getCaller(); // Try to inline the function. Get the list of static allocas that were @@ -273,7 +273,7 @@ // Listen to the inlinehint attribute when it would increase the threshold // and the caller does not need to minimize its size. - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); bool InlineHint = Callee && !Callee->isDeclaration() && Callee->hasFnAttribute(Attribute::InlineHint); if (InlineHint && HintThreshold > thres && @@ -308,7 +308,7 @@ if (IC.isAlways()) { DEBUG(dbgs() << " Inlining: cost=always" << ", Call: " << *CS.getInstruction() << "\n"); - emitAnalysis(CS, Twine(CS.getCalledFunction()->getName()) + + emitAnalysis(CS, Twine(CS.getTargetFunction()->getName()) + " should always be inlined (cost=always)"); return true; } @@ -316,7 +316,7 @@ if (IC.isNever()) { DEBUG(dbgs() << " NOT Inlining: cost=never" << ", Call: " << *CS.getInstruction() << "\n"); - emitAnalysis(CS, Twine(CS.getCalledFunction()->getName() + + emitAnalysis(CS, Twine(CS.getTargetFunction()->getName() + " should never be inlined (cost=never)")); return false; } @@ -326,7 +326,7 @@ DEBUG(dbgs() << " NOT Inlining: cost=" << IC.getCost() << ", thres=" << (IC.getCostDelta() + IC.getCost()) << ", Call: " << *CS.getInstruction() << "\n"); - emitAnalysis(CS, Twine(CS.getCalledFunction()->getName() + + emitAnalysis(CS, Twine(CS.getTargetFunction()->getName() + " too costly to inline (cost=") + Twine(IC.getCost()) + ", threshold=" + Twine(IC.getCostDelta() + IC.getCost()) + ")"); @@ -363,7 +363,7 @@ // If this isn't a call to Caller (it could be some other sort // of reference) skip it. Such references will prevent the caller // from being removed. - if (!CS2 || CS2.getCalledFunction() != Caller) { + if (!CS2 || CS2.getTargetFunction() != Caller) { callerWillBeRemoved = false; continue; } @@ -398,7 +398,7 @@ ", outer Cost = " << TotalSecondaryCost << '\n'); emitAnalysis( CS, Twine("Not inlining. Cost of inlining " + - CS.getCalledFunction()->getName() + + CS.getTargetFunction()->getName() + " increases the cost of inlining " + CS.getCaller()->getName() + " in other contexts")); return false; @@ -409,7 +409,7 @@ << ", thres=" << (IC.getCostDelta() + IC.getCost()) << ", Call: " << *CS.getInstruction() << '\n'); emitAnalysis( - CS, CS.getCalledFunction()->getName() + Twine(" can be inlined into ") + + CS, CS.getTargetFunction()->getName() + Twine(" can be inlined into ") + CS.getCaller()->getName() + " with cost=" + Twine(IC.getCost()) + " (threshold=" + Twine(IC.getCostDelta() + IC.getCost()) + ")"); return true; @@ -462,15 +462,21 @@ for (BasicBlock &BB : *F) for (Instruction &I : BB) { CallSite CS(cast(&I)); - // If this isn't a call, or it is a call to an intrinsic, it can - // never be inlined. - if (!CS || isa(I)) + // If this isn't a call never be inlined. + if (!CS) continue; - + + // We can inline through gc_statepoint calls. Calls to other intrinsics + // cannot be inlined. + if (isa(I) && !isStatepoint(I)) + continue; + + Function *CalledFunction = CS.getTargetFunction(); + // If this is a direct call to an external function, we can never inline // it. If it is an indirect call, inlining may resolve it to be a // direct call, so we keep it. - if (CS.getCalledFunction() && CS.getCalledFunction()->isDeclaration()) + if (CalledFunction && CalledFunction->isDeclaration()) continue; CallSites.push_back(std::make_pair(CS, -1)); @@ -487,7 +493,7 @@ // current SCC to the end of the list. unsigned FirstCallInSCC = CallSites.size(); for (unsigned i = 0; i < FirstCallInSCC; ++i) - if (Function *F = CallSites[i].first.getCalledFunction()) + if (Function *F = CallSites[i].first.getTargetFunction()) if (SCCFunctions.count(F)) std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); @@ -508,7 +514,7 @@ CallSite CS = CallSites[CSi].first; Function *Caller = CS.getCaller(); - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); // If this call site is dead and it is to a readonly function, we should // just delete the call instead of trying to inline it, regardless of Index: test/Transforms/Inline/statepoints-basic.ll =================================================================== --- /dev/null +++ test/Transforms/Inline/statepoints-basic.ll @@ -0,0 +1,69 @@ +; RUN: opt -S -always-inline < %s | FileCheck %s + +declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) + +@A = global i32 7 + +declare i32* @fake_personality_function() + +define void @callee() gc "statepoint-example" { + store i32 123, i32* @A + ret void +} + +define void @callee_nogc() { + store i32 123, i32* @A + ret void +} + +define i8 addrspace(1)* @test_inline_call(i8 addrspace(1)* %obj) gc "statepoint-example" { +; CHECK-LABEL: test_inline_call( +entry: +; CHECK: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +} + +define i8 addrspace(1)* @test_inline_invoke(i8 addrspace(1)* %obj) gc "statepoint-example" personality i32* ()* @fake_personality_function { +; CHECK-LABEL: test_inline_invoke( +entry: +; CHECK: store i32 123, i32* @A + invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline to label %normal unwind label %exc + + normal: + ret i8 addrspace(1)* %obj + + exc: + %lpad = landingpad { i8*, i32 } cleanup + ret i8 addrspace(1)* null +} + +define i8 addrspace(1)* @test_no_inline_1(i8 addrspace(1)* %obj) gc "statepoint-example" { +; We don't inline function bodies that have a mismatching GC type. + +; CHECK-LABEL: test_no_inline_1( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +} + +define i8 addrspace(1)* @test_no_inline_2(i8 addrspace(1)* %obj) gc "statepoint-example" { +; Inlining through a statepoint that has gc arguments is not currently supported + +; CHECK-LABEL: test_no_inline_2( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj) alwaysinline + ret i8 addrspace(1)* null +} + +define i8 addrspace(1)* @test_no_inline_3(i8 addrspace(1)* %obj) gc "statepoint-example" { +; Inlining through a statepoint that has deopt arguments is not currenlty supported + +; CHECK-LABEL: test_no_inline_3( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 1, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +}