Index: include/llvm/IR/CallSite.h =================================================================== --- include/llvm/IR/CallSite.h +++ include/llvm/IR/CallSite.h @@ -31,6 +31,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" namespace llvm { @@ -107,6 +108,19 @@ *getCallee() = V; } + /// getTargetFunction - Return the final destination this call or invoke will + /// dispatch to if it is a function literal, with consideration to the fact + /// that the control flow may be carried out indirectly via gc_statepoint. + /// + FunTy *getTargetFunction() const { + FunTy *LiteralDest = getCalledFunction(); + if (LiteralDest && + LiteralDest->getIntrinsicID() == Intrinsic::experimental_gc_statepoint) + return dyn_cast(getArgument(2)); + + return LiteralDest; + } + /// isCallee - Determine whether the passed iterator points to the /// callee operand's Use. bool isCallee(Value::const_user_iterator UI) const { Index: lib/Analysis/IPA/InlineCost.cpp =================================================================== --- lib/Analysis/IPA/InlineCost.cpp +++ lib/Analysis/IPA/InlineCost.cpp @@ -1354,8 +1354,8 @@ InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee, int Threshold) { - // Cannot inline indirect calls. - if (!Callee) + // Cannot inline indirect calls or calls through intrinsics. + if (!Callee || Callee->isIntrinsic()) return llvm::InlineCost::getNever(); // Calls to functions with always-inline attributes should be inlined Index: lib/Transforms/IPO/InlineAlways.cpp =================================================================== --- lib/Transforms/IPO/InlineAlways.cpp +++ lib/Transforms/IPO/InlineAlways.cpp @@ -93,7 +93,7 @@ /// small functions which have the explicit attribute to force inlining, it is /// likely not worth it in practice. InlineCost AlwaysInliner::getInlineCost(CallSite CS) { - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); // Only inline direct calls to functions with always-inline attributes // that are viable for inlining. FIXME: We shouldn't even get here for Index: lib/Transforms/IPO/Inliner.cpp =================================================================== --- lib/Transforms/IPO/Inliner.cpp +++ lib/Transforms/IPO/Inliner.cpp @@ -27,6 +27,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Statepoint.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -129,7 +130,7 @@ static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime) { - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); Function *Caller = CS.getCaller(); // Try to inline the function. Get the list of static allocas that were @@ -272,7 +273,7 @@ // Listen to the inlinehint attribute when it would increase the threshold // and the caller does not need to minimize its size. - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); bool InlineHint = Callee && !Callee->isDeclaration() && Callee->hasFnAttribute(Attribute::InlineHint); if (InlineHint && HintThreshold > thres && @@ -307,7 +308,7 @@ if (IC.isAlways()) { DEBUG(dbgs() << " Inlining: cost=always" << ", Call: " << *CS.getInstruction() << "\n"); - emitAnalysis(CS, Twine(CS.getCalledFunction()->getName()) + + emitAnalysis(CS, Twine(CS.getTargetFunction()->getName()) + " should always be inlined (cost=always)"); return true; } @@ -315,7 +316,7 @@ if (IC.isNever()) { DEBUG(dbgs() << " NOT Inlining: cost=never" << ", Call: " << *CS.getInstruction() << "\n"); - emitAnalysis(CS, Twine(CS.getCalledFunction()->getName() + + emitAnalysis(CS, Twine(CS.getTargetFunction()->getName() + " should never be inlined (cost=never)")); return false; } @@ -325,7 +326,7 @@ DEBUG(dbgs() << " NOT Inlining: cost=" << IC.getCost() << ", thres=" << (IC.getCostDelta() + IC.getCost()) << ", Call: " << *CS.getInstruction() << "\n"); - emitAnalysis(CS, Twine(CS.getCalledFunction()->getName() + + emitAnalysis(CS, Twine(CS.getTargetFunction()->getName() + " too costly to inline (cost=") + Twine(IC.getCost()) + ", threshold=" + Twine(IC.getCostDelta() + IC.getCost()) + ")"); @@ -362,7 +363,7 @@ // If this isn't a call to Caller (it could be some other sort // of reference) skip it. Such references will prevent the caller // from being removed. - if (!CS2 || CS2.getCalledFunction() != Caller) { + if (!CS2 || CS2.getTargetFunction() != Caller) { callerWillBeRemoved = false; continue; } @@ -397,7 +398,7 @@ ", outer Cost = " << TotalSecondaryCost << '\n'); emitAnalysis( CS, Twine("Not inlining. Cost of inlining " + - CS.getCalledFunction()->getName() + + CS.getTargetFunction()->getName() + " increases the cost of inlining " + CS.getCaller()->getName() + " in other contexts")); return false; @@ -408,7 +409,7 @@ << ", thres=" << (IC.getCostDelta() + IC.getCost()) << ", Call: " << *CS.getInstruction() << '\n'); emitAnalysis( - CS, CS.getCalledFunction()->getName() + Twine(" can be inlined into ") + + CS, CS.getTargetFunction()->getName() + Twine(" can be inlined into ") + CS.getCaller()->getName() + " with cost=" + Twine(IC.getCost()) + " (threshold=" + Twine(IC.getCostDelta() + IC.getCost()) + ")"); return true; @@ -461,15 +462,21 @@ for (BasicBlock &BB : *F) for (Instruction &I : BB) { CallSite CS(cast(&I)); - // If this isn't a call, or it is a call to an intrinsic, it can - // never be inlined. - if (!CS || isa(I)) + // If this isn't a call never be inlined. + if (!CS) continue; - + + // We can inline through gc_statepoint calls. Calls to other intrinsics + // cannot be inlined. + if (isa(I) && !isStatepoint(I)) + continue; + + Function *CalledFunction = CS.getTargetFunction(); + // If this is a direct call to an external function, we can never inline // it. If it is an indirect call, inlining may resolve it to be a // direct call, so we keep it. - if (CS.getCalledFunction() && CS.getCalledFunction()->isDeclaration()) + if (CalledFunction && CalledFunction->isDeclaration()) continue; CallSites.push_back(std::make_pair(CS, -1)); @@ -486,7 +493,7 @@ // current SCC to the end of the list. unsigned FirstCallInSCC = CallSites.size(); for (unsigned i = 0; i < FirstCallInSCC; ++i) - if (Function *F = CallSites[i].first.getCalledFunction()) + if (Function *F = CallSites[i].first.getTargetFunction()) if (SCCFunctions.count(F)) std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); @@ -507,7 +514,7 @@ CallSite CS = CallSites[CSi].first; Function *Caller = CS.getCaller(); - Function *Callee = CS.getCalledFunction(); + Function *Callee = CS.getTargetFunction(); // If this call site is dead and it is to a readonly function, we should // just delete the call instead of trying to inline it, regardless of Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -38,6 +38,7 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Statepoint.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Support/CommandLine.h" #include @@ -66,31 +67,54 @@ class InlineSite { CallSite CS; +#define INLINESITE_DELEGATE_GETTER(METHOD) do { \ + if (Statepoint SP = Statepoint(CS)) \ + return SP.METHOD; \ + return CS.METHOD; \ + } while (0) + bool paramHasAttr(unsigned i, Attribute::AttrKind A) const { - return CS.paramHasAttr(i, A); + INLINESITE_DELEGATE_GETTER(paramHasAttr(i, A)); } public: explicit InlineSite(CallSite CS) : CS(CS) { } - Function *getCalledFunction() const { return CS.getCalledFunction(); } + Function *getCalledFunction() const { + INLINESITE_DELEGATE_GETTER(getCalledFunction()); + } - Function *getCaller() const { return CS.getCaller(); } + Function *getCaller() const { + INLINESITE_DELEGATE_GETTER(getCaller()); + } - Instruction *getInstruction() const { return CS.getInstruction(); } + Instruction *getInstruction() const { + INLINESITE_DELEGATE_GETTER(getInstruction()); + } Value *getArgument(unsigned Index) const { - return CS.getArgument(Index); + INLINESITE_DELEGATE_GETTER(getArgument(Index)); + } + + bool doesNotThrow() const { + INLINESITE_DELEGATE_GETTER(doesNotThrow()); + } + + unsigned arg_size() const { + INLINESITE_DELEGATE_GETTER(arg_size()); + } + + CallSite::arg_iterator arg_begin() const { + INLINESITE_DELEGATE_GETTER(arg_begin()); + } + + CallSite::arg_iterator arg_end() const { + INLINESITE_DELEGATE_GETTER(arg_end()); } bool isByValArgument(unsigned ArgNo) const { return paramHasAttr(ArgNo + 1, Attribute::ByVal); } - - bool doesNotThrow() const { return CS.doesNotThrow(); } - unsigned arg_size() const { return CS.arg_size(); } - CallSite::arg_iterator arg_begin() const { return CS.arg_begin(); } - CallSite::arg_iterator arg_end() const { return CS.arg_end(); } }; } @@ -98,8 +122,8 @@ bool InsertLifetime, Value *&ReturnVal, bool &InlinedMustTailCalls); -bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, - bool InsertLifetime) { +static bool InlineGenericCallSite(CallSite CS, InlineFunctionInfo &IFI, + bool InsertLifetime) { Value *ReturnValOut = nullptr; bool InlinedMustTailCalls = false; if (!InlineThroughInlineSite(InlineSite(CS), IFI, InsertLifetime, @@ -124,6 +148,52 @@ return true; } +static bool InlineStatepoint(Statepoint SP, InlineFunctionInfo &IFI, + bool InsertLifetime) { + + // We currently only inline through statepoints that do not have any + // associated GC or deopt state. + + if (SP.gc_args_begin() != SP.gc_args_end() || + SP.vm_state_begin() != SP.vm_state_end() || + SP.gc_transition_args_begin() != SP.gc_transition_args_end()) + return false; + + Function *Callee = SP.getCalledFunction(); + if (!Callee->hasGC()) + return false; + + if (StringRef(Callee->getGC()) != SP.getCaller()->getGC()) + return false; + + Value *ReturnValOut = nullptr; + bool InlinedMustTailCalls = false; + if (!InlineThroughInlineSite(InlineSite(SP.getCallSite()), IFI, + InsertLifetime, ReturnValOut, + InlinedMustTailCalls)) + return false; + + // TODO: add handling for unreachable code that may violate + // defs-dominate-uses. + if (CallInst *Result = cast_or_null(SP.getGCResult())) { + Result->replaceAllUsesWith(ReturnValOut); + Result->removeFromParent(); + } + + Instruction *StatepointInst = SP.getInstruction(); + assert(StatepointInst->use_empty() && "gc_result should be the only use!"); + StatepointInst->eraseFromParent(); + + return true; +} + +bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, + bool InsertLifetime) { + if (Statepoint SP = Statepoint(CS)) + return InlineStatepoint(SP, IFI, InsertLifetime); + return InlineGenericCallSite(CS, IFI, InsertLifetime); +} + namespace { /// A class for recording information about inlining through an invoke. class InvokeInliningInfo { Index: test/Transforms/Inline/statepoints-basic.ll =================================================================== --- /dev/null +++ test/Transforms/Inline/statepoints-basic.ll @@ -0,0 +1,69 @@ +; RUN: opt -S -always-inline < %s | FileCheck %s + +declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) + +@A = global i32 7 + +declare i32* @fake_personality_function() + +define void @callee() gc "statepoint-example" { + store i32 123, i32* @A + ret void +} + +define void @callee_nogc() { + store i32 123, i32* @A + ret void +} + +define i8 addrspace(1)* @test_inline_call(i8 addrspace(1)* %obj) gc "statepoint-example" { +; CHECK-LABEL: test_inline_call( +entry: +; CHECK: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +} + +define i8 addrspace(1)* @test_inline_invoke(i8 addrspace(1)* %obj) gc "statepoint-example" personality i32* ()* @fake_personality_function { +; CHECK-LABEL: test_inline_invoke( +entry: +; CHECK: store i32 123, i32* @A + invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline to label %normal unwind label %exc + + normal: + ret i8 addrspace(1)* %obj + + exc: + %lpad = landingpad { i8*, i32 } cleanup + ret i8 addrspace(1)* null +} + +define i8 addrspace(1)* @test_no_inline_1(i8 addrspace(1)* %obj) gc "statepoint-example" { +; We don't inline function bodies that have a mismatching GC type. + +; CHECK-LABEL: test_no_inline_1( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +} + +define i8 addrspace(1)* @test_no_inline_2(i8 addrspace(1)* %obj) gc "statepoint-example" { +; Inlining through a statepoint that has gc arguments is not currently supported + +; CHECK-LABEL: test_no_inline_2( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj) alwaysinline + ret i8 addrspace(1)* null +} + +define i8 addrspace(1)* @test_no_inline_3(i8 addrspace(1)* %obj) gc "statepoint-example" { +; Inlining through a statepoint that has deopt arguments is not currently supported + +; CHECK-LABEL: test_no_inline_3( +entry: +; CHECK-NOT: store i32 123, i32* @A + call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 1, i32 0) alwaysinline + ret i8 addrspace(1)* %obj +}