Index: include/llvm/IR/CallSite.h
===================================================================
--- include/llvm/IR/CallSite.h
+++ include/llvm/IR/CallSite.h
@@ -31,6 +31,7 @@
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
namespace llvm {
@@ -103,6 +104,19 @@
*getCallee() = V;
}
+ /// getTargetFunction - Return the final destination this call or invoke will
+ /// dispatch to if it is a function literal, with consideration to the fact
+ /// that the control flow may be carried out indirectly via gc_statepoint.
+ ///
+ FunTy *getTargetFunction() const {
+ FunTy *LiteralDest = getCalledFunction();
+ if (LiteralDest &&
+ LiteralDest->getIntrinsicID() == Intrinsic::experimental_gc_statepoint)
+ return dyn_cast<FunTy>(getArgument(2));
+
+ return LiteralDest;
+ }
+
/// isCallee - Determine whether the passed iterator points to the
/// callee operand's Use.
bool isCallee(Value::const_user_iterator UI) const {
Index: lib/Analysis/IPA/InlineCost.cpp
===================================================================
--- lib/Analysis/IPA/InlineCost.cpp
+++ lib/Analysis/IPA/InlineCost.cpp
@@ -1315,8 +1315,8 @@
InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee,
int Threshold) {
- // Cannot inline indirect calls.
- if (!Callee)
+ // Cannot inline indirect calls or calls through intrinsics.
+ if (!Callee || Callee->isIntrinsic())
return llvm::InlineCost::getNever();
// Calls to functions with always-inline attributes should be inlined
Index: lib/Transforms/IPO/InlineAlways.cpp
===================================================================
--- lib/Transforms/IPO/InlineAlways.cpp
+++ lib/Transforms/IPO/InlineAlways.cpp
@@ -93,7 +93,7 @@
/// small functions which have the explicit attribute to force inlining, it is
/// likely not worth it in practice.
InlineCost AlwaysInliner::getInlineCost(CallSite CS) {
- Function *Callee = CS.getCalledFunction();
+ Function *Callee = CS.getTargetFunction();
// Only inline direct calls to functions with always-inline attributes
// that are viable for inlining. FIXME: We shouldn't even get here for
Index: lib/Transforms/IPO/Inliner.cpp
===================================================================
--- lib/Transforms/IPO/Inliner.cpp
+++ lib/Transforms/IPO/Inliner.cpp
@@ -129,7 +129,7 @@
static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI,
InlinedArrayAllocasTy &InlinedArrayAllocas,
int InlineHistory, bool InsertLifetime) {
- Function *Callee = CS.getCalledFunction();
+ Function *Callee = CS.getTargetFunction();
Function *Caller = CS.getCaller();
// Try to inline the function. Get the list of static allocas that were
@@ -273,7 +273,7 @@
// Listen to the inlinehint attribute when it would increase the threshold
// and the caller does not need to minimize its size.
- Function *Callee = CS.getCalledFunction();
+ Function *Callee = CS.getTargetFunction();
bool InlineHint = Callee && !Callee->isDeclaration() &&
Callee->hasFnAttribute(Attribute::InlineHint);
if (InlineHint && HintThreshold > thres &&
@@ -308,7 +308,7 @@
if (IC.isAlways()) {
DEBUG(dbgs() << " Inlining: cost=always"
<< ", Call: " << *CS.getInstruction() << "\n");
- emitAnalysis(CS, Twine(CS.getCalledFunction()->getName()) +
+ emitAnalysis(CS, Twine(CS.getTargetFunction()->getName()) +
" should always be inlined (cost=always)");
return true;
}
@@ -316,7 +316,7 @@
if (IC.isNever()) {
DEBUG(dbgs() << " NOT Inlining: cost=never"
<< ", Call: " << *CS.getInstruction() << "\n");
- emitAnalysis(CS, Twine(CS.getCalledFunction()->getName() +
+ emitAnalysis(CS, Twine(CS.getTargetFunction()->getName() +
" should never be inlined (cost=never)"));
return false;
}
@@ -326,7 +326,7 @@
DEBUG(dbgs() << " NOT Inlining: cost=" << IC.getCost()
<< ", thres=" << (IC.getCostDelta() + IC.getCost())
<< ", Call: " << *CS.getInstruction() << "\n");
- emitAnalysis(CS, Twine(CS.getCalledFunction()->getName() +
+ emitAnalysis(CS, Twine(CS.getTargetFunction()->getName() +
" too costly to inline (cost=") +
Twine(IC.getCost()) + ", threshold=" +
Twine(IC.getCostDelta() + IC.getCost()) + ")");
@@ -363,7 +363,7 @@
// If this isn't a call to Caller (it could be some other sort
// of reference) skip it. Such references will prevent the caller
// from being removed.
- if (!CS2 || CS2.getCalledFunction() != Caller) {
+ if (!CS2 || CS2.getTargetFunction() != Caller) {
callerWillBeRemoved = false;
continue;
}
@@ -398,7 +398,7 @@
", outer Cost = " << TotalSecondaryCost << '\n');
emitAnalysis(
CS, Twine("Not inlining. Cost of inlining " +
- CS.getCalledFunction()->getName() +
+ CS.getTargetFunction()->getName() +
" increases the cost of inlining " +
CS.getCaller()->getName() + " in other contexts"));
return false;
@@ -409,7 +409,7 @@
<< ", thres=" << (IC.getCostDelta() + IC.getCost())
<< ", Call: " << *CS.getInstruction() << '\n');
emitAnalysis(
- CS, CS.getCalledFunction()->getName() + Twine(" can be inlined into ") +
+ CS, CS.getTargetFunction()->getName() + Twine(" can be inlined into ") +
CS.getCaller()->getName() + " with cost=" + Twine(IC.getCost()) +
" (threshold=" + Twine(IC.getCostDelta() + IC.getCost()) + ")");
return true;
@@ -462,15 +462,21 @@
for (BasicBlock &BB : *F)
for (Instruction &I : BB) {
CallSite CS(cast<Value>(&I));
- // If this isn't a call, or it is a call to an intrinsic, it can
- // never be inlined.
- if (!CS || isa<IntrinsicInst>(I))
+ // If this isn't a call never be inlined.
+ if (!CS)
continue;
-
+
+ // We can inline through gc_statepoint calls. Calls to other intrinsics
+ // cannot be inlined.
+ if (isa<IntrinsicInst>(I) && !isStatepoint(I))
+ continue;
+
+ Function *CalledFunction = CS.getTargetFunction();
+
// If this is a direct call to an external function, we can never inline
// it. If it is an indirect call, inlining may resolve it to be a
// direct call, so we keep it.
- if (CS.getCalledFunction() && CS.getCalledFunction()->isDeclaration())
+ if (CalledFunction && CalledFunction->isDeclaration())
continue;
CallSites.push_back(std::make_pair(CS, -1));
@@ -487,7 +493,7 @@
// current SCC to the end of the list.
unsigned FirstCallInSCC = CallSites.size();
for (unsigned i = 0; i < FirstCallInSCC; ++i)
- if (Function *F = CallSites[i].first.getCalledFunction())
+ if (Function *F = CallSites[i].first.getTargetFunction())
if (SCCFunctions.count(F))
std::swap(CallSites[i--], CallSites[--FirstCallInSCC]);
@@ -508,7 +514,7 @@
CallSite CS = CallSites[CSi].first;
Function *Caller = CS.getCaller();
- Function *Callee = CS.getCalledFunction();
+ Function *Callee = CS.getTargetFunction();
// If this call site is dead and it is to a readonly function, we should
// just delete the call instead of trying to inline it, regardless of
Index: test/Transforms/Inline/statepoints-basic.ll
===================================================================
--- /dev/null
+++ test/Transforms/Inline/statepoints-basic.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -always-inline < %s | FileCheck %s
+
+declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+@A = global i32 7
+
+declare i32* @fake_personality_function()
+
+define void @callee() gc "statepoint-example" {
+ store i32 123, i32* @A
+ ret void
+}
+
+define void @callee_nogc() {
+ store i32 123, i32* @A
+ ret void
+}
+
+define i8 addrspace(1)* @test_inline_call(i8 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: test_inline_call(
+entry:
+; CHECK: store i32 123, i32* @A
+ call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline
+ ret i8 addrspace(1)* %obj
+}
+
+define i8 addrspace(1)* @test_inline_invoke(i8 addrspace(1)* %obj) gc "statepoint-example" personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: test_inline_invoke(
+entry:
+; CHECK: store i32 123, i32* @A
+ invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee, i32 0, i32 0, i32 0, i32 0) alwaysinline to label %normal unwind label %exc
+
+ normal:
+ ret i8 addrspace(1)* %obj
+
+ exc:
+ %lpad = landingpad { i8*, i32 } cleanup
+ ret i8 addrspace(1)* null
+}
+
+define i8 addrspace(1)* @test_no_inline_1(i8 addrspace(1)* %obj) gc "statepoint-example" {
+; We don't inline function bodies that have a mismatching GC type.
+
+; CHECK-LABEL: test_no_inline_1(
+entry:
+; CHECK-NOT: store i32 123, i32* @A
+ call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0) alwaysinline
+ ret i8 addrspace(1)* %obj
+}
+
+define i8 addrspace(1)* @test_no_inline_2(i8 addrspace(1)* %obj) gc "statepoint-example" {
+; Inlining through a statepoint that has gc arguments is not currently supported
+
+; CHECK-LABEL: test_no_inline_2(
+entry:
+; CHECK-NOT: store i32 123, i32* @A
+ call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 0, i8 addrspace(1)* %obj) alwaysinline
+ ret i8 addrspace(1)* null
+}
+
+define i8 addrspace(1)* @test_no_inline_3(i8 addrspace(1)* %obj) gc "statepoint-example" {
+; Inlining through a statepoint that has deopt arguments is not currenlty supported
+
+; CHECK-LABEL: test_no_inline_3(
+entry:
+; CHECK-NOT: store i32 123, i32* @A
+ call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @callee_nogc, i32 0, i32 0, i32 0, i32 1, i32 0) alwaysinline
+ ret i8 addrspace(1)* %obj
+}