Index: include/llvm/Transforms/Utils/Cloning.h =================================================================== --- include/llvm/Transforms/Utils/Cloning.h +++ include/llvm/Transforms/Utils/Cloning.h @@ -20,6 +20,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" +#include "llvm/IR/Statepoint.h" #include "llvm/IR/ValueHandle.h" #include "llvm/IR/ValueMap.h" #include "llvm/Transforms/Utils/ValueMapper.h" Index: lib/Transforms/Scalar/PlaceSafepoints.cpp =================================================================== --- lib/Transforms/Scalar/PlaceSafepoints.cpp +++ lib/Transforms/Scalar/PlaceSafepoints.cpp @@ -109,6 +109,17 @@ // Print tracing output static cl::opt TraceLSP("spp-trace", cl::Hidden, cl::init(false)); + +// If this flag is true, polls are inserted as normal "alwaysinline" calls to +// gc.safepoint_poll that get wrapped in a statepoint by the call safepoint +// placement logic. The client is expected to run the AlwaysInliner pass +// afterwards to get the desired poll logic inlined into the IR. Once LLVM is +// adequately good at inlining through gc_statepoint wrapped calls it is +// expected that this will be on by default. +// +static cl::opt DontPreInlinePolls("spp-dont-pre-inline-polls", + cl::Hidden, cl::init(false)); + namespace { /// An analysis pass whose purpose is to identify each of the backedges in @@ -519,8 +530,26 @@ // TODO: These should become properties of the GCStrategy, possibly with // command line overrides. -static bool enableEntrySafepoints(Function &F) { return !NoEntry; } -static bool enableBackedgeSafepoints(Function &F) { return !NoBackedge; } +static bool enableEntrySafepoints(Function &F) { + if (isGCSafepointPoll(F)) { + // The poll function itself should not get a (recursive) poll. + assert(DontPreInlinePolls && "otherwise we should not be asking this!"); + return false; + } + + return !NoEntry; +} + +static bool enableBackedgeSafepoints(Function &F) { + if (isGCSafepointPoll(F)) { + // The poll function itself should not get a (recursive) poll. + assert(DontPreInlinePolls && "otherwise we should not be asking this!"); + return false; + } + + return !NoBackedge; +} + static bool enableCallSafepoints(Function &F) { return !NoCall; } // Normalize basic block to make it ready to be target of invoke statepoint. @@ -548,10 +577,10 @@ return false; } - if (isGCSafepointPoll(F)) { - // Given we're inlining this inside of safepoint poll insertion, this - // doesn't make any sense. Note that we do make any contained calls - // parseable after we inline a poll. + if (isGCSafepointPoll(F) && !DontPreInlinePolls) { + // If we're inlining this inside of safepoint poll insertion, this doesn't + // make any sense. Note that we do make any contained calls parseable after + // we inline a poll. return false; } @@ -802,6 +831,13 @@ assert(!F->empty() && "gc.safepoint_poll must be a non-empty function"); CallInst *PollCall = CallInst::Create(F, "", InsertBefore); + if (DontPreInlinePolls) { + ParsePointsNeeded.push_back(PollCall); + PollCall->addAttribute(AttributeSet::FunctionIndex, + Attribute::AlwaysInline); + return; + } + // Record some information about the call site we're replacing BasicBlock::iterator before(PollCall), after(PollCall); bool isBegin(false); Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -66,31 +66,54 @@ class InlineSite { CallSite CS; +#define INLINESITE_DELEGATE_GETTER(METHOD) do { \ + if (Statepoint SP = Statepoint(CS)) \ + return SP.METHOD; \ + return CS.METHOD; \ + } while (0) + bool paramHasAttr(unsigned i, Attribute::AttrKind A) const { - return CS.paramHasAttr(i, A); + INLINESITE_DELEGATE_GETTER(paramHasAttr(i, A)); } public: explicit InlineSite(CallSite CS) : CS(CS) { } - Function *getCalledFunction() const { return CS.getCalledFunction(); } + Function *getCalledFunction() const { + INLINESITE_DELEGATE_GETTER(getCalledFunction()); + } - Function *getCaller() const { return CS.getCaller(); } + Function *getCaller() const { + INLINESITE_DELEGATE_GETTER(getCaller()); + } - Instruction *getInstruction() const { return CS.getInstruction(); } + Instruction *getInstruction() const { + INLINESITE_DELEGATE_GETTER(getInstruction()); + } Value *getArgument(unsigned Index) const { - return CS.getArgument(Index); + INLINESITE_DELEGATE_GETTER(getArgument(Index)); + } + + bool doesNotThrow() const { + INLINESITE_DELEGATE_GETTER(doesNotThrow()); + } + + unsigned arg_size() const { + INLINESITE_DELEGATE_GETTER(arg_size()); + } + + CallSite::arg_iterator arg_begin() const { + INLINESITE_DELEGATE_GETTER(arg_begin()); + } + + CallSite::arg_iterator arg_end() const { + INLINESITE_DELEGATE_GETTER(arg_end()); } bool isByValArgument(unsigned ArgNo) const { return paramHasAttr(ArgNo + 1, Attribute::ByVal); } - - bool doesNotThrow() const { return CS.doesNotThrow(); } - unsigned arg_size() const { return CS.arg_size(); } - CallSite::arg_iterator arg_begin() const { return CS.arg_begin(); } - CallSite::arg_iterator arg_end() const { return CS.arg_end(); } }; } @@ -98,8 +121,8 @@ bool InsertLifetime, Value *&ReturnVal, bool &InlinedMustTailCalls); -bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, - bool InsertLifetime) { +static bool InlineGenericCallSite(CallSite CS, InlineFunctionInfo &IFI, + bool InsertLifetime) { Value *ReturnValOut = nullptr; bool InlinedMustTailCalls = false; if (!InlineThroughInlineSite(InlineSite(CS), IFI, InsertLifetime, @@ -124,6 +147,48 @@ return true; } +static bool InlineStatepoint(Statepoint SP, InlineFunctionInfo &IFI, + bool InsertLifetime) { + if (SP.gc_args_begin() != SP.gc_args_begin() || + SP.vm_state_begin() != SP.vm_state_end() || + SP.gc_transition_args_begin() != SP.gc_transition_args_end()) + return false; + + Function *Callee = SP.getCalledFunction(); + if (!Callee->hasGC()) + return false; + + if (StringRef(Callee->getGC()) != SP.getCaller()->getGC()) + return false; + + Value *ReturnValOut = nullptr; + bool InlinedMustTailCalls = false; + if (!InlineThroughInlineSite(InlineSite(SP.getCallSite()), IFI, + InsertLifetime, ReturnValOut, + InlinedMustTailCalls)) + return false; + + // TODO: add handling for unreachable code that may violate + // defs-dominate-uses. + if (CallInst *Result = cast_or_null(SP.getGCResult())) { + Result->replaceAllUsesWith(ReturnValOut); + Result->removeFromParent(); + } + + Instruction *StatepointInst = SP.getInstruction(); + assert(StatepointInst->use_empty() && "gc_result should be the only use!"); + StatepointInst->eraseFromParent(); + + return true; +} + +bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, + bool InsertLifetime) { + if (Statepoint SP = Statepoint(CS)) + return InlineStatepoint(SP, IFI, InsertLifetime); + return InlineGenericCallSite(CS, IFI, InsertLifetime); +} + namespace { /// A class for recording information about inlining through an invoke. class InvokeInliningInfo { Index: test/Transforms/PlaceSafepoints/basic.ll =================================================================== --- test/Transforms/PlaceSafepoints/basic.ll +++ test/Transforms/PlaceSafepoints/basic.ll @@ -1,5 +1,5 @@ -; RUN: opt %s -S -place-safepoints | FileCheck %s - +; RUN: opt %s -S -place-safepoints | FileCheck %s --check-prefix CHECK --check-prefix WITH-PRE-INLINE +; RUN: opt %s -S -spp-dont-pre-inline-polls -place-safepoints -always-inline | FileCheck %s --check-prefix CHECK --check-prefix WITHOUT-PRE-INLINE ; Do we insert a simple entry safepoint? define void @test_entry() gc "statepoint-example" { @@ -14,7 +14,7 @@ define void @test_negative() { ; CHECK-LABEL: @test_negative entry: -; CHECK-NOT: statepoint +; CHECK-NOT: statepoint{{.*}}do_safepoint ret void } @@ -25,7 +25,7 @@ entry: ; CHECK-LABEL: entry ; This statepoint is technically not required, but we don't exploit that yet. -; CHECK: statepoint +; CHECK: statepoint{{.*}}do_safepoint br label %other ; CHECK-LABEL: other @@ -41,11 +41,11 @@ ; CHECK-LABEL: test_unreachable entry: ; CHECK-LABEL: entry -; CHECK: statepoint +; CHECK: statepoint{{.*}}do_safepoint ret void ; CHECK-NOT: other -; CHECK-NOT: statepoint +; CHECK-NOT: statepoint{{.*}}do_safepoint other: br label %other } @@ -59,7 +59,7 @@ br label %other other: ; CHECK-LABEL: other -; CHECK: statepoint +; CHECK: statepoint{{.*}}do_safepoint ; CHECK-NOT: gc.result call void @foo() ret void @@ -71,7 +71,7 @@ ; CHECK-LABEL: test_call_with_result ; This is checking that a statepoint_poll + statepoint + result is ; inserted for a function that takes 1 argument. -; CHECK: gc.statepoint.p0f_isVoidf +; CHECK: gc.statepoint.p0f_isVoidf{{.*}}do_safepoint ; CHECK: gc.statepoint.p0f_i1i1f ; CHECK: (i64 2882400000, i32 0, i1 (i1)* @i1_return_i1, i32 1, i32 0, i1 false, i32 0, i32 0) ; CHECK: %call1.2 = call i1 @llvm.experimental.gc.result.i1 @@ -83,11 +83,18 @@ ; This function is inlined when inserting a poll. To avoid recursive ; issues, make sure we don't place safepoints in it. declare void @do_safepoint() -define void @gc.safepoint_poll() { -; CHECK-LABEL: gc.safepoint_poll -; CHECK-LABEL: entry -; CHECK-NEXT: do_safepoint -; CHECK-NEXT: ret void +define void @gc.safepoint_poll() gc "statepoint-example" { +; WITH-PRE-INLINE-LABEL: gc.safepoint_poll +; WITH-PRE-INLINE-NOT: gc.safepoint_poll +; WITH-PRE-INLINE-LABEL: entry +; WITH-PRE-INLINE-NOT: statepoint +; WITH-PRE-INLINE-NEXT: do_safepoint +; WITH-PRE-INLINE-NEXT: ret void + +; WITHOUT-PRE-INLINE-LABEL: gc.safepoint_poll +; WITHOUT-PRE-INLINE-NOT: gc.safepoint_poll +; WITHOUT-PRE-INLINE-LABEL: entry +; WITHOUT-PRE-INLINE: statepoint entry: call void @do_safepoint() ret void