Index: include/llvm/IR/Statepoint.h =================================================================== --- include/llvm/IR/Statepoint.h +++ include/llvm/IR/Statepoint.h @@ -235,6 +235,10 @@ return getCallSite().arg_end(); } + unsigned gcArgsStartIdx() const { + return gc_args_begin() - getInstruction()->op_begin(); + } + /// range adapter for gc arguments iterator_range gc_args() const { return iterator_range(gc_args_begin(), gc_args_end()); Index: lib/Transforms/Scalar/RewriteStatepointsForGC.cpp =================================================================== --- lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -72,6 +72,12 @@ cl::location(ClobberNonLive), cl::Hidden); +static cl::opt UseDeoptBundles("rs4gc-use-deopt-bundles", cl::Hidden, + cl::init(false)); +static cl::opt + AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", + cl::Hidden, cl::init(false)); + namespace { struct RewriteStatepointsForGC : public ModulePass { static char ID; // Pass identification, replacement for typeid @@ -183,6 +189,36 @@ }; } +static bool CallsGCLeafFunction(ImmutableCallSite CS) { + if (isa(CS.getInstruction())) + // Most LLVM intrinsics are things which can never take a safepoint. + // As a result, we don't need to have the stack parsable at the + // callsite. This is a highly useful optimization since intrinsic + // calls are fairly prevalent, particularly in debug builds. + return true; + + // If this function is marked explicitly as a leaf call, we don't need to + // place a safepoint of it. + if (const Function *F = CS.getCalledFunction()) + return F->hasFnAttribute("gc-leaf-function"); + + return false; +} + +static ArrayRef GetDeoptBundleOperands(ImmutableCallSite CS) { + assert(UseDeoptBundles && "Should not be called otherwise!"); + + for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) { + OperandBundleUse Bundle = CS.getOperandBundle(i); + if (Bundle.Tag == "deopt") + return Bundle.Inputs; + } + assert(AllowStatepointWithNoDeoptInfo && + "Found non-leaf call without deopt info!"); + + return None; +} + /// Compute the live-in set for every basic block in the function static void computeLiveInValues(DominatorTree &DT, Function &F, GCPtrLivenessData &Data); @@ -1335,7 +1371,7 @@ const SmallVectorImpl &LiveVariables, PartiallyConstructedSafepointRecord &Result) { assert(BasePtrs.size() == LiveVariables.size()); - assert(isStatepoint(CS) && + assert((UseDeoptBundles || isStatepoint(CS)) && "This method expects to be rewriting a statepoint"); BasicBlock *BB = CS.getInstruction()->getParent(); @@ -1343,10 +1379,6 @@ Module *M = F->getParent(); assert(M && "must be set"); - // We're not changing the function signature of the statepoint since the gc - // arguments go into the var args section. - Function *GCStatepointDecl = CS.getCalledFunction(); - // Then go ahead and use the builder do actually do the inserts. We insert // immediately before the previous instruction under the assumption that all // arguments will be available here. We can't insert afterwards since we may @@ -1354,24 +1386,59 @@ Instruction *InsertBefore = CS.getInstruction(); IRBuilder<> Builder(InsertBefore); - // Copy all of the arguments from the original statepoint - this includes the - // target, call args, and deopt args - SmallVector Args; - Args.insert(Args.end(), CS.arg_begin(), CS.arg_end()); - // TODO: Clear the 'needs rewrite' flag + ArrayRef GCArgs(LiveVariables); + + uint64_t StatepointID = 0xABCDEF00; + uint32_t NumPatchBytes = 0; + StatepointFlags Flags = StatepointFlags::None; + + ArrayRef CallArgs; + ArrayRef DeoptArgs; + ArrayRef TransitionArgs; + + Value *CallTarget = nullptr; + + if (UseDeoptBundles) { + CallArgs = {CS.arg_begin(), CS.arg_end()}; + DeoptArgs = GetDeoptBundleOperands(CS); + // We don't fill in TransitionArgs or Flags in this branch, but we could + // have an operand bundle for that too. + AttributeSet OriginalAttrs = CS.getAttributes(); + + Attribute AttrID = OriginalAttrs.getAttribute(AttributeSet::FunctionIndex, + "statepoint-id"); + if (AttrID.isStringAttribute()) + AttrID.getValueAsString().getAsInteger(10, StatepointID); - // Add all the pointers to be relocated (gc arguments) and capture the start - // of the live variable list for use in the gc_relocates - const int LiveStartIdx = Args.size(); - Args.insert(Args.end(), LiveVariables.begin(), LiveVariables.end()); + Attribute AttrNumPatchBytes = OriginalAttrs.getAttribute( + AttributeSet::FunctionIndex, "statepoint-num-patch-bytes"); + if (AttrNumPatchBytes.isStringAttribute()) + AttrNumPatchBytes.getValueAsString().getAsInteger(10, NumPatchBytes); + + CallTarget = CS.getCalledValue(); + } else { + // This branch will be gone soon, and we will soon only support the + // UseDeoptBundles == true configuration. + Statepoint SP(CS); + StatepointID = SP.getID(); + NumPatchBytes = SP.getNumPatchBytes(); + Flags = StatepointFlags(SP.getFlags()); + + CallArgs = {SP.arg_begin(), SP.arg_end()}; + DeoptArgs = {SP.vm_state_begin(), SP.vm_state_end()}; + TransitionArgs = {SP.gc_transition_args_begin(), + SP.gc_transition_args_end()}; + CallTarget = SP.getCalledValue(); + } // Create the statepoint given all the arguments Instruction *Token = nullptr; AttributeSet ReturnAttrs; if (CS.isCall()) { CallInst *ToReplace = cast(CS.getInstruction()); - CallInst *Call = - Builder.CreateCall(GCStatepointDecl, Args, "safepoint_token"); + CallInst *Call = Builder.CreateGCStatepointCall( + StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs, + TransitionArgs, DeoptArgs, GCArgs, "safepoint_token"); Call->setTailCall(ToReplace->isTailCall()); Call->setCallingConv(ToReplace->getCallingConv()); @@ -1396,10 +1463,10 @@ // Insert the new invoke into the old block. We'll remove the old one in a // moment at which point this will become the new terminator for the // original block. - InvokeInst *Invoke = - InvokeInst::Create(GCStatepointDecl, ToReplace->getNormalDest(), - ToReplace->getUnwindDest(), Args, "statepoint_token", - ToReplace->getParent()); + InvokeInst *Invoke = Builder.CreateGCStatepointInvoke( + StatepointID, NumPatchBytes, CallTarget, ToReplace->getNormalDest(), + ToReplace->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs, + GCArgs, "statepoint_token"); Invoke->setCallingConv(ToReplace->getCallingConv()); // Currently we will fail on parameter attributes and on certain @@ -1428,6 +1495,7 @@ UnwindBlock->getLandingPadInst(), 1, "relocate_token")); Result.UnwindToken = ExceptionalToken; + unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, ExceptionalToken, Builder); @@ -1447,22 +1515,34 @@ // Take the name of the original value call if it had one. Token->takeName(CS.getInstruction()); -// The GCResult is already inserted, we just need to find it #ifndef NDEBUG - Instruction *ToReplace = CS.getInstruction(); - assert(!ToReplace->hasNUsesOrMore(2) && - "only valid use before rewrite is gc.result"); - assert(!ToReplace->hasOneUse() || - isGCResult(cast(*ToReplace->user_begin()))); + if (!UseDeoptBundles) { + Instruction *ToReplace = CS.getInstruction(); + assert(!ToReplace->hasNUsesOrMore(2) && + "only valid use before rewrite is gc.result"); + assert(!ToReplace->hasOneUse() || + isGCResult(cast(*ToReplace->user_begin()))); + } #endif - // Update the gc.result of the original statepoint (if any) to use the newly - // inserted statepoint. This is safe to do here since the token can't be - // considered a live reference. - CS.getInstruction()->replaceAllUsesWith(Token); + if (UseDeoptBundles) { + if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) { + StringRef Name = + CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : ""; + CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), Name); + GCResult->setAttributes(CS.getAttributes().getRetAttributes()); + CS.getInstruction()->replaceAllUsesWith(GCResult); + } + } else { + // Update the gc.result of the original statepoint (if any) to use the newly + // inserted statepoint. This is safe to do here since the token can't be + // considered a live reference. + CS.getInstruction()->replaceAllUsesWith(Token); + } Result.StatepointToken = Token; + unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); // Second, create a gc.relocate for every live variable CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, Token, Builder); } @@ -2188,7 +2268,8 @@ for (CallSite CS : ToUpdate) { assert(CS.getInstruction()->getParent()->getParent() == &F); - assert(isStatepoint(CS) && "expected to already be a deopt statepoint"); + assert((UseDeoptBundles || isStatepoint(CS)) && + "expected to already be a deopt statepoint"); } #endif @@ -2213,16 +2294,20 @@ // the deopt argument list are considered live through the safepoint (and // thus makes sure they get relocated.) for (CallSite CS : ToUpdate) { - Statepoint StatepointCS(CS); - SmallVector DeoptValues; - for (Use &U : StatepointCS.vm_state_args()) { - Value *Arg = cast(&U); + + iterator_range DeoptStateRange = + UseDeoptBundles + ? iterator_range(GetDeoptBundleOperands(CS)) + : iterator_range(Statepoint(CS).vm_state_args()); + + for (Value *Arg : DeoptStateRange) { assert(!isUnhandledGCPointerType(Arg->getType()) && "support for FCA unimplemented"); if (isHandledGCPointerType(Arg->getType())) DeoptValues.push_back(Arg); } + insertUseHolderAfter(CS, DeoptValues, Holders); } @@ -2471,6 +2556,16 @@ DominatorTree &DT = getAnalysis(F).getDomTree(); + auto NeedsRewrite = [](Instruction &I) { + if (UseDeoptBundles) { + if (ImmutableCallSite CS = ImmutableCallSite(&I)) + return !CallsGCLeafFunction(CS); + return false; + } + + return isStatepoint(I); + }; + // Gather all the statepoints which need rewritten. Be careful to only // consider those in reachable code since we need to ask dominance queries // when rewriting. We'll delete the unreachable ones in a moment. @@ -2478,7 +2573,7 @@ bool HasUnreachableStatepoint = false; for (Instruction &I : instructions(F)) { // TODO: only the ones with the flag set! - if (isStatepoint(I)) { + if (NeedsRewrite(I)) { if (DT.isReachableFromEntry(I.getParent())) ParsePointNeeded.push_back(CallSite(&I)); else Index: test/Transforms/RewriteStatepointsForGC/deopt-bundles/basic.ll =================================================================== --- /dev/null +++ test/Transforms/RewriteStatepointsForGC/deopt-bundles/basic.ll @@ -0,0 +1,65 @@ +; RUN: opt -S -rewrite-statepoints-for-gc -rs4gc-use-deopt-bundles < %s | FileCheck %s + +declare void @g() +declare i32 @h() + +define i32 addrspace(1)* @f0(i32 addrspace(1)* %arg) gc "statepoint-example" { +; CHECK-LABEL: @f0( + entry: +; CHECK: [[TOKEN_0:%[^ ]+]] = call i32 {{[^@]*}} @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg) + call void @g() [ "deopt"(i32 100) ] + +; CHECK: %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 [[TOKEN_0]], i32 8, i32 8) + ret i32 addrspace(1)* %arg +} + +define i32 addrspace(1)* @f1(i32 addrspace(1)* %arg) gc "statepoint-example" personality i32 8 { +; CHECK-LABEL: @f1( + entry: +; CHECK: [[TOKEN_1:%[^ ]+]] = invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg) + invoke void @g() [ "deopt"(i32 100) ] to label %normal_dest unwind label %unwind_dest + + normal_dest: +; CHECK: %arg.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 [[TOKEN_1]], i32 8, i32 8) + ret i32 addrspace(1)* %arg + + unwind_dest: + %lpad = landingpad { i8*, i32 } cleanup + resume { i8*, i32 } undef +} + +define i32 addrspace(1)* @f2(i32 addrspace(1)* %arg) gc "statepoint-example" { +; CHECK-LABEL: @f2( + entry: +; CHECK: [[TOKEN_2:%[^ ]+]] = call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 2882400000, i32 0, i32 ()* @h, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg) + %val = call i32 @h() [ "deopt"(i32 100) ] + +; CHECK: [[RESULT_F2:%[^ ]+]] = call i32 @llvm.experimental.gc.result.i32(i32 [[TOKEN_2]]) +; CHECK: %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 [[TOKEN_2]], i32 8, i32 8) +; CHECK: %arg.relocated.casted = bitcast i8 addrspace(1)* %arg.relocated to i32 addrspace(1)* + + store i32 %val, i32 addrspace(1)* %arg +; CHECK: store i32 [[RESULT_F2]], i32 addrspace(1)* %arg.relocated.casted + ret i32 addrspace(1)* %arg +} + +define i32 addrspace(1)* @f3(i32 addrspace(1)* %arg) gc "statepoint-example" personality i32 8 { +; CHECK-LABEL: @f3( + entry: +; CHECK: [[TOKEN_3:%[^ ]+]] = invoke i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 2882400000, i32 0, i32 ()* @h, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg) + %val = invoke i32 @h() [ "deopt"(i32 100) ] to label %normal_dest unwind label %unwind_dest + + normal_dest: +; CHECK: [[RESULT_F3:%[^ ]+]] = call i32 @llvm.experimental.gc.result.i32(i32 [[TOKEN_3]]) +; CHECK: %arg.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 [[TOKEN_3]], i32 8, i32 8) +; CHECK %arg.relocated1.casted = bitcast i8 addrspace(1)* %arg.relocated1 to i32 addrspace(1)* + + store i32 %val, i32 addrspace(1)* %arg + +; CHECK: store i32 [[RESULT_F3]], i32 addrspace(1)* %arg.relocated1.casted + ret i32 addrspace(1)* %arg + + unwind_dest: + %lpad = landingpad { i8*, i32 } cleanup + resume { i8*, i32 } undef +} Index: test/Transforms/RewriteStatepointsForGC/relocation.ll =================================================================== --- test/Transforms/RewriteStatepointsForGC/relocation.ll +++ test/Transforms/RewriteStatepointsForGC/relocation.ll @@ -94,7 +94,7 @@ ; CHECK-LABEL: join: ; CHECK: phi i8 addrspace(1)* ; CHECK-DAG: [ %arg.relocated, %if_branch ] -; CHECK-DAG: [ %arg.relocated4, %else_branch ] +; CHECK-DAG: [ %arg.relocated3, %else_branch ] ; CHECK-NOT: phi call void (i8 addrspace(1)*) @some_call(i8 addrspace(1)* %arg) ret void