Index: include/llvm/IR/Statepoint.h =================================================================== --- include/llvm/IR/Statepoint.h +++ include/llvm/IR/Statepoint.h @@ -411,6 +411,7 @@ Optional StatepointID; static const uint64_t DefaultStatepointID = 0xABCDEF00; + static const uint64_t DeoptBundleStatepointID = 0xABCDEF0F; }; /// Parse out statepoint directives from the function attributes present in \p Index: lib/CodeGen/SelectionDAG/FastISel.cpp =================================================================== --- lib/CodeGen/SelectionDAG/FastISel.cpp +++ lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1351,6 +1351,12 @@ return false; } + // FastISel does not handle any operand bundles except OB_funclet. + if (ImmutableCallSite CS = ImmutableCallSite(I)) + for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) + if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet) + return false; + DbgLoc = I->getDebugLoc(); SavedInsertPt = FuncInfo.InsertPt; Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -772,12 +772,16 @@ }; /// Lower \p SLI into a STATEPOINT instruction. - SDValue LowerAsStatepoint(StatepointLoweringInfo &SLI); + SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SLI); // This function is responsible for the whole statepoint lowering process. // It uniformly handles invoke and call statepoints. void LowerStatepoint(ImmutableStatepoint Statepoint, const BasicBlock *EHPadBB = nullptr); + + void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee, + const BasicBlock *EHPadBB); + private: // Terminator instructions. void visitRet(const ReturnInst &I); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2127,6 +2127,15 @@ MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; const BasicBlock *EHPadBB = I.getSuccessor(1); +#ifndef NDEBUG + // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't + // have to do anything here to lower funclet bundles. + for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) + assert((I.getOperandBundleAt(i).isDeoptOperandBundle() || + I.getOperandBundleAt(i).isFuncletOperandBundle()) && + "Cannot lower invokes with arbitrary operand bundles yet!"); +#endif + const Value *Callee(I.getCalledValue()); const Function *Fn = dyn_cast(Callee); if (isa(Callee)) @@ -2146,8 +2155,15 @@ LowerStatepoint(ImmutableStatepoint(&I), EHPadBB); break; } - } else + } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) { + // Currently we do not lower any intrinsic calls with deopt operand bundles. + // Eventually we will support lowering the @llvm.experimental.deoptimize + // intrinsic, and right now there are no plans to support other intrinsics + // with deopt state. + LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB); + } else { LowerCallTo(&I, getValue(Callee), false, EHPadBB); + } // If the value of the invoke is used outside of its defining block, make it // available as a virtual register. @@ -6100,9 +6116,22 @@ RenameFn, DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); - // Check if we can potentially perform a tail call. More detailed checking is - // be done within LowerCallTo, after more information about the call is known. - LowerCallTo(&I, Callee, I.isTailCall()); +#ifndef NDEBUG + // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't + // have to do anything here to lower funclet bundles. + for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) + assert((I.getOperandBundleAt(i).isDeoptOperandBundle() || + I.getOperandBundleAt(i).isFuncletOperandBundle()) && + "Cannot lower calls with arbitrary operand bundles!"); +#endif + + if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) + LowerCallSiteWithDeoptBundle(&I, Callee, nullptr); + else + // Check if we can potentially perform a tail call. More detailed checking + // is be done within LowerCallTo, after more information about the call is + // known. + LowerCallTo(&I, Callee, I.isTailCall()); } namespace { Index: lib/CodeGen/SelectionDAG/StatepointLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -441,25 +441,30 @@ // Lower the deopt and gc arguments for this statepoint. Layout will be: // deopt argument length, deopt arguments.., gc arguments... #ifndef NDEBUG - // Check that each of the gc pointer and bases we've gotten out of the - // safepoint is something the strategy thinks might be a pointer (or vector - // of pointers) into the GC heap. This is basically just here to help catch - // errors during statepoint insertion. TODO: This should actually be in the - // Verifier, but we can't get to the GCStrategy from there (yet). - GCStrategy &S = Builder.GFI->getStrategy(); - for (const Value *V : SI.Bases) { - auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); - if (Opt.hasValue()) { - assert(Opt.getValue() && - "non gc managed base pointer found in statepoint"); + if (auto *GFI = Builder.GFI) { + // Check that each of the gc pointer and bases we've gotten out of the + // safepoint is something the strategy thinks might be a pointer (or vector + // of pointers) into the GC heap. This is basically just here to help catch + // errors during statepoint insertion. TODO: This should actually be in the + // Verifier, but we can't get to the GCStrategy from there (yet). + GCStrategy &S = GFI->getStrategy(); + for (const Value *V : SI.Bases) { + auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); + if (Opt.hasValue()) { + assert(Opt.getValue() && + "non gc managed base pointer found in statepoint"); + } } - } - for (const Value *V : SI.Ptrs) { - auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); - if (Opt.hasValue()) { - assert(Opt.getValue() && - "non gc managed derived pointer found in statepoint"); + for (const Value *V : SI.Ptrs) { + auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); + if (Opt.hasValue()) { + assert(Opt.getValue() && + "non gc managed derived pointer found in statepoint"); + } } + } else { + assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!"); + assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!"); } #endif @@ -550,7 +555,7 @@ } } -SDValue SelectionDAGBuilder::LowerAsStatepoint( +SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( SelectionDAGBuilder::StatepointLoweringInfo &SI) { // The basic scheme here is that information about both the original call and // the safepoint is encoded in the CallInst. We create a temporary call and @@ -794,7 +799,7 @@ SI.NumPatchBytes = ISP.getNumPatchBytes(); SI.EHPadBB = EHPadBB; - SDValue ReturnValue = LowerAsStatepoint(SI); + SDValue ReturnValue = LowerAsSTATEPOINT(SI); // Export the result value if needed const Instruction *GCResult = ISP.getGCResult(); @@ -830,6 +835,37 @@ } } +void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle( + ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) { + assert(CS.getNumOperandBundles() == 1 && + "Only deopt operand bundles can be lowered!"); + + StatepointLoweringInfo SI(DAG); + unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin(); + populateCallLoweringInfo(SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), + Callee, CS.getType(), false); + + auto DeoptBundle = CS.getOperandBundleAt(0); + assert(DeoptBundle.getTagID() == LLVMContext::OB_deopt && "Should be!"); + + unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID; + + auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes()); + SI.ID = SD.StatepointID.getValueOr(DefaultID); + SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0); + + SI.DeoptState = + ArrayRef(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end()); + SI.StatepointFlags = static_cast(StatepointFlags::None); + SI.EHPadBB = EHPadBB; + + if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) { + const Instruction *Inst = CS.getInstruction(); + ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal); + setValue(Inst, ReturnVal); + } +} + void SelectionDAGBuilder::visitGCResult(const CallInst &CI) { // The result value of the gc_result is simply the result of the actual // call. We've already emitted this, so just grab the value. Index: test/CodeGen/X86/deopt-bundles.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/deopt-bundles.ll @@ -0,0 +1,104 @@ +; RUN: llc -debug-only=stackmaps < %s 2>&1 | FileCheck %s +; RUN: llc -debug-only=stackmaps -O3 < %s 2>&1 | FileCheck %s +; REQUIRES: asserts + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.11.0" + + +; CHECK: Stack Maps: callsite 2882400015 +; CHECK-NEXT: Stack Maps: has 4 locations +; CHECK-NEXT: Stack Maps: Loc 0: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 1: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 2: Constant 1 [encoding: .byte 4, .byte 8, .short 0, .int 1] +; CHECK-NEXT: Stack Maps: Loc 3: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: has 0 live-out registers +; CHECK-NEXT: Stack Maps: callsite 4242 +; CHECK-NEXT: Stack Maps: has 4 locations +; CHECK-NEXT: Stack Maps: Loc 0: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 1: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 2: Constant 1 [encoding: .byte 4, .byte 8, .short 0, .int 1] +; CHECK-NEXT: Stack Maps: Loc 3: Constant 1 [encoding: .byte 4, .byte 8, .short 0, .int 1] +; CHECK-NEXT: Stack Maps: has 0 live-out registers +; CHECK-NEXT: Stack Maps: callsite 2882400015 +; CHECK-NEXT: Stack Maps: has 4 locations +; CHECK-NEXT: Stack Maps: Loc 0: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 1: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 2: Constant 1 [encoding: .byte 4, .byte 8, .short 0, .int 1] +; CHECK-NEXT: Stack Maps: Loc 3: Constant 2 [encoding: .byte 4, .byte 8, .short 0, .int 2] +; CHECK-NEXT: Stack Maps: has 0 live-out registers +; CHECK-NEXT: Stack Maps: callsite 2882400015 +; CHECK-NEXT: Stack Maps: has 4 locations +; CHECK-NEXT: Stack Maps: Loc 0: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 1: Constant 0 [encoding: .byte 4, .byte 8, .short 0, .int 0] +; CHECK-NEXT: Stack Maps: Loc 2: Constant 1 [encoding: .byte 4, .byte 8, .short 0, .int 1] +; CHECK-NEXT: Stack Maps: Loc 3: Constant 3 [encoding: .byte 4, .byte 8, .short 0, .int 3] +; CHECK-NEXT: Stack Maps: has 0 live-out registers + + +declare i32 @callee_0() +declare i32 @callee_1(i32) + +define i32 @caller_0() { +; CHECK-LABEL: _caller_0 +entry: + %v = call i32 @callee_0() [ "deopt"(i32 0) ] + %v2 = add i32 %v, 1 + ret i32 %v2 +; CHECK: callq _callee_0 +; CHECK: incl %eax +; CHECK: retq +} + +define i32 @caller_1() { +; CHECK-LABEL: _caller_1 +entry: + %v = call i32 @callee_1(i32 42) "statepoint-id"="4242" [ "deopt"(i32 1) ] + ret i32 %v +; CHECK: callq _callee_1 +; CHECK: popq %rcx +; CHECK: retq +} + +define i32 @invoker_0() personality i8 0 { +; CHECK-LABEL: _invoker_0 +entry: + %v = invoke i32 @callee_0() [ "deopt"(i32 2) ] + to label %normal unwind label %uw + +normal: + ret i32 %v + +uw: + %ehvals = landingpad { i8*, i32 } + cleanup + ret i32 1 +; CHECK: callq _callee_0 +; CHECK: popq %rcx +; CHECK: retq +; CHECK: movl $1, %eax +; CHECK: popq %rcx +; CHECK: retq +} + +define i32 @invoker_1() personality i8 0 { +; CHECK-LABEL: _invoker_1 +entry: + %v = invoke i32 @callee_1(i32 45) "statepoint-num-patch-bytes"="9" [ "deopt"(i32 3) ] + to label %normal unwind label %uw + +normal: + ret i32 %v + +uw: + %ehvals = landingpad { i8*, i32 } + cleanup + ret i32 1 +; CHECK: movl $45, %edi +; CHECK: nopw 512(%rax,%rax) +; CHECK: popq %rcx +; CHECK: retq +; CHECK: movl $1, %eax +; CHECK: popq %rcx +; CHECK: retq +}