Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -483,11 +483,13 @@ def int_experimental_patchpoint_void : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, - llvm_vararg_ty]>; + llvm_vararg_ty], + [Throws]>; def int_experimental_patchpoint_i64 : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, - llvm_vararg_ty]>; + llvm_vararg_ty], + [Throws]>; //===-------------------------- Other Intrinsics --------------------------===// // Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -634,11 +634,13 @@ void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad = nullptr); - std::pair LowerCallOperands(const CallInst &CI, - unsigned ArgIdx, - unsigned NumArgs, - SDValue Callee, - bool useVoidTy = false); + std::pair lowerCallOperands( + ImmutableCallSite CS, + unsigned ArgIdx, + unsigned NumArgs, + SDValue Callee, + bool UseVoidTy = false, + MachineBasicBlock *LandingPad = nullptr); /// UpdateSplitBlock - When an MBB was split during scheduling, update the /// references that need to refer to the last resulting block. @@ -778,7 +780,8 @@ void visitVAEnd(const CallInst &I); void visitVACopy(const CallInst &I); void visitStackmap(const CallInst &I); - void visitPatchpoint(const CallInst &I); + void visitPatchpoint(ImmutableCallSite CS, + MachineBasicBlock *LandingPad = nullptr); void visitUserOp1(const Instruction &I) { llvm_unreachable("UserOp1 should not exist at instruction selection time!"); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2008,8 +2008,17 @@ if (isa(Callee)) visitInlineAsm(&I); else if (Fn && Fn->isIntrinsic()) { - assert(Fn->getIntrinsicID() == Intrinsic::donothing); - // Ignore invokes to @llvm.donothing: jump directly to the next BB. + switch (Fn->getIntrinsicID()) { + default: + llvm_unreachable("Cannot invoke this intrinsic"); + case Intrinsic::donothing: + // Ignore invokes to @llvm.donothing: jump directly to the next BB. + break; + case Intrinsic::experimental_patchpoint_void: + case Intrinsic::experimental_patchpoint_i64: + visitPatchpoint(ImmutableCallSite(&I), LandingPad); + break; + } } else LowerCallTo(&I, getValue(Callee), false, LandingPad); @@ -5429,7 +5438,7 @@ } case Intrinsic::experimental_patchpoint_void: case Intrinsic::experimental_patchpoint_i64: { - visitPatchpoint(I); + visitPatchpoint(ImmutableCallSite(&I)); return nullptr; } } @@ -6786,18 +6795,18 @@ /// convention or require stack pointer adjustment. Only a subset of the /// intrinsic's operands need to participate in the calling convention. std::pair -SelectionDAGBuilder::LowerCallOperands(const CallInst &CI, unsigned ArgIdx, +SelectionDAGBuilder::lowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, - bool useVoidTy) { + bool UseVoidTy, + MachineBasicBlock *LandingPad) { TargetLowering::ArgListTy Args; Args.reserve(NumArgs); // Populate the argument list. // Attributes for args start at offset 1, after the return attribute. - ImmutableCallSite CS(&CI); for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1; ArgI != ArgE; ++ArgI) { - const Value *V = CI.getOperand(ArgI); + const Value *V = CS->getOperand(ArgI); assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); @@ -6808,13 +6817,13 @@ Args.push_back(Entry); } - Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CI.getType(); + Type *retTy = UseVoidTy ? Type::getVoidTy(*DAG.getContext()) : CS->getType(); TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot()) - .setCallee(CI.getCallingConv(), retTy, Callee, std::move(Args), NumArgs) - .setDiscardResult(!CI.use_empty()); + .setCallee(CS.getCallingConv(), retTy, Callee, std::move(Args), NumArgs) + .setDiscardResult(CS->use_empty()); - return lowerInvokable(CLI, nullptr); + return lowerInvokable(CLI, LandingPad); } /// \brief Add a stack map intrinsic call's live variable operands to a stackmap @@ -6834,11 +6843,11 @@ /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were /// only available in a register, then the runtime would need to trap when /// execution reaches the StackMap in order to read the alloca's location. -static void addStackMapLiveVars(const CallInst &CI, unsigned StartIdx, +static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, SmallVectorImpl &Ops, SelectionDAGBuilder &Builder) { - for (unsigned i = StartIdx, e = CI.getNumArgOperands(); i != e; ++i) { - SDValue OpVal = Builder.getValue(CI.getArgOperand(i)); + for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) { + SDValue OpVal = Builder.getValue(CS.getArgument(i)); if (ConstantSDNode *C = dyn_cast(OpVal)) { Ops.push_back( Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); @@ -6889,7 +6898,7 @@ cast(NBytesVal)->getZExtValue(), MVT::i32)); // Push live variables for the stack map. - addStackMapLiveVars(CI, 2, Ops, *this); + addStackMapLiveVars(ImmutableCallSite(&CI), 2, Ops, *this); // We are not pushing any register mask info here on the operands list, // because the stackmap doesn't clobber anything. @@ -6916,7 +6925,8 @@ } /// \brief Lower llvm.experimental.patchpoint directly to its target opcode. -void SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) { +void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS, + MachineBasicBlock *LandingPad) { // void|i64 @llvm.experimental.patchpoint.void|i64(i64 , // i32 , // i8* , @@ -6924,29 +6934,30 @@ // [Args...], // [live variables...]) - CallingConv::ID CC = CI.getCallingConv(); - bool isAnyRegCC = CC == CallingConv::AnyReg; - bool hasDef = !CI.getType()->isVoidTy(); - SDValue Callee = getValue(CI.getOperand(2)); // + CallingConv::ID CC = CS.getCallingConv(); + bool IsAnyRegCC = CC == CallingConv::AnyReg; + bool HasDef = !CS->getType()->isVoidTy(); + SDValue Callee = getValue(CS->getOperand(2)); // // Get the real number of arguments participating in the call - SDValue NArgVal = getValue(CI.getArgOperand(PatchPointOpers::NArgPos)); + SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos)); unsigned NumArgs = cast(NArgVal)->getZExtValue(); // Skip the four meta args: , , , // Intrinsics include all meta-operands up to but not including CC. unsigned NumMetaOpers = PatchPointOpers::CCPos; - assert(CI.getNumArgOperands() >= NumMetaOpers + NumArgs && + assert(CS.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic"); // For AnyRegCC the arguments are lowered later on manually. - unsigned NumCallArgs = isAnyRegCC ? 0 : NumArgs; + unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs; std::pair Result = - LowerCallOperands(CI, NumMetaOpers, NumCallArgs, Callee, isAnyRegCC); + lowerCallOperands(CS, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, + LandingPad); SDValue Chain = Result.second; SDNode *CallEnd = Chain.getNode(); - if (hasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) + if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) CallEnd = CallEnd->getOperand(0).getNode(); /// Get a call instruction from the call sequence chain. @@ -6954,16 +6965,16 @@ assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node."); SDNode *Call = CallEnd->getOperand(0).getNode(); - bool hasGlue = Call->getGluedNode(); + bool HasGlue = Call->getGluedNode(); // Replace the target specific call node with the patchable intrinsic. SmallVector Ops; // Add the and constants. - SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); + SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos)); Ops.push_back(DAG.getTargetConstant( cast(IDVal)->getZExtValue(), MVT::i64)); - SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); + SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos)); Ops.push_back(DAG.getTargetConstant( cast(NBytesVal)->getZExtValue(), MVT::i32)); @@ -6976,8 +6987,8 @@ // Adjust to account for any arguments that have been passed on the // stack instead. // Call Node: Chain, Target, {Args}, RegMask, [Glue] - unsigned NumCallRegArgs = Call->getNumOperands() - (hasGlue ? 4 : 3); - NumCallRegArgs = isAnyRegCC ? NumArgs : NumCallRegArgs; + unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3); + NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs; Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, MVT::i32)); // Add the calling convention @@ -6985,20 +6996,20 @@ // Add the arguments we omitted previously. The register allocator should // place these in any free register. - if (isAnyRegCC) + if (IsAnyRegCC) for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) - Ops.push_back(getValue(CI.getArgOperand(i))); + Ops.push_back(getValue(CS.getArgument(i))); // Push the arguments from the call instruction up to the register mask. - SDNode::op_iterator e = hasGlue ? Call->op_end()-2 : Call->op_end()-1; + SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1; for (SDNode::op_iterator i = Call->op_begin()+2; i != e; ++i) Ops.push_back(*i); // Push live variables for the stack map. - addStackMapLiveVars(CI, NumMetaOpers + NumArgs, Ops, *this); + addStackMapLiveVars(CS, NumMetaOpers + NumArgs, Ops, *this); // Push the register mask info. - if (hasGlue) + if (HasGlue) Ops.push_back(*(Call->op_end()-2)); else Ops.push_back(*(Call->op_end()-1)); @@ -7008,15 +7019,15 @@ Ops.push_back(*(Call->op_begin())); // Push the glue flag (last operand). - if (hasGlue) + if (HasGlue) Ops.push_back(*(Call->op_end()-1)); SDVTList NodeTys; - if (isAnyRegCC && hasDef) { + if (IsAnyRegCC && HasDef) { // Create the return types based on the intrinsic definition const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SmallVector ValueVTs; - ComputeValueVTs(TLI, CI.getType(), ValueVTs); + ComputeValueVTs(TLI, CS->getType(), ValueVTs); assert(ValueVTs.size() == 1 && "Expected only one return value type."); // There is always a chain and a glue type at the end @@ -7031,18 +7042,18 @@ getCurSDLoc(), NodeTys, Ops); // Update the NodeMap. - if (hasDef) { - if (isAnyRegCC) - setValue(&CI, SDValue(MN, 0)); + if (HasDef) { + if (IsAnyRegCC) + setValue(CS.getInstruction(), SDValue(MN, 0)); else - setValue(&CI, Result.first); + setValue(CS.getInstruction(), Result.first); } // Fixup the consumers of the intrinsic. The chain and glue may be used in the // call sequence. Furthermore the location of the chain and glue can change // when the AnyReg calling convention is used and the intrinsic returns a // value. - if (isAnyRegCC && hasDef) { + if (IsAnyRegCC && HasDef) { SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)}; SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)}; DAG.ReplaceAllUsesOfValuesWith(From, To, 2); Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -2212,11 +2212,15 @@ if (Function *F = dyn_cast(I.getOperand(i))) { // Check to make sure that the "address of" an intrinsic function is never // taken. - Assert1(!F->isIntrinsic() || i == (isa(I) ? e-1 : 0), + Assert1(!F->isIntrinsic() || i == (isa(I) ? e-1 : + isa(I) ? e-3 : 0), "Cannot take the address of an intrinsic!", &I); Assert1(!F->isIntrinsic() || isa(I) || - F->getIntrinsicID() == Intrinsic::donothing, - "Cannot invoke an intrinsinc other than donothing", &I); + F->getIntrinsicID() == Intrinsic::donothing || + F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void || + F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64, + "Cannot invoke an intrinsinc other than" + " donothing or patchpoint", &I); Assert1(F->getParent() == M, "Referencing function in another module!", &I); } else if (BasicBlock *OpBB = dyn_cast(I.getOperand(i))) { Index: test/CodeGen/X86/patchpoint-invoke.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/patchpoint-invoke.ll @@ -0,0 +1,65 @@ +; RUN: llc -mtriple=x86_64-unknown-linux -mcpu=corei7 < %s | FileCheck %s + +; Test invoking of patchpoints +; +define i64 @patchpoint_invoke(i64 %p1, i64 %p2) { +entry: +; CHECK-LABEL: patchpoint_invoke: +; CHECK-NEXT: .cfi_startproc +; CHECK: [[FUNC_BEGIN:.L.*]]: +; CHECK: .cfi_lsda 3, [[EXCEPTION_LABEL:.L[^ ]*]] +; CHECK: pushq %rbp + +; Unfortunately, hardcode the name of the label that begins the patchpoint: +; CHECK: .Ltmp0: +; CHECK: movabsq $-559038736, %r11 +; CHECK-NEXT: callq *%r11 +; CHECK-NEXT: xchgw %ax, %ax +; CHECK-NEXT: [[PP_END:.L.*]]: +; CHECK: ret + %resolveCall = inttoptr i64 -559038736 to i8* + %result = invoke i64 (i64, i32, i8*, i32, ...)* @llvm.experimental.patchpoint.i64(i64 2, i32 15, i8* %resolveCall, i32 1, i64 %p1, i64 %p2) + to label %success unwind label %threw + +success: + ret i64 %result + +threw: + %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) + catch i8* null + ret i64 0 +} + +; Verify that the exception table was emitted: +; CHECK: [[EXCEPTION_LABEL]]: +; CHECK-NEXT: .byte 255 +; CHECK-NEXT: .byte 3 +; CHECK-NEXT: .byte 21 +; CHECK-NEXT: .byte 3 +; CHECK-NEXT: .byte 13 +; Verify that the unwind data covers the entire patchpoint region: +; CHECK-NEXT: [[RANGE_OFFSET:.L[^ ]*]] = .Ltmp0-[[FUNC_BEGIN]] +; CHECK-NEXT: .long [[RANGE_OFFSET]] +; CHECK-NEXT: [[RANGE_LENGTH:.L[^ ]*]] = [[PP_END]]-.Ltmp0 +; CHECK-NEXT: .long [[RANGE_LENGTH]] + + +; Verify that the stackmap section got emitted: +; CHECK-LABEL: __LLVM_StackMaps: +; Header +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 0 +; Num Functions +; CHECK-NEXT: .long 1 +; Num LargeConstants +; CHECK-NEXT: .long 0 +; Num Callsites +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .quad patchpoint_invoke + + +declare void @llvm.experimental.stackmap(i64, i32, ...) +declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...) +declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...) +declare i32 @__gxx_personality_v0(...) Index: test/Verifier/invoke.ll =================================================================== --- test/Verifier/invoke.ll +++ test/Verifier/invoke.ll @@ -46,7 +46,7 @@ define i8 @f2() { entry: -; CHECK: Cannot invoke an intrinsinc other than donothing +; CHECK: Cannot invoke an intrinsinc other than donothing or patchpoint invoke void @llvm.trap() to label %cont unwind label %lpad