Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -21,6 +21,7 @@ #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Target/TargetLowering.h" #include namespace llvm { @@ -633,17 +634,23 @@ void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad = nullptr); - std::pair LowerCallOperands(const CallInst &CI, - unsigned ArgIdx, - unsigned NumArgs, - SDValue Callee, - bool useVoidTy = false); + std::pair LowerCallOperands( + ImmutableCallSite CS, + unsigned ArgIdx, + unsigned NumArgs, + SDValue Callee, + bool useVoidTy = false, + MachineBasicBlock *LandingPad = nullptr); /// UpdateSplitBlock - When an MBB was split during scheduling, update the /// references that need to refer to the last resulting block. void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last); private: + std::pair LowerInvokable( + TargetLowering::CallLoweringInfo &CLI, + MachineBasicBlock *LandingPad); + // Terminator instructions. void visitRet(const ReturnInst &I); void visitBr(const BranchInst &I); @@ -774,7 +781,7 @@ void visitVAEnd(const CallInst &I); void visitVACopy(const CallInst &I); void visitStackmap(const CallInst &I); - void visitPatchpoint(const CallInst &I); + void visitPatchpoint(ImmutableCallSite CS, MachineBasicBlock *LandingPad); void visitUserOp1(const Instruction &I) { llvm_unreachable("UserOp1 should not exist at instruction selection time!"); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2015,13 +2015,23 @@ const Value *Callee(I.getCalledValue()); const Function *Fn = dyn_cast(Callee); - if (isa(Callee)) + if (isa(Callee)) { visitInlineAsm(&I); - else if (Fn && Fn->isIntrinsic()) { - assert(Fn->getIntrinsicID() == Intrinsic::donothing); - // Ignore invokes to @llvm.donothing: jump directly to the next BB. - } else + } else if (Fn && Fn->isIntrinsic()) { + switch (Fn->getIntrinsicID()) { + default: + llvm_unreachable("Cannot invoke this intrinsic"); + case Intrinsic::donothing: + // Ignore invokes to @llvm.donothing: jump directly to the next BB. + break; + case Intrinsic::experimental_patchpoint_void: + case Intrinsic::experimental_patchpoint_i64: + visitPatchpoint(ImmutableCallSite(&I), LandingPad); + break; + } + } else { LowerCallTo(&I, getValue(Callee), false, LandingPad); + } // If the value of the invoke is used outside of its defining block, make it // available as a virtual register. @@ -5527,42 +5537,18 @@ } case Intrinsic::experimental_patchpoint_void: case Intrinsic::experimental_patchpoint_i64: { - visitPatchpoint(I); + visitPatchpoint(ImmutableCallSite(&I), nullptr); return nullptr; } } } -void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, - bool isTailCall, - MachineBasicBlock *LandingPad) { - const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering(); - PointerType *PT = cast(CS.getCalledValue()->getType()); - FunctionType *FTy = cast(PT->getElementType()); - Type *RetTy = FTy->getReturnType(); +std::pair +SelectionDAGBuilder::LowerInvokable(TargetLowering::CallLoweringInfo &CLI, + MachineBasicBlock *LandingPad) { MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); MCSymbol *BeginLabel = nullptr; - TargetLowering::ArgListTy Args; - TargetLowering::ArgListEntry Entry; - Args.reserve(CS.arg_size()); - - for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); - i != e; ++i) { - const Value *V = *i; - - // Skip empty types - if (V->getType()->isEmptyTy()) - continue; - - SDValue ArgNode = getValue(V); - Entry.Node = ArgNode; Entry.Ty = V->getType(); - - // Skip the first return-type Attribute to get to params. - Entry.setAttributes(&CS, i - CS.arg_begin() + 1); - Args.push_back(Entry); - } - if (LandingPad) { // Insert a label before the invoke call to mark the try range. This can be // used to detect deletion of the invoke via the MachineModuleInfo. @@ -5583,24 +5569,17 @@ // this call might not return. (void)getRoot(); DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel)); - } - // Check if target-independent constraints permit a tail call here. - // Target-dependent constraints are checked within TLI->LowerCallTo. - if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget())) - isTailCall = false; + CLI.setChain(getRoot()); + } - TargetLowering::CallLoweringInfo CLI(DAG); - CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot()) - .setCallee(RetTy, FTy, Callee, std::move(Args), CS).setTailCall(isTailCall); + const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering(); + std::pair Result = TLI->LowerCallTo(CLI); - std::pair Result = TLI->LowerCallTo(CLI); - assert((isTailCall || Result.second.getNode()) && + assert((CLI.IsTailCall || Result.second.getNode()) && "Non-null chain expected with non-tail call!"); assert((Result.second.getNode() || !Result.first.getNode()) && "Null value expected with tail call!"); - if (Result.first.getNode()) - setValue(CS.getInstruction(), Result.first); if (!Result.second.getNode()) { // As a special case, a null chain means that a tail call has been emitted @@ -5623,6 +5602,50 @@ // Inform MachineModuleInfo of range. MMI.addInvoke(LandingPad, BeginLabel, EndLabel); } + + return Result; +} + +void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, + bool isTailCall, + MachineBasicBlock *LandingPad) { + PointerType *PT = cast(CS.getCalledValue()->getType()); + FunctionType *FTy = cast(PT->getElementType()); + Type *RetTy = FTy->getReturnType(); + + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + Args.reserve(CS.arg_size()); + + for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); + i != e; ++i) { + const Value *V = *i; + + // Skip empty types + if (V->getType()->isEmptyTy()) + continue; + + SDValue ArgNode = getValue(V); + Entry.Node = ArgNode; Entry.Ty = V->getType(); + + // Skip the first return-type Attribute to get to params. + Entry.setAttributes(&CS, i - CS.arg_begin() + 1); + Args.push_back(Entry); + } + + // Check if target-independent constraints permit a tail call here. + // Target-dependent constraints are checked within TLI->LowerCallTo. + if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget())) + isTailCall = false; + + TargetLowering::CallLoweringInfo CLI(DAG); + CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot()) + .setCallee(RetTy, FTy, Callee, std::move(Args), CS) + .setTailCall(isTailCall); + std::pair Result = LowerInvokable(CLI, LandingPad); + + if (Result.first.getNode()) + setValue(CS.getInstruction(), Result.first); } /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the @@ -6873,18 +6896,18 @@ /// convention or require stack pointer adjustment. Only a subset of the /// intrinsic's operands need to participate in the calling convention. std::pair -SelectionDAGBuilder::LowerCallOperands(const CallInst &CI, unsigned ArgIdx, +SelectionDAGBuilder::LowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, - bool useVoidTy) { + bool useVoidTy, + MachineBasicBlock *LandingPad) { TargetLowering::ArgListTy Args; Args.reserve(NumArgs); // Populate the argument list. // Attributes for args start at offset 1, after the return attribute. - ImmutableCallSite CS(&CI); for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1; ArgI != ArgE; ++ArgI) { - const Value *V = CI.getOperand(ArgI); + const Value *V = CS->getOperand(ArgI); assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); @@ -6895,14 +6918,13 @@ Args.push_back(Entry); } - Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CI.getType(); + Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CS->getType(); TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot()) - .setCallee(CI.getCallingConv(), retTy, Callee, std::move(Args), NumArgs) - .setDiscardResult(!CI.use_empty()); + .setCallee(CS.getCallingConv(), retTy, Callee, std::move(Args), NumArgs) + .setDiscardResult(CS->use_empty()); - const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering(); - return TLI->LowerCallTo(CLI); + return LowerInvokable(CLI, LandingPad); } /// \brief Add a stack map intrinsic call's live variable operands to a stackmap @@ -6922,11 +6944,11 @@ /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were /// only available in a register, then the runtime would need to trap when /// execution reaches the StackMap in order to read the alloca's location. -static void addStackMapLiveVars(const CallInst &CI, unsigned StartIdx, +static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, SmallVectorImpl &Ops, SelectionDAGBuilder &Builder) { - for (unsigned i = StartIdx, e = CI.getNumArgOperands(); i != e; ++i) { - SDValue OpVal = Builder.getValue(CI.getArgOperand(i)); + for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) { + SDValue OpVal = Builder.getValue(CS.getArgument(i)); if (ConstantSDNode *C = dyn_cast(OpVal)) { Ops.push_back( Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); @@ -6977,7 +6999,7 @@ cast(NBytesVal)->getZExtValue(), MVT::i32)); // Push live variables for the stack map. - addStackMapLiveVars(CI, 2, Ops, *this); + addStackMapLiveVars(ImmutableCallSite(&CI), 2, Ops, *this); // We are not pushing any register mask info here on the operands list, // because the stackmap doesn't clobber anything. @@ -7004,7 +7026,8 @@ } /// \brief Lower llvm.experimental.patchpoint directly to its target opcode. -void SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) { +void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS, + MachineBasicBlock *LandingPad) { // void|i64 @llvm.experimental.patchpoint.void|i64(i64 , // i32 , // i8* , @@ -7012,29 +7035,29 @@ // [Args...], // [live variables...]) - CallingConv::ID CC = CI.getCallingConv(); + CallingConv::ID CC = CS.getCallingConv(); bool isAnyRegCC = CC == CallingConv::AnyReg; - bool hasDef = !CI.getType()->isVoidTy(); - SDValue Callee = getValue(CI.getOperand(2)); // + bool hasDef = !CS->getType()->isVoidTy(); + SDValue Callee = getValue(CS->getOperand(2)); // // Get the real number of arguments participating in the call - SDValue NArgVal = getValue(CI.getArgOperand(PatchPointOpers::NArgPos)); + SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos)); unsigned NumArgs = cast(NArgVal)->getZExtValue(); // Skip the four meta args: , , , // Intrinsics include all meta-operands up to but not including CC. unsigned NumMetaOpers = PatchPointOpers::CCPos; - assert(CI.getNumArgOperands() >= NumMetaOpers + NumArgs && + assert(CS.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic"); // For AnyRegCC the arguments are lowered later on manually. unsigned NumCallArgs = isAnyRegCC ? 0 : NumArgs; std::pair Result = - LowerCallOperands(CI, NumMetaOpers, NumCallArgs, Callee, isAnyRegCC); + LowerCallOperands(CS, NumMetaOpers, NumCallArgs, Callee, isAnyRegCC, + LandingPad); // Set the root to the target-lowered call chain. SDValue Chain = Result.second; - DAG.setRoot(Chain); SDNode *CallEnd = Chain.getNode(); if (hasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) @@ -7051,10 +7074,10 @@ SmallVector Ops; // Add the and constants. - SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); + SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos)); Ops.push_back(DAG.getTargetConstant( cast(IDVal)->getZExtValue(), MVT::i64)); - SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); + SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos)); Ops.push_back(DAG.getTargetConstant( cast(NBytesVal)->getZExtValue(), MVT::i32)); @@ -7078,7 +7101,7 @@ // place these in any free register. if (isAnyRegCC) for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) - Ops.push_back(getValue(CI.getArgOperand(i))); + Ops.push_back(getValue(CS.getArgument(i))); // Push the arguments from the call instruction up to the register mask. SDNode::op_iterator e = hasGlue ? Call->op_end()-2 : Call->op_end()-1; @@ -7086,7 +7109,7 @@ Ops.push_back(*i); // Push live variables for the stack map. - addStackMapLiveVars(CI, NumMetaOpers + NumArgs, Ops, *this); + addStackMapLiveVars(CS, NumMetaOpers + NumArgs, Ops, *this); // Push the register mask info. if (hasGlue) @@ -7107,7 +7130,7 @@ // Create the return types based on the intrinsic definition const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SmallVector ValueVTs; - ComputeValueVTs(TLI, CI.getType(), ValueVTs); + ComputeValueVTs(TLI, CS->getType(), ValueVTs); assert(ValueVTs.size() == 1 && "Expected only one return value type."); // There is always a chain and a glue type at the end @@ -7124,9 +7147,9 @@ // Update the NodeMap. if (hasDef) { if (isAnyRegCC) - setValue(&CI, SDValue(MN, 0)); + setValue(CS.getInstruction(), SDValue(MN, 0)); else - setValue(&CI, Result.first); + setValue(CS.getInstruction(), Result.first); } // Fixup the consumers of the intrinsic. The chain and glue may be used in the Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -2212,11 +2212,15 @@ if (Function *F = dyn_cast(I.getOperand(i))) { // Check to make sure that the "address of" an intrinsic function is never // taken. - Assert1(!F->isIntrinsic() || i == (isa(I) ? e-1 : 0), + Assert1(!F->isIntrinsic() || i == (isa(I) ? e-1 : + isa(I) ? e-3 : 0), "Cannot take the address of an intrinsic!", &I); Assert1(!F->isIntrinsic() || isa(I) || - F->getIntrinsicID() == Intrinsic::donothing, - "Cannot invoke an intrinsinc other than donothing", &I); + F->getIntrinsicID() == Intrinsic::donothing || + F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void || + F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64, + "Cannot invoke an intrinsinc other than" + " donothing or patchpoint", &I); Assert1(F->getParent() == M, "Referencing function in another module!", &I); } else if (BasicBlock *OpBB = dyn_cast(I.getOperand(i))) { Index: test/CodeGen/X86/patchpoint-invoke.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/patchpoint-invoke.ll @@ -0,0 +1,65 @@ +; RUN: llc -mtriple=x86_64-linux-generic -mcpu=corei7 < %s | FileCheck %s + +; Test invoking of patchpoints +; +define i64 @patchpoint_invoke(i64 %p1, i64 %p2) { +entry: +; CHECK-LABEL: patchpoint_invoke: +; CHECK-NEXT: .cfi_startproc +; CHECK: .Leh_func_begin0: +; CHECK: .cfi_lsda 3, .Lexception0 +; CHECK: pushq %rbp + +; CHECK: .Ltmp0: +; CHECK-NEXT: .Ltmp6: +; CHECK-NEXT: movabsq $-559038736, %r11 +; CHECK-NEXT: callq *%r11 +; CHECK-NEXT: xchgw %ax, %ax +; CHECK-NEXT: .Ltmp1: +; CHECK: ret + %resolveCall = inttoptr i64 -559038736 to i8* + %result = invoke i64 (i64, i32, i8*, i32, ...)* @llvm.experimental.patchpoint.i64(i64 2, i32 15, i8* %resolveCall, i32 1, i64 %p1, i64 %p2) + to label %success unwind label %threw + +success: + ret i64 %result + +threw: + %0 = landingpad { i8*, i32 } personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) + catch i8* null + ret i64 0 +} + +; Verify that the exception table was emitted: +; CHECK-LABEL: .Lexception0: +; Header? +; CHECK-NEXT: .byte 255 +; CHECK-NEXT: .byte 3 +; CHECK-NEXT: .byte 21 +; CHECK-NEXT: .byte 3 +; CHECK-NEXT: .byte 13 +; CHECK-NEXT: .Lset0 = .Ltmp0-.Leh_func_begin0 +; CHECK-NEXT: .long .Lset0 +; CHECK-NEXT: .Lset1 = .Ltmp1-.Ltmp0 +; CHECK-NEXT: .long .Lset1 + + +; Verify that the stackmap section got emitted: +; CHECK-LABEL: __LLVM_StackMaps: +; Header +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 0 +; Num Functions +; CHECK-NEXT: .long 1 +; Num LargeConstants +; CHECK-NEXT: .long 0 +; Num Callsites +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .quad patchpoint_invoke + + +declare void @llvm.experimental.stackmap(i64, i32, ...) +declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...) +declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...) +declare i32 @__gxx_personality_v0(...) Index: test/Verifier/invoke.ll =================================================================== --- test/Verifier/invoke.ll +++ test/Verifier/invoke.ll @@ -46,7 +46,7 @@ define i8 @f2() { entry: -; CHECK: Cannot invoke an intrinsinc other than donothing +; CHECK: Cannot invoke an intrinsinc other than donothing or patchpoint invoke void @llvm.trap() to label %cont unwind label %lpad