Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -480,11 +480,13 @@ def int_experimental_patchpoint_void : Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, - llvm_vararg_ty]>; + llvm_vararg_ty], + [Throws]>; def int_experimental_patchpoint_i64 : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, - llvm_vararg_ty]>; + llvm_vararg_ty], + [Throws]>; //===-------------------------- Other Intrinsics --------------------------===// // Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -633,7 +633,7 @@ void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad = nullptr); - std::pair LowerCallOperands(const CallInst &CI, + std::pair LowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, @@ -774,7 +774,7 @@ void visitVAEnd(const CallInst &I); void visitVACopy(const CallInst &I); void visitStackmap(const CallInst &I); - void visitPatchpoint(const CallInst &I); + void visitPatchpoint(ImmutableCallSite CS); void visitUserOp1(const Instruction &I) { llvm_unreachable("UserOp1 should not exist at instruction selection time!"); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2014,13 +2014,18 @@ const Value *Callee(I.getCalledValue()); const Function *Fn = dyn_cast(Callee); - if (isa(Callee)) + if (isa(Callee)) { visitInlineAsm(&I); - else if (Fn && Fn->isIntrinsic()) { + } else if (Fn && Fn->isIntrinsic() && + (Fn->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 || + Fn->getIntrinsicID() == Intrinsic::experimental_patchpoint_void)) { + visitPatchpoint(ImmutableCallSite(&I)); + } else if (Fn && Fn->isIntrinsic()) { assert(Fn->getIntrinsicID() == Intrinsic::donothing); // Ignore invokes to @llvm.donothing: jump directly to the next BB. - } else + } else { LowerCallTo(&I, getValue(Callee), false, LandingPad); + } // If the value of the invoke is used outside of its defining block, make it // available as a virtual register. @@ -5519,7 +5524,7 @@ } case Intrinsic::experimental_patchpoint_void: case Intrinsic::experimental_patchpoint_i64: { - visitPatchpoint(I); + visitPatchpoint(ImmutableCallSite(&I)); return nullptr; } } @@ -6865,7 +6870,7 @@ /// convention or require stack pointer adjustment. Only a subset of the /// intrinsic's operands need to participate in the calling convention. std::pair -SelectionDAGBuilder::LowerCallOperands(const CallInst &CI, unsigned ArgIdx, +SelectionDAGBuilder::LowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, bool useVoidTy) { TargetLowering::ArgListTy Args; @@ -6873,10 +6878,9 @@ // Populate the argument list. // Attributes for args start at offset 1, after the return attribute. - ImmutableCallSite CS(&CI); for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1; ArgI != ArgE; ++ArgI) { - const Value *V = CI.getOperand(ArgI); + const Value *V = CS->getOperand(ArgI); assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); @@ -6887,11 +6891,11 @@ Args.push_back(Entry); } - Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CI.getType(); + Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(); TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot()) - .setCallee(CI.getCallingConv(), retTy, Callee, std::move(Args), NumArgs) - .setDiscardResult(!CI.use_empty()); + .setCallee(CS.getCallingConv(), retTy, Callee, std::move(Args), NumArgs) + .setDiscardResult(!CS->use_empty()); const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering(); return TLI->LowerCallTo(CLI); @@ -6914,11 +6918,11 @@ /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were /// only available in a register, then the runtime would need to trap when /// execution reaches the StackMap in order to read the alloca's location. -static void addStackMapLiveVars(const CallInst &CI, unsigned StartIdx, +static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, SmallVectorImpl &Ops, SelectionDAGBuilder &Builder) { - for (unsigned i = StartIdx, e = CI.getNumArgOperands(); i != e; ++i) { - SDValue OpVal = Builder.getValue(CI.getArgOperand(i)); + for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) { + SDValue OpVal = Builder.getValue(CS.getArgument(i)); if (ConstantSDNode *C = dyn_cast(OpVal)) { Ops.push_back( Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); @@ -6969,7 +6973,7 @@ cast(NBytesVal)->getZExtValue(), MVT::i32)); // Push live variables for the stack map. - addStackMapLiveVars(CI, 2, Ops, *this); + addStackMapLiveVars(ImmutableCallSite(&CI), 2, Ops, *this); // We are not pushing any register mask info here on the operands list, // because the stackmap doesn't clobber anything. @@ -6996,7 +7000,7 @@ } /// \brief Lower llvm.experimental.patchpoint directly to its target opcode. -void SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) { +void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS) { // void|i64 @llvm.experimental.patchpoint.void|i64(i64 , // i32 , // i8* , @@ -7004,30 +7008,71 @@ // [Args...], // [live variables...]) - CallingConv::ID CC = CI.getCallingConv(); + CallingConv::ID CC = CS.getCallingConv(); bool isAnyRegCC = CC == CallingConv::AnyReg; - bool hasDef = !CI.getType()->isVoidTy(); - SDValue Callee = getValue(CI.getOperand(2)); // + bool hasDef = !CS.getType()->isVoidTy(); + SDValue Callee = getValue(CS->getOperand(2)); // // Get the real number of arguments participating in the call - SDValue NArgVal = getValue(CI.getArgOperand(PatchPointOpers::NArgPos)); + SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos)); unsigned NumArgs = cast(NArgVal)->getZExtValue(); // Skip the four meta args: , , , // Intrinsics include all meta-operands up to but not including CC. unsigned NumMetaOpers = PatchPointOpers::CCPos; - assert(CI.getNumArgOperands() >= NumMetaOpers + NumArgs && + assert(CS.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic"); + MachineBasicBlock *LandingPad = nullptr; + if (const InvokeInst* Inv = dyn_cast(CS.getInstruction())) { + LandingPad = FuncInfo.MBBMap[Inv->getSuccessor(1)]; + } + + // FIXME Copied from SelectionDAGBuilder::LowerCallTo: + MCSymbol *BeginLabel = nullptr; + MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); + if (LandingPad) { + // Insert a label before the invoke call to mark the try range. This can be + // used to detect deletion of the invoke via the MachineModuleInfo. + BeginLabel = MMI.getContext().CreateTempSymbol(); + + // For SjLj, keep track of which landing pads go with which invokes + // so as to maintain the ordering of pads in the LSDA. + unsigned CallSiteIndex = MMI.getCurrentCallSite(); + if (CallSiteIndex) { + MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); + LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex); + + // Now that the call site is handled, stop tracking it. + MMI.setCurrentCallSite(0); + } + + // Both PendingLoads and PendingExports must be flushed here; + // this call might not return. + (void)getRoot(); + DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel)); + } + // For AnyRegCC the arguments are lowered later on manually. unsigned NumCallArgs = isAnyRegCC ? 0 : NumArgs; std::pair Result = - LowerCallOperands(CI, NumMetaOpers, NumCallArgs, Callee, isAnyRegCC); + LowerCallOperands(CS, NumMetaOpers, NumCallArgs, Callee, isAnyRegCC); // Set the root to the target-lowered call chain. SDValue Chain = Result.second; DAG.setRoot(Chain); + // FIXME Copied from SelectionDAGBuilder::LowerCallTo: + if (LandingPad) { + // Insert a label at the end of the invoke call to mark the try range. This + // can be used to detect deletion of the invoke via the MachineModuleInfo. + MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol(); + DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel)); + + // Inform MachineModuleInfo of range. + MMI.addInvoke(LandingPad, BeginLabel, EndLabel); + } + SDNode *CallEnd = Chain.getNode(); if (hasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) CallEnd = CallEnd->getOperand(0).getNode(); @@ -7043,10 +7088,10 @@ SmallVector Ops; // Add the and constants. - SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos)); + SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos)); Ops.push_back(DAG.getTargetConstant( cast(IDVal)->getZExtValue(), MVT::i64)); - SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos)); + SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos)); Ops.push_back(DAG.getTargetConstant( cast(NBytesVal)->getZExtValue(), MVT::i32)); @@ -7070,7 +7115,7 @@ // place these in any free register. if (isAnyRegCC) for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) - Ops.push_back(getValue(CI.getArgOperand(i))); + Ops.push_back(getValue(CS.getArgument(i))); // Push the arguments from the call instruction up to the register mask. SDNode::op_iterator e = hasGlue ? Call->op_end()-2 : Call->op_end()-1; @@ -7078,7 +7123,7 @@ Ops.push_back(*i); // Push live variables for the stack map. - addStackMapLiveVars(CI, NumMetaOpers + NumArgs, Ops, *this); + addStackMapLiveVars(CS, NumMetaOpers + NumArgs, Ops, *this); // Push the register mask info. if (hasGlue) @@ -7099,7 +7144,7 @@ // Create the return types based on the intrinsic definition const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SmallVector ValueVTs; - ComputeValueVTs(TLI, CI.getType(), ValueVTs); + ComputeValueVTs(TLI, CS.getType(), ValueVTs); assert(ValueVTs.size() == 1 && "Expected only one return value type."); // There is always a chain and a glue type at the end @@ -7116,9 +7161,9 @@ // Update the NodeMap. if (hasDef) { if (isAnyRegCC) - setValue(&CI, SDValue(MN, 0)); + setValue(CS.getInstruction(), SDValue(MN, 0)); else - setValue(&CI, Result.first); + setValue(CS.getInstruction(), Result.first); } // Fixup the consumers of the intrinsic. The chain and glue may be used in the Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -2215,8 +2215,10 @@ Assert1(!F->isIntrinsic() || i == (isa(I) ? e-1 : 0), "Cannot take the address of an intrinsic!", &I); Assert1(!F->isIntrinsic() || isa(I) || - F->getIntrinsicID() == Intrinsic::donothing, - "Cannot invoke an intrinsinc other than donothing", &I); + F->getIntrinsicID() == Intrinsic::donothing || + F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void || + F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64, + "Cannot invoke an intrinsinc other than donothing or patchpoint", &I); Assert1(F->getParent() == M, "Referencing function in another module!", &I); } else if (BasicBlock *OpBB = dyn_cast(I.getOperand(i))) {