Index: include/llvm/IR/Statepoint.h =================================================================== --- include/llvm/IR/Statepoint.h +++ include/llvm/IR/Statepoint.h @@ -72,6 +72,13 @@ ValueTy *actualCallee() { return StatepointCS.getArgument(0); } + /// Return the type of the value returned by the call underlying the + /// statepoint. + Type *actualReturnType() { + auto *FTy = cast( + cast(actualCallee()->getType())->getElementType()); + return FTy->getReturnType(); + } /// Number of arguments to be passed to the actual callee. int numCallArgs() { return cast(StatepointCS.getArgument(1))->getZExtValue(); @@ -82,14 +89,16 @@ return cast(StatepointCS.getArgument(3 + numCallArgs()))->getZExtValue(); } + int callArgsBeginOffset() { return 3; } + typename CallSiteTy::arg_iterator call_args_begin() { // 3 = callTarget, #callArgs, flag - int Offset = 3; + int Offset = callArgsBeginOffset(); assert(Offset <= (int)StatepointCS.arg_size()); return StatepointCS.arg_begin() + Offset; } typename CallSiteTy::arg_iterator call_args_end() { - int Offset = 3 + numCallArgs(); + int Offset = callArgsBeginOffset() + numCallArgs(); assert(Offset <= (int)StatepointCS.arg_size()); return StatepointCS.arg_begin() + Offset; } Index: lib/CodeGen/SelectionDAG/StatepointLowering.cpp =================================================================== --- lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -219,103 +219,6 @@ assert(Ptrs.size() == Relocs.size()); } -/// Extract call from statepoint, lower it and return pointer to the -/// call node. Also update NodeMap so that getValue(statepoint) will -/// reference lowered call result -static SDNode *lowerCallFromStatepoint(ImmutableStatepoint StatepointSite, - MachineBasicBlock *LandingPad, - SelectionDAGBuilder &Builder) { - - ImmutableCallSite CS(StatepointSite.getCallSite()); - - // Lower the actual call itself - This is a bit of a hack, but we want to - // avoid modifying the actual lowering code. This is similiar in intent to - // the LowerCallOperands mechanism used by PATCHPOINT, but is structured - // differently. Hopefully, this is slightly more robust w.r.t. calling - // convention, return values, and other function attributes. - Value *ActualCallee = const_cast(StatepointSite.actualCallee()); - - std::vector Args; - CallInst::const_op_iterator arg_begin = StatepointSite.call_args_begin(); - CallInst::const_op_iterator arg_end = StatepointSite.call_args_end(); - Args.insert(Args.end(), arg_begin, arg_end); - // TODO: remove the creation of a new instruction! We should not be - // modifying the IR (even temporarily) at this point. - CallInst *Tmp = CallInst::Create(ActualCallee, Args); - Tmp->setTailCall(CS.isTailCall()); - Tmp->setCallingConv(CS.getCallingConv()); - Tmp->setAttributes(CS.getAttributes()); - Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false, LandingPad); - - // Handle the return value of the call iff any. - const bool HasDef = !Tmp->getType()->isVoidTy(); - if (HasDef) { - if (CS.isInvoke()) { - // Result value will be used in different basic block for invokes - // so we need to export it now. But statepoint call has a different type - // than the actuall call. It means that standart exporting mechanism will - // create register of the wrong type. So instead we need to create - // register with correct type and save value into it manually. - // TODO: To eliminate this problem we can remove gc.result intrinsics - // completelly and make statepoint call to return a tuple. - unsigned reg = Builder.FuncInfo.CreateRegs(Tmp->getType()); - Builder.CopyValueToVirtualRegister(Tmp, reg); - Builder.FuncInfo.ValueMap[CS.getInstruction()] = reg; - } else { - // The value of the statepoint itself will be the value of call itself. - // We'll replace the actually call node shortly. gc_result will grab - // this value. - Builder.setValue(CS.getInstruction(), Builder.getValue(Tmp)); - } - } else { - // The token value is never used from here on, just generate a poison value - Builder.setValue(CS.getInstruction(), - Builder.DAG.getIntPtrConstant(-1, Builder.getCurSDLoc())); - } - // Remove the fake entry we created so we don't have a hanging reference - // after we delete this node. - Builder.removeValue(Tmp); - delete Tmp; - Tmp = nullptr; - - // Search for the call node - // The following code is essentially reverse engineering X86's - // LowerCallTo. - // We are expecting DAG to have the following form: - // ch = eh_label (only in case of invoke statepoint) - // ch, glue = callseq_start ch - // ch, glue = X86::Call ch, glue - // ch, glue = callseq_end ch, glue - // ch = eh_label ch (only in case of invoke statepoint) - // - // DAG root will be either last eh_label or callseq_end. - - SDNode *CallNode = nullptr; - - // We just emitted a call, so it should be last thing generated - SDValue Chain = Builder.DAG.getRoot(); - - // Find closest CALLSEQ_END walking back through lowered nodes if needed - SDNode *CallEnd = Chain.getNode(); - int Sanity = 0; - while (CallEnd->getOpcode() != ISD::CALLSEQ_END) { - assert(CallEnd->getNumOperands() >= 1 && - CallEnd->getOperand(0).getValueType() == MVT::Other); - - CallEnd = CallEnd->getOperand(0).getNode(); - - assert(Sanity < 20 && "should have found call end already"); - Sanity++; - } - assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && - "Expected a callseq node."); - assert(CallEnd->getGluedNode()); - - // Step back inside the CALLSEQ - CallNode = CallEnd->getGluedNode(); - return CallNode; -} - /// Callect all gc pointers coming into statepoint intrinsic, clean them up, /// and return two arrays: /// Bases - base pointers incoming to this statepoint @@ -585,8 +488,44 @@ SmallVector LoweredMetaArgs; lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this); - // Get call node, we will replace it later with statepoint - SDNode *CallNode = lowerCallFromStatepoint(ISP, LandingPad, *this); + SDValue ActualCallee = getValue(ISP.actualCallee()); + + // Handle immediate and symbolic callees. + if (auto *ConstCallee = dyn_cast(ActualCallee.getNode())) + ActualCallee = + DAG.getIntPtrConstant(ConstCallee->getZExtValue(), getCurSDLoc(), + /*isTarget=*/true); + else if (auto *SymbolicCallee = + dyn_cast(ActualCallee.getNode())) + ActualCallee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(), + SDLoc(SymbolicCallee), + SymbolicCallee->getValueType(0)); + + assert(CS.getCallingConv() != CallingConv::AnyReg && + "anyregcc is not supported on statepoints!"); + + Type *DefTy = ISP.actualReturnType(); + bool HasDef = !DefTy->isVoidTy(); + + SDValue ReturnValue, CallEndVal; + std::tie(ReturnValue, CallEndVal) = lowerCallOperands( + ISP.getCallSite(), ISP.callArgsBeginOffset(), ISP.numCallArgs(), + ActualCallee, DefTy, LandingPad, false /* IsPatchPoint */); + + SDNode *CallEnd = CallEndVal.getNode(); + if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) + CallEnd = CallEnd->getOperand(0).getNode(); + + // Get a call instruction from the call sequence chain. Tail calls are not + // allowed. + + if (CallEnd->getOpcode() != ISD::CALLSEQ_END) { + assert(CallEnd->getOpcode() == ISD::LOAD && "only other expected node!"); + CallEnd = CallEnd->getOperand(0).getNode(); + assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!"); + } + SDNode *Call = CallEnd->getOperand(0).getNode(); + bool HasGlue = Call->getGluedNode(); // Construct the actual STATEPOINT node with all the appropriate arguments // and return values. @@ -596,30 +535,17 @@ // and flags to be read-only. SmallVector Ops; - // Calculate and push starting position of vmstate arguments - // Call Node: Chain, Target, {Args}, RegMask, [Glue] - SDValue Glue; - if (CallNode->getGluedNode()) { - // Glue is always last operand - Glue = CallNode->getOperand(CallNode->getNumOperands() - 1); - } // Get number of arguments incoming directly into call node - unsigned NumCallRegArgs = - CallNode->getNumOperands() - (Glue.getNode() ? 4 : 3); + unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3); Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32)); // Add call target - SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0); - Ops.push_back(CallTarget); + Ops.push_back(ActualCallee); // Add call arguments // Get position of register mask in the call - SDNode::op_iterator RegMaskIt; - if (Glue.getNode()) - RegMaskIt = CallNode->op_end() - 2; - else - RegMaskIt = CallNode->op_end() - 1; - Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt); + SDNode::op_iterator RegMaskIt = Call->op_end() - (HasGlue ? 2 : 1); + Ops.insert(Ops.end(), Call->op_begin() + 2, RegMaskIt); // Add a leading constant argument with the Flags and the calling convention // masked together @@ -638,11 +564,11 @@ Ops.push_back(*RegMaskIt); // Add chain - Ops.push_back(CallNode->getOperand(0)); + Ops.push_back(Call->getOperand(0)); // Same for the glue, but we add it only if original call had it - if (Glue.getNode()) - Ops.push_back(Glue); + if (HasGlue) + Ops.push_back(*(Call->op_end() - 1)); // Compute return values. Provide a glue output since we consume one as // input. This allows someone else to chain off us as needed. @@ -651,10 +577,36 @@ SDNode *StatepointMCNode = DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops); + // Update the NodeMap. + if (CS.isInvoke()) { + // Result value will be used in different basic block for invokes so we need + // to export it now. But statepoint call has a different type than the + // actual call. It means that standard exporting mechanism will create + // register of the wrong type. So instead we need to create register with + // correct type and save value into it manually. TODO: To eliminate this + // problem we can remove gc.result intrinsics completelly and make + // statepoint call to return a tuple. + unsigned Reg = FuncInfo.CreateRegs(ISP.actualReturnType()); + + RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), Reg, + ISP.actualReturnType()); + SDValue Chain = DAG.getEntryNode(); + + RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr); + PendingExports.push_back(Chain); + FuncInfo.ValueMap[CS.getInstruction()] = Reg; + } else { + // The value of the statepoint itself will be the value of call itself. + // We'll replace the actually call node shortly. gc_result will grab + // this value. + setValue(CS.getInstruction(), ReturnValue); + } + // Replace original call - DAG.ReplaceAllUsesWith(CallNode, StatepointMCNode); // This may update Root - // Remove originall call node - DAG.DeleteNode(CallNode); + DAG.ReplaceAllUsesWith(Call, StatepointMCNode); // This may update Root + + // Remove original call node + DAG.DeleteNode(Call); // DON'T set the root - under the assumption that it's already set past the // inserted node we created.