diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1068,6 +1068,10 @@ GC_TRANSITION_START, GC_TRANSITION_END, + /// GC_STATEPOINT - Represent STATEPOINT instruction + /// [Relocates,] OUTCHAIN, Glue = GC_STATEPOINT(...) + GC_STATEPOINT, + /// GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of /// the most recent dynamic alloca. For most targets that would be 0, but /// for some others (e.g. PowerPC, PowerPC64) that would be compile-time diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -928,6 +928,10 @@ return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT); } + /// Return a GC_STATEPOINT node. + /// RetVal is the value of actuall call this statepoint wraps. + SDValue getStatepoint(const SDLoc &dl, SDValue RetVal, ArrayRef Ops); + /// Gets or creates the specified node. /// SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -2462,6 +2462,25 @@ } }; +/// This class is used to represent a STATEPOINT node +class StatepointSDNode : public SDNode { +private: + friend class SelectionDAG; + + SDValue ActualRetVal; + +public: + StatepointSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs, + SDValue RetVal) + : SDNode(ISD::GC_STATEPOINT, Order, DL, VTs), ActualRetVal(RetVal) {} + + SDValue getActualRetVal() const { return ActualRetVal; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::GC_STATEPOINT; + } +}; + /// An SDNode that represents everything that will be needed /// to construct a MachineInstr. These nodes are created during the /// instruction selection proper phase. diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -16,6 +16,7 @@ #include "SDNodeDbgValue.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineConstantPool.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -82,6 +83,37 @@ return N; } +/// Collect used physical registers up the glue chain. +static void collectPhysRegsFromGlueChain(const SDNode *Node, + const TargetInstrInfo *TII, + SmallVectorImpl &UsedRegs) { + if (Node->getValueType(Node->getNumValues() - 1) != MVT::Glue) + return; + for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) { + if (F->getOpcode() == ISD::CopyFromReg) { + UsedRegs.push_back(cast(F->getOperand(1))->getReg()); + continue; + } else if (F->getOpcode() == ISD::CopyToReg) { + // Skip CopyToReg nodes that are internal to the glue chain. + continue; + } + // Collect declared implicit uses. + if (F->isMachineOpcode()) { + const MCInstrDesc &MCID = TII->get(F->getMachineOpcode()); + UsedRegs.append(MCID.getImplicitUses(), + MCID.getImplicitUses() + MCID.getNumImplicitUses()); + } + // In addition to declared implicit uses, we must also check for + // direct RegisterSDNode operands. + for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i) + if (RegisterSDNode *R = dyn_cast(F->getOperand(i))) { + Register Reg = R->getReg(); + if (Reg.isPhysical()) + UsedRegs.push_back(Reg); + } + } +} + /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an /// implicit physical register output. void InstrEmitter:: @@ -941,30 +973,7 @@ } } - // Scan the glue chain for any used physregs. - if (Node->getValueType(Node->getNumValues()-1) == MVT::Glue) { - for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) { - if (F->getOpcode() == ISD::CopyFromReg) { - UsedRegs.push_back(cast(F->getOperand(1))->getReg()); - continue; - } else if (F->getOpcode() == ISD::CopyToReg) { - // Skip CopyToReg nodes that are internal to the glue chain. - continue; - } - // Collect declared implicit uses. - const MCInstrDesc &MCID = TII->get(F->getMachineOpcode()); - UsedRegs.append(MCID.getImplicitUses(), - MCID.getImplicitUses() + MCID.getNumImplicitUses()); - // In addition to declared implicit uses, we must also check for - // direct RegisterSDNode operands. - for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i) - if (RegisterSDNode *R = dyn_cast(F->getOperand(i))) { - Register Reg = R->getReg(); - if (Reg.isPhysical()) - UsedRegs.push_back(Reg); - } - } - } + collectPhysRegsFromGlueChain(Node, TII, UsedRegs); // Finally mark unused registers as dead. if (!UsedRegs.empty() || II.getImplicitDefs() || II.hasOptionalDef()) @@ -1042,6 +1051,39 @@ break; } + case ISD::GC_STATEPOINT: { + MachineInstrBuilder MIB = + BuildMI(*MF, Node->getDebugLoc(), TII->get(TargetOpcode::STATEPOINT)); + + unsigned NumOperands = Node->getNumOperands(); + for (unsigned i = 0; i < NumOperands; ++i) { + const SDValue &O = Node->getOperand(i); + if (O.getValueType() == MVT::Other || O.getValueType() == MVT::Glue) + continue; + AddOperand(MIB, O, 0, nullptr, VRBaseMap, false, false, false); + + if (FrameIndexSDNode *FI = dyn_cast(O)) { + auto Flags = MachineMemOperand::MOStore | MachineMemOperand::MOLoad | + MachineMemOperand::MOVolatile; + auto &MFI = MF->getFrameInfo(); + MachineMemOperand *MMO = MF->getMachineMemOperand( + MachinePointerInfo::getFixedStack(*MF, FI->getIndex()), Flags, + MF->getDataLayout().getPointerSize(), + MFI.getObjectAlign(FI->getIndex())); + MIB->addMemOperand(*MF, MMO); + } + } + + MBB->insert(InsertPos, MIB); + + SmallVector UsedRegs; + collectPhysRegsFromGlueChain(Node, TII, UsedRegs); + if (!UsedRegs.empty()) + MIB->setPhysRegsDeadExcept(UsedRegs, *TRI); + + break; + } + case ISD::INLINEASM: case ISD::INLINEASM_BR: { unsigned NumOps = Node->getNumOperands(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -7269,6 +7269,22 @@ return V; } +SDValue SelectionDAG::getStatepoint(const SDLoc &dl, SDValue RetVal, + ArrayRef Ops) { + SmallVector VTs; + + VTs.push_back(MVT::Other); + VTs.push_back(MVT::Glue); + + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), + getVTList(VTs), RetVal); + createOperands(N, Ops); + InsertNode(N); + SDValue V(N, 0); + NewSDValueDbgMsg(V, "Creating new node: ", this); + return V; +} + SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) { // select undef, T, F --> T (if T is a constant), otherwise F // select, ?, undef, F --> F diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -138,6 +138,7 @@ case ISD::TargetIndex: return "TargetIndex"; case ISD::ExternalSymbol: return "ExternalSymbol"; case ISD::BlockAddress: return "BlockAddress"; + case ISD::GC_STATEPOINT: return "STATEPOINT"; case ISD::INTRINSIC_WO_CHAIN: case ISD::INTRINSIC_VOID: case ISD::INTRINSIC_W_CHAIN: { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2816,6 +2816,7 @@ case ISD::ANNOTATION_LABEL: case ISD::LIFETIME_START: case ISD::LIFETIME_END: + case ISD::GC_STATEPOINT: NodeToMatch->setNodeId(-1); // Mark selected. return; case ISD::AssertSext: diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -748,15 +748,9 @@ if (Glue.getNode()) Ops.push_back(Glue); - // Compute return values. Provide a glue output since we consume one as - // input. This allows someone else to chain off us as needed. - SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); + SDValue STV = DAG.getStatepoint(getCurSDLoc(), ReturnVal, Ops); - MachineSDNode *StatepointMCNode = - DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops); - DAG.setNodeMemRefs(StatepointMCNode, MemRefs); - - SDNode *SinkNode = StatepointMCNode; + SDNode *SinkNode = STV.getNode(); // Build the GC_TRANSITION_END node if necessary. // @@ -766,7 +760,7 @@ SmallVector TEOps; // Add chain - TEOps.push_back(SDValue(StatepointMCNode, 0)); + TEOps.push_back(SDValue(STV.getNode(), STV->getNumValues() - 2)); // Add GC transition arguments for (const Value *V : SI.GCTransitionArgs) { @@ -776,7 +770,7 @@ } // Add glue - TEOps.push_back(SDValue(StatepointMCNode, 1)); + TEOps.push_back(SDValue(STV.getNode(), STV->getNumValues() - 1)); SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); @@ -787,7 +781,12 @@ } // Replace original call - DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root + // Call: ch,glue = CALL ... + // Statepoint: [gc relocates],ch,glue = GC_STATEPOINT ... + unsigned NumSinkValues = SinkNode->getNumValues(); + SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2), + SDValue(SinkNode, NumSinkValues - 1)}; + DAG.ReplaceAllUsesWith(CallNode, StatepointValues); // Remove original call node DAG.DeleteNode(CallNode); @@ -800,7 +799,7 @@ // previously emitted STATEPOINT value. Unfortunately, this doesn't appear // to actually be possible today. - return ReturnVal; + return STV; } void @@ -890,13 +889,14 @@ SI.NumPatchBytes = I.getNumPatchBytes(); SI.EHPadBB = EHPadBB; - SDValue ReturnValue = LowerAsSTATEPOINT(SI); + SDValue STV = LowerAsSTATEPOINT(SI); // Export the result value if needed const GCResultInst *GCResult = I.getGCResult(); Type *RetTy = I.getActualReturnType(); if (!RetTy->isVoidTy() && GCResult) { if (GCResult->getParent() != I.getParent()) { + SDValue ReturnValue = cast(STV)->getActualRetVal(); // Result value will be used in a different basic block so we need to // export it now. Default exporting mechanism will not work here because // statepoint call has a different type than the actual call. It means @@ -919,7 +919,7 @@ // perform any explicit register copies. // We'll replace the actuall call node shortly. gc_result will grab // this value. - setValue(&I, ReturnValue); + setValue(&I, STV); } } else { // The token value is never used from here on, just generate a poison value @@ -954,7 +954,9 @@ // NB! The GC arguments are deliberately left empty. - if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) { + SDValue STV = LowerAsSTATEPOINT(SI); + if (SDValue ReturnVal = + cast(STV.getNode())->getActualRetVal()) { ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal); setValue(Call, ReturnVal); } @@ -985,7 +987,8 @@ assert(CopyFromReg.getNode()); setValue(&CI, CopyFromReg); } else { - setValue(&CI, getValue(I)); + StatepointSDNode *STN = cast(getValue(I)); + setValue(&CI, STN->getActualRetVal()); } }