MIR Statepoint refactoring: pass GC pointers in VRegs. Part 1/5.

[dantrushin]

Summary

Introduce new SelectionDAG node, StatepointSDNode, which captures
return value of original call and use it in place of MachineSDNode
used for statepoint lowering.
This lays ground for further changes and is NFC.

[reames]

Please pull out a purely NFC patch without any of the first-N-vregs bits which is purely functional. (i.e. replaces the existing target node with the generic node, introduces the node type and wrapper class, but does nothing else.)

The split point chosen here doesn't work. You're introducing untested and unused code. That's not okay.

[dantrushin]

Rework patch accodring to comments: Replace existing MachineSDNode with
newly introduced StatepointSDNode and use it for statepoint lowering.
No functional changes.

[reames]

Let's discuss offline.

llvm/include/llvm/CodeGen/ISDOpcodes.h

1072

The [Relocates,] part isn't yet present. Remove before commit, readd in relevant follow up.

llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

751

How is it safe not to have memrefs on the node here? Maybe some discussion offline?

802

Please revert this part for now. I'm not sure it's wrong, but I'd like to reduce the scope and audit only this part in it's own change.

[dantrushin]

llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

751

I simply have no space to put them into StatepointSDNode.
(They are part of MachineSDNode). So I add them in InstructionEmitter.

[dantrushin]

Adandoned in favor of D81648

llvm/include/llvm/CodeGen/ISDOpcodes.h

   /// beginning and end of GC transition  sequence, and carry arbitrary
   /// information that target might need for lowering.  The first operand is
   /// a chain, the rest are specified by the target and not touched by the DAG
   /// optimizers. GC_TRANSITION_START..GC_TRANSITION_END pairs may not be
   /// nested.
   GC_TRANSITION_START,
   GC_TRANSITION_END,
 
+  /// GC_STATEPOINT - Represent STATEPOINT instruction
+  /// [Relocates,] OUTCHAIN, Glue = GC_STATEPOINT(...)
+  GC_STATEPOINT,
+
   /// GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of
   /// the most recent dynamic alloca. For most targets that would be 0, but
   /// for some others (e.g. PowerPC, PowerPC64) that would be compile-time
   /// known nonzero constant. The only operand here is the chain.
   GET_DYNAMIC_AREA_OFFSET,
 
   /// VSCALE(IMM) - Returns the runtime scaling factor used to calculate the
   /// number of elements within a scalable vector. IMM is a constant integer

llvm/include/llvm/CodeGen/SelectionDAG.h

SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
     return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
   }
 
   /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
   SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
     return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);
   }
 
+  /// Return a GC_STATEPOINT node.
+  /// RetVal is the value of actuall call this statepoint wraps.
+  SDValue getStatepoint(const SDLoc &dl, SDValue RetVal, ArrayRef<SDValue> Ops);
+
   /// Gets or creates the specified node.
   ///
   SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                   ArrayRef<SDUse> Ops);
   SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                   ArrayRef<SDValue> Ops, const SDNodeFlags Flags = SDNodeFlags());
   SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys,
                   ArrayRef<SDValue> Ops);

llvm/include/llvm/CodeGen/SelectionDAGNodes.h

public:
 
   const SDValue &getValue() const { return getOperand(1); }
 
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::MSCATTER;
   }
 };
 
+/// This class is used to represent a STATEPOINT node
+class StatepointSDNode : public SDNode {
+private:
+  friend class SelectionDAG;
+
+  SDValue ActualRetVal;
+
+public:
+  StatepointSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
+                   SDValue RetVal)
+      : SDNode(ISD::GC_STATEPOINT, Order, DL, VTs), ActualRetVal(RetVal) {}
+
+  SDValue getActualRetVal() const { return ActualRetVal; }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::GC_STATEPOINT;
+  }
+};
+
 /// An SDNode that represents everything that will be needed
 /// to construct a MachineInstr. These nodes are created during the
 /// instruction selection proper phase.
 ///
 /// Note that the only supported way to set the `memoperands` is by calling the
 /// `SelectionDAG::setNodeMemRefs` function as the memory management happens
 /// inside the DAG rather than in the node.
 class MachineSDNode : public SDNode {

llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp

 // selection.
 //
 //===----------------------------------------------------------------------===//
 
 #include "InstrEmitter.h"
 #include "SDNodeDbgValue.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
if (RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Node->getOperand(I - 1)))
         continue;
     NumImpUses = N - I;
     break;
   }
 
   return N;
 }
 
+/// Collect used physical registers up the glue chain.
+static void collectPhysRegsFromGlueChain(const SDNode *Node,
+                                         const TargetInstrInfo *TII,
+                                         SmallVectorImpl<Register> &UsedRegs) {
+  if (Node->getValueType(Node->getNumValues() - 1) != MVT::Glue)
+    return;
+  for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
+    if (F->getOpcode() == ISD::CopyFromReg) {
+      UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg());
+      continue;
+    } else if (F->getOpcode() == ISD::CopyToReg) {
+      // Skip CopyToReg nodes that are internal to the glue chain.
+      continue;
+    }
+    // Collect declared implicit uses.
+    if (F->isMachineOpcode()) {
+      const MCInstrDesc &MCID = TII->get(F->getMachineOpcode());
+      UsedRegs.append(MCID.getImplicitUses(),
+                      MCID.getImplicitUses() + MCID.getNumImplicitUses());
+    }
+    // In addition to declared implicit uses, we must also check for
+    // direct RegisterSDNode operands.
+    for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i)
+      if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(F->getOperand(i))) {
+        Register Reg = R->getReg();
+        if (Reg.isPhysical())
+          UsedRegs.push_back(Reg);
+      }
+  }
+}
+
 /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
 /// implicit physical register output.
 void InstrEmitter::
 EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned,
                 Register SrcReg, DenseMap<SDValue, Register> &VRBaseMap) {
   Register VRBase;
   if (SrcReg.isVirtual()) {
     // Just use the input register directly!
for (unsigned i = NumDefs; i < NumResults; ++i) {
       if (!Node->hasAnyUseOfValue(i))
         continue;
       // This implicitly defined physreg has a use.
       UsedRegs.push_back(Reg);
       EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
     }
   }
 
-  // Scan the glue chain for any used physregs.
-  if (Node->getValueType(Node->getNumValues()-1) == MVT::Glue) {
-    for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
-      if (F->getOpcode() == ISD::CopyFromReg) {
-        UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg());
-        continue;
-      } else if (F->getOpcode() == ISD::CopyToReg) {
-        // Skip CopyToReg nodes that are internal to the glue chain.
-        continue;
-      }
-      // Collect declared implicit uses.
-      const MCInstrDesc &MCID = TII->get(F->getMachineOpcode());
-      UsedRegs.append(MCID.getImplicitUses(),
-                      MCID.getImplicitUses() + MCID.getNumImplicitUses());
-      // In addition to declared implicit uses, we must also check for
-      // direct RegisterSDNode operands.
-      for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i)
-        if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(F->getOperand(i))) {
-          Register Reg = R->getReg();
-          if (Reg.isPhysical())
-            UsedRegs.push_back(Reg);
-        }
-    }
-  }
+  collectPhysRegsFromGlueChain(Node, TII, UsedRegs);
 
   // Finally mark unused registers as dead.
   if (!UsedRegs.empty() || II.getImplicitDefs() || II.hasOptionalDef())
     MIB->setPhysRegsDeadExcept(UsedRegs, *TRI);
 
   // Run post-isel target hook to adjust this instruction if needed.
   if (II.hasPostISelHook())
     TLI->AdjustInstrPostInstrSelection(*MIB, Node);
case ISD::LIFETIME_END: {
     TargetOpcode::LIFETIME_START : TargetOpcode::LIFETIME_END;
 
     FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Node->getOperand(1));
     BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp))
     .addFrameIndex(FI->getIndex());
     break;
   }
 
+  case ISD::GC_STATEPOINT: {
+    MachineInstrBuilder MIB =
+        BuildMI(*MF, Node->getDebugLoc(), TII->get(TargetOpcode::STATEPOINT));
+
+    unsigned NumOperands = Node->getNumOperands();
+    for (unsigned i = 0; i < NumOperands; ++i) {
+      const SDValue &O = Node->getOperand(i);
+      if (O.getValueType() == MVT::Other || O.getValueType() == MVT::Glue)
+        continue;
+      AddOperand(MIB, O, 0, nullptr, VRBaseMap, false, false, false);
+
+      if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(O)) {
+        auto Flags = MachineMemOperand::MOStore | MachineMemOperand::MOLoad |
+                     MachineMemOperand::MOVolatile;
+        auto &MFI = MF->getFrameInfo();
+        MachineMemOperand *MMO = MF->getMachineMemOperand(
+            MachinePointerInfo::getFixedStack(*MF, FI->getIndex()), Flags,
+            MF->getDataLayout().getPointerSize(),
+            MFI.getObjectAlign(FI->getIndex()));
+        MIB->addMemOperand(*MF, MMO);
+      }
+    }
+
+    MBB->insert(InsertPos, MIB);
+
+    SmallVector<Register, 8> UsedRegs;
+    collectPhysRegsFromGlueChain(Node, TII, UsedRegs);
+    if (!UsedRegs.empty())
+      MIB->setPhysRegsDeadExcept(UsedRegs, *TRI);
+
+    break;
+  }
+
   case ISD::INLINEASM:
   case ISD::INLINEASM_BR: {
     unsigned NumOps = Node->getNumOperands();
     if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
       --NumOps;  // Ignore the glue operand.
 
     // Create the inline asm machine instruction.
     unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
 
   CSEMap.InsertNode(N, IP);
   InsertNode(N);
   SDValue V(N, 0);
   NewSDValueDbgMsg(V, "Creating new node: ", this);
   return V;
 }
 
+SDValue SelectionDAG::getStatepoint(const SDLoc &dl, SDValue RetVal,
+                                    ArrayRef<SDValue> Ops) {
+  SmallVector<EVT, 16> VTs;
+
+  VTs.push_back(MVT::Other);
+  VTs.push_back(MVT::Glue);
+
+  auto *N = newSDNode<StatepointSDNode>(dl.getIROrder(), dl.getDebugLoc(),
+                                        getVTList(VTs), RetVal);
+  createOperands(N, Ops);
+  InsertNode(N);
+  SDValue V(N, 0);
+  NewSDValueDbgMsg(V, "Creating new node: ", this);
+  return V;
+}
+
 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
   // select undef, T, F --> T (if T is a constant), otherwise F
   // select, ?, undef, F --> F
   // select, ?, T, undef --> T
   if (Cond.isUndef())
     return isConstantValueOfAnyType(T) ? T : F;
   if (T.isUndef())
     return F;

llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

#endif
   case ISD::EH_RETURN:                  return "EH_RETURN";
   case ISD::EH_SJLJ_SETJMP:             return "EH_SJLJ_SETJMP";
   case ISD::EH_SJLJ_LONGJMP:            return "EH_SJLJ_LONGJMP";
   case ISD::EH_SJLJ_SETUP_DISPATCH:     return "EH_SJLJ_SETUP_DISPATCH";
   case ISD::ConstantPool:               return "ConstantPool";
   case ISD::TargetIndex:                return "TargetIndex";
   case ISD::ExternalSymbol:             return "ExternalSymbol";
   case ISD::BlockAddress:               return "BlockAddress";
+  case ISD::GC_STATEPOINT:              return "STATEPOINT";
   case ISD::INTRINSIC_WO_CHAIN:
   case ISD::INTRINSIC_VOID:
   case ISD::INTRINSIC_W_CHAIN: {
     unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
     unsigned IID = cast<ConstantSDNode>(getOperand(OpNo))->getZExtValue();
     if (IID < Intrinsic::num_intrinsics)
       return Intrinsic::getName((Intrinsic::ID)IID, None);
     else if (!G)

llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
   case ISD::TargetGlobalAddress:
   case ISD::TokenFactor:
   case ISD::CopyFromReg:
   case ISD::CopyToReg:
   case ISD::EH_LABEL:
   case ISD::ANNOTATION_LABEL:
   case ISD::LIFETIME_START:
   case ISD::LIFETIME_END:
+  case ISD::GC_STATEPOINT:
     NodeToMatch->setNodeId(-1); // Mark selected.
     return;
   case ISD::AssertSext:
   case ISD::AssertZext:
     ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0));
     CurDAG->RemoveDeadNode(NodeToMatch);
     return;
   case ISD::INLINEASM:

llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp

#endif
 
   // Add chain
   Ops.push_back(Chain);
 
   // Same for the glue, but we add it only if original call had it
   if (Glue.getNode())
     Ops.push_back(Glue);
 
-  // Compute return values.  Provide a glue output since we consume one as
-  // input.  This allows someone else to chain off us as needed.
-  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
-
-  MachineSDNode *StatepointMCNode =
-    DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
-  DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
+  SDValue STV = DAG.getStatepoint(getCurSDLoc(), ReturnVal, Ops);
 
-  SDNode *SinkNode = StatepointMCNode;
+  SDNode *SinkNode = STV.getNode();
 
   // Build the GC_TRANSITION_END node if necessary.
   //
   // See the comment above regarding GC_TRANSITION_START for the layout of
   // the operands to the GC_TRANSITION_END node.
   if (IsGCTransition) {
     SmallVector<SDValue, 8> TEOps;
 
     // Add chain
-    TEOps.push_back(SDValue(StatepointMCNode, 0));
+    TEOps.push_back(SDValue(STV.getNode(), STV->getNumValues() - 2));
 
     // Add GC transition arguments
     for (const Value *V : SI.GCTransitionArgs) {
       TEOps.push_back(getValue(V));
       if (V->getType()->isPointerTy())
         TEOps.push_back(DAG.getSrcValue(V));
     }
 
     // Add glue
-    TEOps.push_back(SDValue(StatepointMCNode, 1));
+    TEOps.push_back(SDValue(STV.getNode(), STV->getNumValues() - 1));
 
     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
     SDValue GCTransitionStart =
         DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
 
     SinkNode = GCTransitionStart.getNode();
   }
 
   // Replace original call
-  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
+  // Call: ch,glue = CALL ...
+  // Statepoint: [gc relocates],ch,glue = GC_STATEPOINT ...
+  unsigned NumSinkValues = SinkNode->getNumValues();
+  SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2),
+                                 SDValue(SinkNode, NumSinkValues - 1)};
+  DAG.ReplaceAllUsesWith(CallNode, StatepointValues);
   // Remove original call node
   DAG.DeleteNode(CallNode);
 
   // DON'T set the root - under the assumption that it's already set past the
   // inserted node we created.
 
   // TODO: A better future implementation would be to emit a single variable
   // argument, variable return value STATEPOINT node here and then hookup the
   // return value of each gc.relocate to the respective output of the
   // previously emitted STATEPOINT value.  Unfortunately, this doesn't appear
   // to actually be possible today.
 
-  return ReturnVal;
+  return STV;
 }
 
 void
 SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I,
                                      const BasicBlock *EHPadBB /*= nullptr*/) {
   ImmutableStatepoint ISP(&I);
   assert(I.getCallingConv() != CallingConv::AnyReg &&
          "anyregcc is not supported on statepoints!");
if (auto Opt = I.getOperandBundle(LLVMContext::OB_gc_transition)) {
     SI.GCTransitionArgs = ArrayRef<const Use>(ISP.gc_transition_args_begin(),
                                               ISP.gc_transition_args_end());
   }
 
   SI.StatepointFlags = I.getFlags();
   SI.NumPatchBytes = I.getNumPatchBytes();
   SI.EHPadBB = EHPadBB;
 
-  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
+  SDValue STV = LowerAsSTATEPOINT(SI);
 
   // Export the result value if needed
   const GCResultInst *GCResult = I.getGCResult();
   Type *RetTy = I.getActualReturnType();
   if (!RetTy->isVoidTy() && GCResult) {
     if (GCResult->getParent() != I.getParent()) {
+      SDValue ReturnValue = cast<StatepointSDNode>(STV)->getActualRetVal();
       // Result value will be used in a different basic block so we need to
       // export it now.  Default exporting mechanism will not work here because
       // statepoint call has a different type than the actual call. It means
       // that by default llvm will create export register of the wrong type
       // (always i32 in our case). So instead we need to create export register
       // with correct type manually.
       // TODO: To eliminate this problem we can remove gc.result intrinsics
       //       completely and make statepoint call to return a tuple.
       unsigned Reg = FuncInfo.CreateRegs(RetTy);
       RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
                        DAG.getDataLayout(), Reg, RetTy,
                        I.getCallingConv());
       SDValue Chain = DAG.getEntryNode();
 
       RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
       PendingExports.push_back(Chain);
       FuncInfo.ValueMap[&I] = Reg;
     } else {
       // Result value will be used in a same basic block. Don't export it or
       // perform any explicit register copies.
       // We'll replace the actuall call node shortly. gc_result will grab
       // this value.
-      setValue(&I, ReturnValue);
+      setValue(&I, STV);
     }
   } else {
     // The token value is never used from here on, just generate a poison value
     setValue(&I, DAG.getIntPtrConstant(-1, getCurSDLoc()));
   }
 }
 
 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
 
   SI.DeoptState =
       ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
   SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
   SI.EHPadBB = EHPadBB;
 
   // NB! The GC arguments are deliberately left empty.
 
-  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
+  SDValue STV = LowerAsSTATEPOINT(SI);
+  if (SDValue ReturnVal =
+          cast<StatepointSDNode>(STV.getNode())->getActualRetVal()) {
     ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
     setValue(Call, ReturnVal);
   }
 }
 
 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
     const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
   LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
if (I->getParent() != CI.getParent()) {
     // different, and getValue() will use CopyFromReg of the wrong type,
     // which is always i32 in our case.
     Type *RetTy = I->getActualReturnType();
     SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
 
     assert(CopyFromReg.getNode());
     setValue(&CI, CopyFromReg);
   } else {
-    setValue(&CI, getValue(I));
+    StatepointSDNode *STN = cast<StatepointSDNode>(getValue(I));
+    setValue(&CI, STN->getActualRetVal());
   }
 }
 
 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
 #ifndef NDEBUG
   // Consistency check
   // We skip this check for relocates not in the same basic block as their
   // statepoint. It would be too expensive to preserve validation info through