Legalise patchpoint arguments.

[vext01]

Summary

This is similar to D125680, but for llvm.experimental.patchpoint (instead of llvm.experimental.stackmap).

Some test outputs changed, e.g. passing i16 65535 as a live caused:

-; CHECK-NEXT:   .long   -1
+; CHECK-NEXT:   .long   65535

I think this is OK. I think the runtime is expected to know the true size of the constant, and the higher order bits are undefined. Under this assumption, -1 and 65535 are equivalent for a i16.

[The runtime certainly can't rely on the size field in the stackmap record, which for this constant i16 is reporting as 8 bytes long. This is particularly confusing in this instance, as the small constant field can only hold 4-bytes! This bug was not introduced by this diff]

[arsenm]

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

5550–5551

dyn_cast to ConstantSDNode

5563–5565

No return after else

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

9378

You can just forward the type, no need to call getFrameIndexTy

llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

2204

dyn_cast to ConstantSDNode

[vext01]

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

5550–5551

Forgive me. I'm still fairly new to C++, am I doing this right?

diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 4ee4ece7d8ef..89c0738f24b0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -5547,8 +5547,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
   for (unsigned I = 0; I < OpNo; I++)
     NewOps.push_back(N->getOperand(I));
 
-  if (Op->getOpcode() == ISD::Constant) {
-    ConstantSDNode *CN = cast<ConstantSDNode>(Op);
+  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
     EVT Ty = Op.getValueType();
     if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
       NewOps.push_back(
@@ -5587,8 +5586,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
   for (unsigned I = 0; I < OpNo; I++)
     NewOps.push_back(N->getOperand(I));
 
-  if (Op->getOpcode() == ISD::Constant) {
-    ConstantSDNode *CN = cast<ConstantSDNode>(Op);
+  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
     EVT Ty = Op.getValueType();
     if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
       NewOps.push_back(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 25105ee2dbde..0dae6a06ef72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2201,12 +2201,11 @@ void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops,
   // nodes at DAG-construction time.
   assert(OpNode->getOpcode() != ISD::FrameIndex);
 
-  if (OpNode->getOpcode() == ISD::Constant) {
+  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpNode)) {
     Ops.push_back(
         CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
     Ops.push_back(
-        CurDAG->getTargetConstant(cast<ConstantSDNode>(OpNode)->getZExtValue(),
-                                  DL, OpVal.getValueType()));
+        CurDAG->getTargetConstant(CN->getZExtValue(), DL, OpVal.getValueType()));
   } else {
     Ops.push_back(OpVal);
   }

This makes tests fail, so can't be equivalent.

5563–5565

I'm not sure what you mean.

[arsenm]

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

5550–5551

The slight difference is ConstantSDNode accepts Constant and TargetConstant but I would expect consistency at this point

[vext01]

New diff fixing some of @arsenm's comments.

[vext01]

I'm still unsure what you mean by "No return after else".

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

5550–5551

The first two hunks of the above diff are fine, it's the last one that causes test failures.

Not sure why, but I've omitted that change for now anyway.

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

9378

Yep. Fixed.

[arsenm]

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

9373–9374

This comment is kind of a lie. Pointer typed doesn't imply legality but I guess it doesn't really matter

[vext01]

Refactor to remove else after returns.

[vext01]

This should be ready now.

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

9373–9374

I see what you are saying, although the pointer itself should be legal (even if the pointee isn't).

I agree that it doesn't hugely matter.

[arsenm]

LGTM with nits

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

5548

!CN

5550

Don't need braces

5587–5590

Ditto

[vext01]

This landed in 2e62a26

llvm/include/llvm/CodeGen/ISDOpcodes.h

   VECREDUCE_UMAX,
   VECREDUCE_UMIN,
 
   // The `llvm.experimental.stackmap` intrinsic.
   // Operands: input chain, glue, <id>, <numShadowBytes>, [live0[, live1...]]
   // Outputs: output chain, glue
   STACKMAP,
 
+  // The `llvm.experimental.patchpoint.*` intrinsic.
+  // Operands: input chain, [glue], reg-mask, <id>, <numShadowBytes>, callee,
+  //   <numArgs>, cc, ...
+  // Outputs: [rv], output chain, glue
+  PATCHPOINT,
+
 // Vector Predication
 #define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID,
 #include "llvm/IR/VPIntrinsics.def"
 
   /// BUILTIN_OP_END - This must be the last enum value in this list.
   /// The target-specific pre-isel opcode values start here.
   BUILTIN_OP_END
 };

llvm/include/llvm/CodeGen/SelectionDAGISel.h

private:
   void Select_INLINEASM(SDNode *N);
   void Select_READ_REGISTER(SDNode *Op);
   void Select_WRITE_REGISTER(SDNode *Op);
   void Select_UNDEF(SDNode *N);
   void CannotYetSelect(SDNode *N);
 
   void Select_FREEZE(SDNode *N);
   void Select_ARITH_FENCE(SDNode *N);
+
+  void pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops, SDValue Operand,
+                                SDLoc DL);
   void Select_STACKMAP(SDNode *N);
+  void Select_PATCHPOINT(SDNode *N);
 
 private:
   void DoInstructionSelection();
   SDNode *MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
                     ArrayRef<SDValue> Ops, unsigned EmitNodeInfo);
 
   /// Prepares the landing pad to take incoming values or do other EH
   /// personality specific tasks. Returns true if the block should be

llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp

bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
   case ISD::STRICT_FP_EXTEND:
   case ISD::FP_EXTEND:  Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
   case ISD::SELECT_CC:  Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
   case ISD::SETCC:      Res = SoftPromoteHalfOp_SETCC(N); break;
   case ISD::STORE:      Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
   case ISD::STACKMAP:
     Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
     break;
+  case ISD::PATCHPOINT:
+    Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
+    break;
   }
 
   if (!Res.getNode())
     return false;
 
   assert(Res.getNode() != N && "Expected a new node!");
 
   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
   SDValue NewNode =
       DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
 
   for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
     ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
 
   return SDValue(); // Signal that we replaced the node ourselves.
 }
+
+SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
+                                                       unsigned OpNo) {
+  assert(OpNo >= 7);
+  SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
+  SDValue Op = N->getOperand(OpNo);
+  NewOps[OpNo] = GetSoftPromotedHalf(Op);
+  SDValue NewNode =
+      DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
+
+  for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
+    ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
+
+  return SDValue(); // Signal that we replaced the node ourselves.
+}

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::VP_REDUCE_UMIN:
     Res = PromoteIntOp_VP_REDUCE(N, OpNo);
     break;
 
   case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
   case ISD::STACKMAP:
     Res = PromoteIntOp_STACKMAP(N, OpNo);
     break;
+  case ISD::PATCHPOINT:
+    Res = PromoteIntOp_PATCHPOINT(N, OpNo);
+    break;
   }
 
   // If the result is null, the sub-method took care of registering results etc.
   if (!Res.getNode()) return false;
 
   // If the result is N, the sub-method updated N in place.  Tell the legalizer
   // core about this.
   if (Res.getNode() == N)
SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
   assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
   SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
   SDValue Operand = N->getOperand(OpNo);
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
   NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
 }
 
+SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
+  assert(OpNo >= 7);
+  SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
+  SDValue Operand = N->getOperand(OpNo);
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
+  NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
+  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+}
+
 //===----------------------------------------------------------------------===//
 //  Integer Result Expansion
 //===----------------------------------------------------------------------===//
 
 /// ExpandIntegerResult - This method is called when the specified result of the
 /// specified node is found to need expansion.  At this point, the node may also
 /// have invalid operands or may have other results that need promotion, we just
 /// know that (at least) one result needs expansion.
bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
   case ISD::RETURNADDR:
   case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;
 
   case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
   case ISD::STACKMAP:
     Res = ExpandIntOp_STACKMAP(N, OpNo);
     break;
+  case ISD::PATCHPOINT:
+    Res = ExpandIntOp_PATCHPOINT(N, OpNo);
+    break;
   }
 
   // If the result is null, the sub-method took care of registering results etc.
   if (!Res.getNode()) return false;
 
   // If the result is N, the sub-method updated N in place.  Tell the legalizer
   // core about this.
   if (Res.getNode() == N)
for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
     }
   }
 
   return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
 }
 
 SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
   assert(OpNo > 1);
-
   SDValue Op = N->getOperand(OpNo);
-  SDLoc DL = SDLoc(N);
-  SmallVector<SDValue> NewOps;
+
+  // FIXME: Non-constant operands are not yet handled:
+  //  - https://github.com/llvm/llvm-project/issues/26431
+  //  - https://github.com/llvm/llvm-project/issues/55957
+  ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
+  if (CN == nullptr) {
+    return SDValue();
+  }
 
   // Copy operands before the one being expanded.
+  SmallVector<SDValue> NewOps;
   for (unsigned I = 0; I < OpNo; I++)
     NewOps.push_back(N->getOperand(I));
 
-  if (Op->getOpcode() == ISD::Constant) {
-    ConstantSDNode *CN = cast<ConstantSDNode>(Op);
-    EVT Ty = Op.getValueType();
-    if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
-      NewOps.push_back(
-          DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
-      NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
-    } else {
-      // FIXME: https://github.com/llvm/llvm-project/issues/55609
-      return SDValue();
-    }
-  } else {
-    // FIXME: Non-constant operands are not yet handled:
-    //  - https://github.com/llvm/llvm-project/issues/26431
-    //  - https://github.com/llvm/llvm-project/issues/55957
+  SDLoc DL = SDLoc(N);
+
+  // Copy remaining operands.
+  for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
+    NewOps.push_back(N->getOperand(I));
+
+  SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
+
+  for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
+    ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
+
+  return SDValue(); // Signal that we have replaced the node already.
+}
+
+SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
+  assert(OpNo >= 7);
+  SDValue Op = N->getOperand(OpNo);
+
+  ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
+  if (CN == nullptr) {
+    return SDValue();
+  }
+
+  // Copy operands before the one being expanded.
+  SmallVector<SDValue> NewOps;
+  for (unsigned I = 0; I < OpNo; I++)
+    NewOps.push_back(N->getOperand(I));
+
+  EVT Ty = Op.getValueType();
+  SDLoc DL = SDLoc(N);
+  if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
+    NewOps.push_back(
+        DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
+    NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
+  } else {
+    // FIXME: https://github.com/llvm/llvm-project/issues/55609
     return SDValue();
   }
 
   // Copy remaining operands.
   for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
     NewOps.push_back(N->getOperand(I));
 
   SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
 
   for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
     ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
 
   return SDValue(); // Signal that we have replaced the node already.
 }

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
   SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FIX(SDNode *N);
   SDValue PromoteIntOp_FPOWI(SDNode *N);
   SDValue PromoteIntOp_VECREDUCE(SDNode *N);
   SDValue PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_SET_ROUNDING(SDNode *N);
   SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo);
+  SDValue PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo);
 
   void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
 
   //===--------------------------------------------------------------------===//
   // Integer Expansion Support: LegalizeIntegerTypes.cpp
   //===--------------------------------------------------------------------===//
 
   /// Given a processed operand Op which was expanded into two integers of half
private:
   SDValue ExpandIntOp_SINT_TO_FP(SDNode *N);
   SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
   SDValue ExpandIntOp_TRUNCATE(SDNode *N);
   SDValue ExpandIntOp_UINT_TO_FP(SDNode *N);
   SDValue ExpandIntOp_RETURNADDR(SDNode *N);
   SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
   SDValue ExpandIntOp_SPLAT_VECTOR(SDNode *N);
   SDValue ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo);
+  SDValue ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo);
 
   void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
                                   ISD::CondCode &CCCode, const SDLoc &dl);
 
   //===--------------------------------------------------------------------===//
   // Float to Integer Conversion Support: LegalizeFloatTypes.cpp
   //===--------------------------------------------------------------------===//
 
private:
   SDValue SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
   SDValue SoftPromoteHalfOp_FP_EXTEND(SDNode *N);
   SDValue SoftPromoteHalfOp_FP_TO_XINT(SDNode *N);
   SDValue SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N);
   SDValue SoftPromoteHalfOp_SETCC(SDNode *N);
   SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);
   SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);
   SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo);
+  SDValue SoftPromoteHalfOp_PATCHPOINT(SDNode *N, unsigned OpNo);
 
   //===--------------------------------------------------------------------===//
   // Scalarization Support: LegalizeVectorTypes.cpp
   //===--------------------------------------------------------------------===//
 
   /// Given a processed one-element vector Op which was scalarized to its
   /// element type, this returns the element. For example, if Op is a v1i32,
   /// Op = < i32 val >, this method returns val, an i32.

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

 /// StackMap location can be read immediately after compilation and that the
 /// location is valid at any point during execution (this is similar to the
 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
 /// only available in a register, then the runtime would need to trap when
 /// execution reaches the StackMap in order to read the alloca's location.
 static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
                                 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
                                 SelectionDAGBuilder &Builder) {
-  for (unsigned i = StartIdx, e = Call.arg_size(); i != e; ++i) {
-    SDValue OpVal = Builder.getValue(Call.getArgOperand(i));
-    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
-      Ops.push_back(
-        Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
-      Ops.push_back(
-        Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
-    } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
-      const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
-      Ops.push_back(Builder.DAG.getTargetFrameIndex(
-          FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
-    } else
-      Ops.push_back(OpVal);
+  SelectionDAG &DAG = Builder.DAG;
+  for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
+    SDValue Op = Builder.getValue(Call.getArgOperand(I));
+
+    // Things on the stack are pointer-typed, meaning that they are already
+    // legal and can be emitted directly to target nodes.
+    if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
+      Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
+    } else {
+      // Otherwise emit a target independent node to be legalised.
+      Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
+    }
   }
 }
 
 /// Lower llvm.experimental.stackmap.
 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
   // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
   //                                  [live variables...])
 
void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
 
   SDValue Shad = getValue(CI.getArgOperand(1));
   assert(Shad.getValueType() == MVT::i32);
   SDValue ShadConst = DAG.getTargetConstant(
       cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType());
   Ops.push_back(ShadConst);
 
   // Add the live variables.
-  for (unsigned I = 2; I < CI.arg_size(); I++) {
-    SDValue Op = getValue(CI.getArgOperand(I));
-
-    // Things on the stack are pointer-typed, meaning that they are already
-    // legal and can be emitted directly to target nodes.
-    if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
-      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-      Ops.push_back(DAG.getTargetFrameIndex(
-          FI->getIndex(), TLI.getFrameIndexTy(DAG.getDataLayout())));
-    } else {
-      // Otherwise emit a target independent node to be legalised.
-      Ops.push_back(getValue(CI.getArgOperand(I)));
-    }
-  }
+  addStackMapLiveVars(CI, 2, DL, Ops, *this);
 
   // Create the STACKMAP node.
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
   Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
   InFlag = Chain.getValue(1);
 
   Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
 
void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
          "Expected a callseq node.");
   SDNode *Call = CallEnd->getOperand(0).getNode();
   bool HasGlue = Call->getGluedNode();
 
   // Replace the target specific call node with the patchable intrinsic.
   SmallVector<SDValue, 8> Ops;
 
+  // Push the chain.
+  Ops.push_back(*(Call->op_begin()));
+
+  // Optionally, push the glue (if any).
+  if (HasGlue)
+    Ops.push_back(*(Call->op_end() - 1));
+
+  // Push the register mask info.
+  if (HasGlue)
+    Ops.push_back(*(Call->op_end() - 2));
+  else
+    Ops.push_back(*(Call->op_end() - 1));
+
   // Add the <id> and <numBytes> constants.
   SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos));
   Ops.push_back(DAG.getTargetConstant(
                   cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
   SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos));
   Ops.push_back(DAG.getTargetConstant(
                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
                   MVT::i32));
void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
   Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
 
   // Add the arguments we omitted previously. The register allocator should
   // place these in any free register.
   if (IsAnyRegCC)
     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
       Ops.push_back(getValue(CB.getArgOperand(i)));
 
-  // Push the arguments from the call instruction up to the register mask.
+  // Push the arguments from the call instruction.
   SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
   Ops.append(Call->op_begin() + 2, e);
 
   // Push live variables for the stack map.
   addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
 
-  // Push the register mask info.
-  if (HasGlue)
-    Ops.push_back(*(Call->op_end()-2));
-  else
-    Ops.push_back(*(Call->op_end()-1));
-
-  // Push the chain (this is originally the first operand of the call, but
-  // becomes now the last or second to last operand).
-  Ops.push_back(*(Call->op_begin()));
-
-  // Push the glue flag (last operand).
-  if (HasGlue)
-    Ops.push_back(*(Call->op_end()-1));
-
   SDVTList NodeTys;
   if (IsAnyRegCC && HasDef) {
     // Create the return types based on the intrinsic definition
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     SmallVector<EVT, 3> ValueVTs;
     ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
 
     // There is always a chain and a glue type at the end
     ValueVTs.push_back(MVT::Other);
     ValueVTs.push_back(MVT::Glue);
     NodeTys = DAG.getVTList(ValueVTs);
   } else
     NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
   // Replace the target specific call node with a PATCHPOINT node.
-  MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
-                                         dl, NodeTys, Ops);
+  SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
 
   // Update the NodeMap.
   if (HasDef) {
     if (IsAnyRegCC)
-      setValue(&CB, SDValue(MN, 0));
+      setValue(&CB, SDValue(PPV.getNode(), 0));
     else
       setValue(&CB, Result.first);
   }
 
   // Fixup the consumers of the intrinsic. The chain and glue may be used in the
   // call sequence. Furthermore the location of the chain and glue can change
   // when the AnyReg calling convention is used and the intrinsic returns a
   // value.
   if (IsAnyRegCC && HasDef) {
     SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
-    SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
+    SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
     DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
   } else
-    DAG.ReplaceAllUsesWith(Call, MN);
+    DAG.ReplaceAllUsesWith(Call, PPV.getNode());
   DAG.DeleteNode(Call);
 
   // Inform the Frame Information that we have a patchpoint in this function.
   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
 }
 
 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
                                             unsigned Intrinsic) {

llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

#endif
   case ISD::VECREDUCE_SMAX:             return "vecreduce_smax";
   case ISD::VECREDUCE_SMIN:             return "vecreduce_smin";
   case ISD::VECREDUCE_UMAX:             return "vecreduce_umax";
   case ISD::VECREDUCE_UMIN:             return "vecreduce_umin";
   case ISD::VECREDUCE_FMAX:             return "vecreduce_fmax";
   case ISD::VECREDUCE_FMIN:             return "vecreduce_fmin";
   case ISD::STACKMAP:
     return "stackmap";
+  case ISD::PATCHPOINT:
+    return "patchpoint";
 
     // Vector Predication
 #define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...)                    \
   case ISD::SDID:                                                              \
     return #NAME;
 #include "llvm/IR/VPIntrinsics.def"
   }
 }

llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
                        N->getOperand(0));
 }
 
 void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
   CurDAG->SelectNodeTo(N, TargetOpcode::ARITH_FENCE, N->getValueType(0),
                        N->getOperand(0));
 }
 
+void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops,
+                                                SDValue OpVal, SDLoc DL) {
+  SDNode *OpNode = OpVal.getNode();
+
+  // FrameIndex nodes should have been directly emitted to TargetFrameIndex
+  // nodes at DAG-construction time.
+  assert(OpNode->getOpcode() != ISD::FrameIndex);
+
+  if (OpNode->getOpcode() == ISD::Constant) {
+    Ops.push_back(
+        CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
+    Ops.push_back(
+        CurDAG->getTargetConstant(cast<ConstantSDNode>(OpNode)->getZExtValue(),
+                                  DL, OpVal.getValueType()));
+  } else {
+    Ops.push_back(OpVal);
+  }
+}
+
 void SelectionDAGISel::Select_STACKMAP(SDNode *N) {
-  std::vector<SDValue> Ops;
+  SmallVector<SDValue, 32> Ops;
   auto *It = N->op_begin();
   SDLoc DL(N);
 
   // Stash the chain and glue operands so we can move them to the end.
   SDValue Chain = *It++;
   SDValue InFlag = *It++;
 
   // <id> operand.
   SDValue ID = *It++;
   assert(ID.getValueType() == MVT::i64);
   Ops.push_back(ID);
 
   // <numShadowBytes> operand.
   SDValue Shad = *It++;
   assert(Shad.getValueType() == MVT::i32);
   Ops.push_back(Shad);
 
   // Live variable operands.
-  for (; It != N->op_end(); It++) {
-    SDNode *OpNode = It->getNode();
-    SDValue O;
-
-    // FrameIndex nodes should have been directly emitted to TargetFrameIndex
-    // nodes at DAG-construction time.
-    assert(OpNode->getOpcode() != ISD::FrameIndex);
-
-    if (OpNode->getOpcode() == ISD::Constant) {
-      Ops.push_back(
-          CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
-      O = CurDAG->getTargetConstant(
-          cast<ConstantSDNode>(OpNode)->getZExtValue(), DL, It->getValueType());
-    } else {
-      O = *It;
-    }
-    Ops.push_back(O);
-  }
+  for (; It != N->op_end(); It++)
+    pushStackMapLiveVariable(Ops, *It, DL);
 
   Ops.push_back(Chain);
   Ops.push_back(InFlag);
 
   SDVTList NodeTys = CurDAG->getVTList(MVT::Other, MVT::Glue);
   CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops);
 }
 
+void SelectionDAGISel::Select_PATCHPOINT(SDNode *N) {
+  SmallVector<SDValue, 32> Ops;
+  auto *It = N->op_begin();
+  SDLoc DL(N);
+
+  // Cache arguments that will be moved to the end in the target node.
+  SDValue Chain = *It++;
+  Optional<SDValue> Glue;
+  if (It->getValueType() == MVT::Glue)
+    Glue = *It++;
+  SDValue RegMask = *It++;
+
+  // <id> operand.
+  SDValue ID = *It++;
+  assert(ID.getValueType() == MVT::i64);
+  Ops.push_back(ID);
+
+  // <numShadowBytes> operand.
+  SDValue Shad = *It++;
+  assert(Shad.getValueType() == MVT::i32);
+  Ops.push_back(Shad);
+
+  // Add the callee.
+  Ops.push_back(*It++);
+
+  // Add <numArgs>.
+  SDValue NumArgs = *It++;
+  assert(NumArgs.getValueType() == MVT::i32);
+  Ops.push_back(NumArgs);
+
+  // Calling convention.
+  Ops.push_back(*It++);
+
+  // Push the args for the call.
+  for (uint64_t I = cast<ConstantSDNode>(NumArgs)->getZExtValue(); I != 0; I--)
+    Ops.push_back(*It++);
+
+  // Now push the live variables.
+  for (; It != N->op_end(); It++)
+    pushStackMapLiveVariable(Ops, *It, DL);
+
+  // Finally, the regmask, chain and (if present) glue are moved to the end.
+  Ops.push_back(RegMask);
+  Ops.push_back(Chain);
+  if (Glue.hasValue())
+    Ops.push_back(Glue.getValue());
+
+  SDVTList NodeTys = N->getVTList();
+  CurDAG->SelectNodeTo(N, TargetOpcode::PATCHPOINT, NodeTys, Ops);
+}
+
 /// GetVBR - decode a vbr encoding whose top bit is set.
 LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t
 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
   assert(Val >= 128 && "Not a VBR");
   Val &= 127;  // Remove first vbr bit.
 
   unsigned Shift = 7;
   uint64_t NextBits;
case ISD::FREEZE:
     Select_FREEZE(NodeToMatch);
     return;
   case ISD::ARITH_FENCE:
     Select_ARITH_FENCE(NodeToMatch);
     return;
   case ISD::STACKMAP:
     Select_STACKMAP(NodeToMatch);
     return;
+  case ISD::PATCHPOINT:
+    Select_PATCHPOINT(NodeToMatch);
+    return;
   }
 
   assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
 
   // Set up the node stack with NodeToMatch as the only node on the stack.
   SmallVector<SDValue, 8> NodeStack;
   SDValue N = SDValue(NodeToMatch, 0);
   NodeStack.push_back(N);

llvm/test/CodeGen/AArch64/stackmap.ll

 ; CHECK-NEXT:   .hword  0
 ; CHECK-NEXT:   .hword  14
 ; SmallConstant
 ; CHECK-NEXT:   .byte   4
 ; CHECK-NEXT:   .byte   0
 ; CHECK-NEXT:   .hword  8
 ; CHECK-NEXT:   .hword  0
 ; CHECK-NEXT:   .hword  0
-; CHECK-NEXT:   .word   -1
+; CHECK-NEXT:   .word   65535
 ; SmallConstant
 ; CHECK-NEXT:   .byte   4
 ; CHECK-NEXT:   .byte   0
 ; CHECK-NEXT:   .hword  8
 ; CHECK-NEXT:   .hword  0
 ; CHECK-NEXT:   .hword  0
-; CHECK-NEXT:   .word   -1
+; CHECK-NEXT:   .word   65535
 ; SmallConstant
 ; CHECK-NEXT:   .byte   4
 ; CHECK-NEXT:   .byte   0
 ; CHECK-NEXT:   .hword  8
 ; CHECK-NEXT:   .hword  0
 ; CHECK-NEXT:   .hword  0
 ; CHECK-NEXT:   .word   65536
 ; SmallConstant

llvm/test/CodeGen/SystemZ/stackmap.ll

 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .short  14
 ; SmallConstant
 ; CHECK-NEXT:   .byte   4
 ; CHECK-NEXT:   .byte   0
 ; CHECK-NEXT:   .short  8
 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .short  0
-; CHECK-NEXT:   .long   -1
+; CHECK-NEXT:   .long   65535
 ; SmallConstant
 ; CHECK-NEXT:   .byte   4
 ; CHECK-NEXT:   .byte   0
 ; CHECK-NEXT:   .short  8
 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .short  0
-; CHECK-NEXT:   .long   -1
+; CHECK-NEXT:   .long   65535
 ; SmallConstant
 ; CHECK-NEXT:   .byte   4
 ; CHECK-NEXT:   .byte   0
 ; CHECK-NEXT:   .short  8
 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .short  0
 ; CHECK-NEXT:   .long   65536
 ; SmallConstant

llvm/test/CodeGen/X86/selectiondag-patchpoint-legalize.ll

+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -enable-patchpoint-liveness=false | FileCheck %s
+
+; CHECK-LABEL:  .section  __LLVM_STACKMAPS,__llvm_stackmaps
+; CHECK-NEXT:   __LLVM_StackMaps:
+
+; Header
+; CHECK-NEXT:   .byte 3
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 0
+
+; NumFunctions
+; CHECK-NEXT:   .long 1
+; NumConstants
+; CHECK-NEXT:   .long 0
+; NumRecords
+; CHECK-NEXT:   .long 1
+
+; StackSizeRecord[NumFunctions]
+;   StackSizeRecord[0]
+;     CHECK-NEXT:   .quad _main
+;     CHECK-NEXT:   .quad 24
+;     CHECK-NEXT:   .quad 1
+
+; Constants[NumConstants] (empty)
+
+; StkMapRecord[NumRecords]
+;   StkMapRecord[0]
+;     CHECK-NEXT: .quad 0
+;     CHECK-NEXT:   .long {{.*}}
+;     CHECK-NEXT:   .short {{.*}}
+;     NumLocations
+;     CHECK-NEXT:   .short 11
+;     Location[NumLocations]
+;       Location[0]
+;         CHECK-NEXT: .byte   1
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  1
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[1]
+;         CHECK-NEXT: .byte   4
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  8
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   22
+;       Location[2]
+;         CHECK-NEXT: .byte   1
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  1
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[3]
+;         CHECK-NEXT: .byte   1
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  16
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[4]
+;         CHECK-NEXT: .byte   1
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  16
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[5]
+;         CHECK-NEXT: .byte   4
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  8
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   66
+;       Location[6]
+;         CHECK-NEXT: .byte   1
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  4
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[7]
+;         CHECK-NEXT: .byte   4
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  8
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[8]
+;         CHECK-NEXT: .byte   1
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  4
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[9]
+;         CHECK-NEXT: .byte   4
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  8
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+;       Location[10]
+;         CHECK-NEXT: .byte   1
+;         CHECK-NEXT: .byte   0
+;         CHECK-NEXT: .short  1
+;         CHECK-NEXT: .short  {{.*}}
+;         CHECK-NEXT: .short  0
+;         CHECK-NEXT: .long   0
+
+@p32 = external global i8 addrspace(270)*
+
+%struct1 = type {i32, i64}
+%struct2 = type {i1, i1, i1}
+
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
+
+define dso_local i32 @main(i32 %argc, i8** %argv) {
+entry:
+  %i1reg = icmp eq i32 %argc, 5
+  %i7reg = zext i1 %i1reg to i7
+  %halfreg = sitofp i32 %argc to half
+  %ptr32 = load i8 addrspace(270)*, i8 addrspace(270)** @p32
+  %structreg1 = insertvalue %struct1 zeroinitializer, i32 %argc, 0
+  %structreg2 = insertvalue %struct2 zeroinitializer, i1 %i1reg, 0
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(
+    i64 0,
+    i32 0,
+    i8* null,
+    i32 0,
+    i1 %i1reg,
+    i7 22,
+    i7 %i7reg,
+    half 1.0,
+    half %halfreg,
+    i128 66,
+    ; FIXME: fix and test vectors. At the moment even legally sized vectors
+    ; are broken:
+    ; https://github.com/llvm/llvm-project/issues/55613
+    ;
+    ; FIXME: test non-constant i128 once these are fixed:
+    ;  - https://github.com/llvm/llvm-project/issues/26431
+    ;  - https://github.com/llvm/llvm-project/issues/55957
+    i8 addrspace(270)* %ptr32,
+    ; FIXME: The stackmap record generated for structs is incorrect:
+    ;  - https://github.com/llvm/llvm-project/issues/55649
+    ;  - https://github.com/llvm/llvm-project/issues/55957
+    %struct1 zeroinitializer,
+    %struct1 %structreg1,
+    %struct2 zeroinitializer,
+    %struct2 %structreg2)
+  ret i32 0
+}