[RISCV] Convert VSLIDE(UP|DOWN) nodes to "VL" versions (NFC)

[frasercrmck]

Summary

This patch prepares the RISCV VSLIDEUP and VSLIDEDOWN custom nodes to
ones carrying additional mask and vector-length operands. This is
primarily so they can be used by both systems.

This also takes the opportunity to create some helper functions to deal
with the common task of getting the default (unmasked) VL operands.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

823

Do these need to be EVT or can they be MVT? May need to use getSimpleValueType in some of the callers.

[frasercrmck]

• rebase on main
• use MVTs instead of EVTs

[frasercrmck]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

823

Thinking again, I don't think they need to be EVTs since these methods should only be used to create legal nodes. Will update.

[craig.topper]

LGTM

[arcbbb]

LGTM!

[frasercrmck]

• rebase
• add another trivial use of getDefaultVLOps

llvm/lib/Target/RISCV/RISCVISelLowering.h

enum NodeType : unsigned {
   // Read VLENB CSR
   READ_VLENB,
   // Truncates a RVV integer vector by one power-of-two.
   TRUNCATE_VECTOR,
   // Unit-stride fault-only-first load
   VLEFF,
   VLEFF_MASK,
   // Matches the semantics of vslideup/vslidedown. The first operand is the
-  // pass-thru operand, the second is the source vector, and the third is the
-  // XLenVT index (either constant or non-constant).
-  VSLIDEUP,
-  VSLIDEDOWN,
+  // pass-thru operand, the second is the source vector, the third is the
+  // XLenVT index (either constant or non-constant), the fourth is the mask
+  // and the fifth the VL.
+  VSLIDEUP_VL,
+  VSLIDEDOWN_VL,
   // Matches the semantics of the vid.v instruction, with a mask and VL
   // operand.
   VID_VL,
   // Matches the semantics of the vfcnvt.rod function (Convert double-width
   // float to single-width float, rounding towards odd). Takes a double-width
   // float vector and produces a single-width float vector.
   VFNCVT_ROD,
   // These nodes match the semantics of the corresponding RVV vector reduction

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

assert(VT.isFixedLengthVector() &&
          "Expected to convert into a fixed length vector!");
   assert(V.getValueType().isScalableVector() &&
          "Expected a scalable vector operand!");
   SDLoc DL(V);
   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
 }
 
+// Gets the two common "VL" operands: an all-ones mask and the vector length.
+// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
+// the vector type that it is contained in.
+static std::pair<SDValue, SDValue>
+getDefaultVLOps(EVT VecVT, EVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
+                const RISCVSubtarget &Subtarget) {
+  assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
+  MVT XLenVT = Subtarget.getXLenVT();
+  SDValue VL = VecVT.isFixedLengthVector()
+                   ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
+                   : DAG.getRegister(RISCV::X0, XLenVT);
+  MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
+  SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
+  return {Mask, VL};
+}
+
+// As above but assuming the given type is a scalable vector type.
+static std::pair<SDValue, SDValue>
+getDefaultScalableVLOps(EVT VecVT, SDLoc DL, SelectionDAG &DAG,
+                        const RISCVSubtarget &Subtarget) {
+  assert(VecVT.isScalableVector() && "Expecting a scalable vector");
+  return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
+}
+
 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
   MVT VT = Op.getSimpleValueType();
   assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
 
   SDLoc DL(Op);
-  SDValue VL =
-      DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
+  SDValue Mask, VL;
+  std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
     unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
                                         : RISCVISD::VMV_V_X_VL;
     Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
     return convertFromScalableVector(VT, Splat, DAG, Subtarget);
   }
 
   // Try and match an index sequence, which we can lower directly to the vid
   // instruction. An all-undef vector is matched by getSplatValue, above.
   bool IsVID = true;
   if (VT.isInteger())
     for (unsigned i = 0, e = Op.getNumOperands(); i < e && IsVID; i++)
       IsVID &= Op.getOperand(i).isUndef() ||
                (isa<ConstantSDNode>(Op.getOperand(i)) &&
                 Op.getConstantOperandVal(i) == i);
 
   if (IsVID) {
-    MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
-    SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
     SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
     return convertFromScalableVector(VT, VID, DAG, Subtarget);
   }
 
   return SDValue();
 }
 
 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
 
   // Custom-legalize INSERT_VECTOR_ELT where XLEN>=SEW, so that the vector is
   // first slid down into position, the value is inserted into the first
   // position, and the vector is slid back up. We do this to simplify patterns.
   //   (slideup vec, (insertelt (slidedown impdef, vec, idx), val, 0), idx),
   if (Subtarget.is64Bit() || VecVT.getVectorElementType() != MVT::i64) {
     if (isNullConstant(Idx))
       return Op;
-    SDValue Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN, DL, VecVT,
-                                    DAG.getUNDEF(VecVT), Vec, Idx);
+    SDValue Mask, VL;
+    std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+    SDValue Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VecVT,
+                                    DAG.getUNDEF(VecVT), Vec, Idx, Mask, VL);
     SDValue InsertElt0 =
         DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecVT, Slidedown, Val,
                     DAG.getConstant(0, DL, Subtarget.getXLenVT()));
 
-    return DAG.getNode(RISCVISD::VSLIDEUP, DL, VecVT, Vec, InsertElt0, Idx);
+    return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VecVT, Vec, InsertElt0, Idx,
+                       Mask, VL);
   }
 
   // Custom-legalize INSERT_VECTOR_ELT where XLEN<SEW, as the SEW element type
   // is illegal (currently only vXi64 RV32).
   // Since there is no easy way of getting a single element into a vector when
   // XLEN<SEW, we lower the operation to the following sequence:
   //   splat      vVal, rVal
   //   vid.v      vVid
   //   vmseq.vx   mMask, vVid, rIdx
   //   vmerge.vvm vDest, vSrc, vVal, mMask
   // This essentially merges the original vector with the inserted element by
   // using a mask whose only set bit is that corresponding to the insert
   // index.
   SDValue SplattedVal = DAG.getSplatVector(VecVT, DL, Val);
   SDValue SplattedIdx = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Idx);
 
-  SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
-  MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
-  SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
+  SDValue Mask, VL;
+  std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
   SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VecVT, Mask, VL);
   auto SetCCVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VecVT);
   SDValue SelectCond = DAG.getSetCC(DL, SetCCVT, VID, SplattedIdx, ISD::SETEQ);
 
   return DAG.getNode(ISD::VSELECT, DL, VecVT, SelectCond, SplattedVal, Vec);
 }
 
 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
 // types this is done using VMV_X_S to allow us to glean information about the
 // sign bits of the result.
 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
                                                      SelectionDAG &DAG) const {
   SDLoc DL(Op);
   SDValue Idx = Op.getOperand(1);
   SDValue Vec = Op.getOperand(0);
   EVT EltVT = Op.getValueType();
   EVT VecVT = Vec.getValueType();
   MVT XLenVT = Subtarget.getXLenVT();
 
   // If the index is 0, the vector is already in the right position.
   if (!isNullConstant(Idx)) {
-    Vec = DAG.getNode(RISCVISD::VSLIDEDOWN, DL, VecVT, DAG.getUNDEF(VecVT), Vec,
-                      Idx);
+    SDValue Mask, VL;
+    std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+    Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VecVT, DAG.getUNDEF(VecVT),
+                      Vec, Idx, Mask, VL);
   }
 
   if (!EltVT.isInteger()) {
     // Floating-point extracts are handled in TableGen.
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
                        DAG.getConstant(0, DL, XLenVT));
   }
 
for (const SDValue &V : Op->op_values()) {
 
     // "cast" fixed length vector to a scalable vector.
     assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
            "Only fixed length vectors are supported!");
     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
   }
 
   SDLoc DL(Op);
-  SDValue VL =
-      DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
-  MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
-  SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
+  SDValue Mask, VL;
+  std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
   Ops.push_back(Mask);
   Ops.push_back(VL);
 
   SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
 }
 
 // Returns the opcode of the target-specific SDNode that implements the 32-bit
case ISD::EXTRACT_VECTOR_ELT: {
     SDValue Vec = N->getOperand(0);
     SDValue Idx = N->getOperand(1);
     EVT VecVT = Vec.getValueType();
     assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
            VecVT.getVectorElementType() == MVT::i64 &&
            "Unexpected EXTRACT_VECTOR_ELT legalization");
 
     SDValue Slidedown = Vec;
+    MVT XLenVT = Subtarget.getXLenVT();
     // Unless the index is known to be 0, we must slide the vector down to get
     // the desired element into index 0.
-    if (!isNullConstant(Idx))
-      Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN, DL, VecVT,
-                              DAG.getUNDEF(VecVT), Vec, Idx);
+    if (!isNullConstant(Idx)) {
+      SDValue Mask, VL;
+      std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+      Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VecVT,
+                              DAG.getUNDEF(VecVT), Vec, Idx, Mask, VL);
+    }
 
-    MVT XLenVT = Subtarget.getXLenVT();
     // Extract the lower XLEN bits of the correct vector element.
     SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Slidedown, Idx);
 
     // To extract the upper XLEN bits of the vector element, shift the first
     // element right by 32 bits and re-extract the lower XLEN bits.
     SDValue ThirtyTwoV =
         DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT,
                     DAG.getConstant(32, DL, Subtarget.getXLenVT()));
#define NODE_NAME_CASE(NODE)                                                   \
   NODE_NAME_CASE(VMV_V_X_VL)
   NODE_NAME_CASE(VFMV_V_F_VL)
   NODE_NAME_CASE(VMV_X_S)
   NODE_NAME_CASE(SPLAT_VECTOR_I64)
   NODE_NAME_CASE(READ_VLENB)
   NODE_NAME_CASE(TRUNCATE_VECTOR)
   NODE_NAME_CASE(VLEFF)
   NODE_NAME_CASE(VLEFF_MASK)
-  NODE_NAME_CASE(VSLIDEUP)
-  NODE_NAME_CASE(VSLIDEDOWN)
+  NODE_NAME_CASE(VSLIDEUP_VL)
+  NODE_NAME_CASE(VSLIDEDOWN_VL)
   NODE_NAME_CASE(VID_VL)
   NODE_NAME_CASE(VFNCVT_ROD)
   NODE_NAME_CASE(VECREDUCE_ADD)
   NODE_NAME_CASE(VECREDUCE_UMAX)
   NODE_NAME_CASE(VECREDUCE_SMAX)
   NODE_NAME_CASE(VECREDUCE_UMIN)
   NODE_NAME_CASE(VECREDUCE_SMIN)
   NODE_NAME_CASE(VECREDUCE_AND)

llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td

def : Pat<(vti.Scalar (extractelt (vti.Vector vti.RegClass:$rs2), 0)),
             (vmv_f_s_inst vti.RegClass:$rs2, vti.SEW)>;
 
   def : Pat<(vti.Vector (insertelt (vti.Vector vti.RegClass:$merge),
                                    vti.ScalarRegClass:$rs1, 0)),
             (vmv_s_f_inst vti.RegClass:$merge,
                     (vti.Scalar vti.ScalarRegClass:$rs1),
                     vti.AVL, vti.SEW)>;
 }
-
-def SDTRVVSlide : SDTypeProfile<1, 3, [
-  SDTCisVec<0>, SDTCisSameAs<1, 0>, SDTCisSameAs<2, 0>, SDTCisVT<3, XLenVT>
-]>;
-
-def riscv_slideup : SDNode<"RISCVISD::VSLIDEUP", SDTRVVSlide, []>;
-def riscv_slidedown : SDNode<"RISCVISD::VSLIDEDOWN", SDTRVVSlide, []>;
-
-let Predicates = [HasStdExtV] in {
-
-foreach vti = !listconcat(AllIntegerVectors, AllFloatVectors) in {
-    def : Pat<(vti.Vector (riscv_slideup (vti.Vector vti.RegClass:$rs3),
-                                         (vti.Vector vti.RegClass:$rs1),
-                                         uimm5:$rs2)),
-              (!cast<Instruction>("PseudoVSLIDEUP_VI_"#vti.LMul.MX)
-                  vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
-                  vti.AVL, vti.SEW)>;
-
-    def : Pat<(vti.Vector (riscv_slideup (vti.Vector vti.RegClass:$rs3),
-                                         (vti.Vector vti.RegClass:$rs1),
-                                         GPR:$rs2)),
-              (!cast<Instruction>("PseudoVSLIDEUP_VX_"#vti.LMul.MX)
-                  vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
-                  vti.AVL, vti.SEW)>;
-
-    def : Pat<(vti.Vector (riscv_slidedown (vti.Vector vti.RegClass:$rs3),
-                                           (vti.Vector vti.RegClass:$rs1),
-                                           uimm5:$rs2)),
-              (!cast<Instruction>("PseudoVSLIDEDOWN_VI_"#vti.LMul.MX)
-                  vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
-                  vti.AVL, vti.SEW)>;
-
-    def : Pat<(vti.Vector (riscv_slidedown (vti.Vector vti.RegClass:$rs3),
-                                           (vti.Vector vti.RegClass:$rs1),
-                                           GPR:$rs2)),
-              (!cast<Instruction>("PseudoVSLIDEDOWN_VX_"#vti.LMul.MX)
-                  vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
-                  vti.AVL, vti.SEW)>;
-}
-} // Predicates = [HasStdExtV]

llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td

 //===----------------------------------------------------------------------===//
 // Miscellaneous RISCVISD SDNodes
 //===----------------------------------------------------------------------===//
 
 def riscv_vid_vl : SDNode<"RISCVISD::VID_VL", SDTypeProfile<1, 2,
                           [SDTCisVec<0>, SDTCisVec<1>, SDTCVecEltisVT<1, i1>,
                            SDTCisSameNumEltsAs<0, 1>, SDTCisVT<2, XLenVT>]>, []>;
 
+def SDTRVVSlide : SDTypeProfile<1, 5, [
+  SDTCisVec<0>, SDTCisSameAs<1, 0>, SDTCisSameAs<2, 0>, SDTCisVT<3, XLenVT>,
+  SDTCVecEltisVT<4, i1>, SDTCisSameNumEltsAs<0, 4>, SDTCisVT<5, XLenVT>
+]>;
+
+def riscv_slideup_vl   : SDNode<"RISCVISD::VSLIDEUP_VL", SDTRVVSlide, []>;
+def riscv_slidedown_vl : SDNode<"RISCVISD::VSLIDEDOWN_VL", SDTRVVSlide, []>;
+
 let Predicates = [HasStdExtV] in {
 
 foreach vti = AllIntegerVectors in
   def : Pat<(vti.Vector (riscv_vid_vl (vti.Mask true_mask),
                                       (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVID_V_"#vti.LMul.MX) GPR:$vl, vti.SEW)>;
 
+foreach vti = !listconcat(AllIntegerVectors, AllFloatVectors) in {
+  def : Pat<(vti.Vector (riscv_slideup_vl (vti.Vector vti.RegClass:$rs3),
+                                          (vti.Vector vti.RegClass:$rs1),
+                                          uimm5:$rs2, (vti.Mask true_mask),
+                                          (XLenVT (VLOp GPR:$vl)))),
+            (!cast<Instruction>("PseudoVSLIDEUP_VI_"#vti.LMul.MX)
+                vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
+                GPR:$vl, vti.SEW)>;
+
+  def : Pat<(vti.Vector (riscv_slideup_vl (vti.Vector vti.RegClass:$rs3),
+                                          (vti.Vector vti.RegClass:$rs1),
+                                          GPR:$rs2, (vti.Mask true_mask),
+                                          (XLenVT (VLOp GPR:$vl)))),
+            (!cast<Instruction>("PseudoVSLIDEUP_VX_"#vti.LMul.MX)
+                vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
+                GPR:$vl, vti.SEW)>;
+
+  def : Pat<(vti.Vector (riscv_slidedown_vl (vti.Vector vti.RegClass:$rs3),
+                                            (vti.Vector vti.RegClass:$rs1),
+                                            uimm5:$rs2, (vti.Mask true_mask),
+                                            (XLenVT (VLOp GPR:$vl)))),
+            (!cast<Instruction>("PseudoVSLIDEDOWN_VI_"#vti.LMul.MX)
+                vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
+                GPR:$vl, vti.SEW)>;
+
+  def : Pat<(vti.Vector (riscv_slidedown_vl (vti.Vector vti.RegClass:$rs3),
+                                            (vti.Vector vti.RegClass:$rs1),
+                                            GPR:$rs2, (vti.Mask true_mask),
+                                            (XLenVT (VLOp GPR:$vl)))),
+            (!cast<Instruction>("PseudoVSLIDEDOWN_VX_"#vti.LMul.MX)
+                vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
+                GPR:$vl, vti.SEW)>;
+}
+
 } // Predicates = [HasStdExtV]