[RISCV] Merge the handlers for masked and unmasked segment loads/stores.

[craig.topper]

Summary

A lot of the code for the masked and unmasked is the same. This
patch adds a boolean to handle the differences so we can share
the code.

[frasercrmck]

LGTM.

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h

template <unsigned Width> bool selectRVVSimm5(SDValue N, SDValue &Imm) {
     return selectRVVSimm5(N, Width, Imm);
   }
 
   bool selectRVVUimm5(SDValue N, unsigned Width, SDValue &Imm);
   template <unsigned Width> bool selectRVVUimm5(SDValue N, SDValue &Imm) {
     return selectRVVUimm5(N, Width, Imm);
   }
 
-  void selectVLSEG(SDNode *Node, unsigned IntNo, bool IsStrided);
-  void selectVLSEGMask(SDNode *Node, unsigned IntNo, bool IsStrided);
-  void selectVLSEGFF(SDNode *Node);
-  void selectVLSEGFFMask(SDNode *Node);
-  void selectVLXSEG(SDNode *Node, unsigned IntNo);
-  void selectVLXSEGMask(SDNode *Node, unsigned IntNo);
-  void selectVSSEG(SDNode *Node, unsigned IntNo, bool IsStrided);
-  void selectVSSEGMask(SDNode *Node, unsigned IntNo, bool IsStrided);
-  void selectVSXSEG(SDNode *Node, unsigned IntNo);
-  void selectVSXSEGMask(SDNode *Node, unsigned IntNo);
+  void selectVLSEG(SDNode *Node, unsigned IntNo, bool IsMasked, bool IsStrided);
+  void selectVLSEGFF(SDNode *Node, bool IsMasked);
+  void selectVLXSEG(SDNode *Node, unsigned IntNo, bool IsMasked);
+  void selectVSSEG(SDNode *Node, unsigned IntNo, bool IsMasked, bool IsStrided);
+  void selectVSXSEG(SDNode *Node, unsigned IntNo, bool IsMasked);
 
 // Include the pieces autogenerated from the target description.
 #include "RISCVGenDAGISel.inc"
 
 private:
   void doPeepholeLoadStoreADDI();
 };
 }
 
 #endif

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

case RISCVVLMUL::LMUL_1:
     return createM1Tuple(CurDAG, Regs, NF);
   case RISCVVLMUL::LMUL_2:
     return createM2Tuple(CurDAG, Regs, NF);
   case RISCVVLMUL::LMUL_4:
     return createM4Tuple(CurDAG, Regs, NF);
   }
 }
 
-void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned IntNo,
+void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned IntNo, bool IsMasked,
                                     bool IsStrided) {
   SDLoc DL(Node);
   unsigned NF = Node->getNumValues() - 1;
   MVT VT = Node->getSimpleValueType(0);
   unsigned ScalarSize = VT.getScalarSizeInBits();
   MVT XLenVT = Subtarget->getXLenVT();
   RISCVVLMUL LMUL = getLMUL(VT);
   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SmallVector<SDValue, 5> Operands;
-  Operands.push_back(Node->getOperand(2)); // Base pointer.
-  if (IsStrided) {
-    Operands.push_back(Node->getOperand(3)); // Stride.
-    Operands.push_back(Node->getOperand(4)); // VL.
-  } else {
-    Operands.push_back(Node->getOperand(3)); // VL.
-  }
-  Operands.push_back(SEW);
-  Operands.push_back(Node->getOperand(0)); // Chain.
-  const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
-      IntNo, ScalarSize, static_cast<unsigned>(LMUL),
-      static_cast<unsigned>(RISCVVLMUL::LMUL_1));
-  SDNode *Load =
-      CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
-  SDValue SuperReg = SDValue(Load, 0);
-  for (unsigned I = 0; I < NF; ++I)
-    ReplaceUses(SDValue(Node, I),
-                CurDAG->getTargetExtractSubreg(getSubregIndexByMVT(VT, I), DL,
-                                               VT, SuperReg));
-
-  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
-  CurDAG->RemoveDeadNode(Node);
-}
-
-void RISCVDAGToDAGISel::selectVLSEGMask(SDNode *Node, unsigned IntNo,
-                                        bool IsStrided) {
-  SDLoc DL(Node);
-  unsigned NF = Node->getNumValues() - 1;
-  MVT VT = Node->getSimpleValueType(0);
-  unsigned ScalarSize = VT.getScalarSizeInBits();
-  MVT XLenVT = Subtarget->getXLenVT();
-  RISCVVLMUL LMUL = getLMUL(VT);
-  SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
-  SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
+  unsigned CurOp = 2;
   SmallVector<SDValue, 7> Operands;
+  if (IsMasked) {
+    SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
+                                 Node->op_begin() + CurOp + NF);
+    SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
     Operands.push_back(MaskedOff);
-  Operands.push_back(Node->getOperand(NF + 2)); // Base pointer.
-  if (IsStrided) {
-    Operands.push_back(Node->getOperand(NF + 3)); // Stride.
-    Operands.push_back(Node->getOperand(NF + 4)); // Mask.
-    Operands.push_back(Node->getOperand(NF + 5)); // VL.
-  } else {
-    Operands.push_back(Node->getOperand(NF + 3)); // Mask.
-    Operands.push_back(Node->getOperand(NF + 4)); // VL.
+    CurOp += NF;
   }
+  Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
+  if (IsStrided)
+    Operands.push_back(Node->getOperand(CurOp++)); // Stride.
+  if (IsMasked)
+    Operands.push_back(Node->getOperand(CurOp++)); // Mask.
+  Operands.push_back(Node->getOperand(CurOp++));   // VL.
   Operands.push_back(SEW);
-  Operands.push_back(Node->getOperand(0)); /// Chain.
+  Operands.push_back(Node->getOperand(0)); // Chain.
   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
   SDNode *Load =
       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
   SDValue SuperReg = SDValue(Load, 0);
   for (unsigned I = 0; I < NF; ++I)
     ReplaceUses(SDValue(Node, I),
                 CurDAG->getTargetExtractSubreg(getSubregIndexByMVT(VT, I), DL,
                                                VT, SuperReg));
 
   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
   CurDAG->RemoveDeadNode(Node);
 }
 
-void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) {
+void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
   SDLoc DL(Node);
   unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
   unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
   MVT VT = Node->getSimpleValueType(0);
-  unsigned ScalarSize = VT.getScalarSizeInBits();
   MVT XLenVT = Subtarget->getXLenVT();
-  RISCVVLMUL LMUL = getLMUL(VT);
-  SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SmallVector<SDValue, 5> Operands;
-  Operands.push_back(Node->getOperand(2)); // Base pointer.
-  Operands.push_back(Node->getOperand(3)); // VL.
-  Operands.push_back(SEW);
-  Operands.push_back(Node->getOperand(0)); // Chain.
-  const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
-      IntNo, ScalarSize, static_cast<unsigned>(LMUL),
-      static_cast<unsigned>(RISCVVLMUL::LMUL_1));
-  SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
-                                        MVT::Glue, Operands);
-  SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
-                                          /*Glue*/ SDValue(Load, 2));
-
-  SDValue SuperReg = SDValue(Load, 0);
-  for (unsigned I = 0; I < NF; ++I)
-    ReplaceUses(SDValue(Node, I),
-                CurDAG->getTargetExtractSubreg(getSubregIndexByMVT(VT, I), DL,
-                                               VT, SuperReg));
-
-  ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
-  ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
-  CurDAG->RemoveDeadNode(Node);
-}
-
-void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) {
-  SDLoc DL(Node);
-  unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
-  unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
-  MVT VT = Node->getSimpleValueType(0);
   unsigned ScalarSize = VT.getScalarSizeInBits();
-  MVT XLenVT = Subtarget->getXLenVT();
   RISCVVLMUL LMUL = getLMUL(VT);
   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
-  SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
+
+  unsigned CurOp = 2;
   SmallVector<SDValue, 7> Operands;
+  if (IsMasked) {
+    SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
+                                 Node->op_begin() + CurOp + NF);
+    SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
     Operands.push_back(MaskedOff);
-  Operands.push_back(Node->getOperand(NF + 2)); // Base pointer.
-  Operands.push_back(Node->getOperand(NF + 3)); // Mask.
-  Operands.push_back(Node->getOperand(NF + 4)); // VL.
+    CurOp += NF;
+  }
+  Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
+  if (IsMasked)
+    Operands.push_back(Node->getOperand(CurOp++)); // Mask.
+  Operands.push_back(Node->getOperand(CurOp++));   // VL.
   Operands.push_back(SEW);
-  Operands.push_back(Node->getOperand(0)); /// Chain.
+  Operands.push_back(Node->getOperand(0)); // Chain.
   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
   SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
                                         MVT::Glue, Operands);
   SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
                                           /*Glue*/ SDValue(Load, 2));
 
   SDValue SuperReg = SDValue(Load, 0);
   for (unsigned I = 0; I < NF; ++I)
     ReplaceUses(SDValue(Node, I),
                 CurDAG->getTargetExtractSubreg(getSubregIndexByMVT(VT, I), DL,
                                                VT, SuperReg));
 
   ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
   CurDAG->RemoveDeadNode(Node);
 }
 
-void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned IntNo) {
+void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned IntNo,
+                                     bool IsMasked) {
   SDLoc DL(Node);
   unsigned NF = Node->getNumValues() - 1;
   MVT VT = Node->getSimpleValueType(0);
   unsigned ScalarSize = VT.getScalarSizeInBits();
   MVT XLenVT = Subtarget->getXLenVT();
   RISCVVLMUL LMUL = getLMUL(VT);
   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SDValue Operands[] = {
-      Node->getOperand(2),     // Base pointer.
-      Node->getOperand(3),     // Index.
-      Node->getOperand(4),     // VL.
-      SEW, Node->getOperand(0) // Chain.
-  };
-
-  MVT IndexVT = Node->getOperand(3)->getSimpleValueType(0);
-  RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
-  unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
-  const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
-      IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
-      static_cast<unsigned>(IndexLMUL));
-  SDNode *Load =
-      CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
-  SDValue SuperReg = SDValue(Load, 0);
-  for (unsigned I = 0; I < NF; ++I)
-    ReplaceUses(SDValue(Node, I),
-                CurDAG->getTargetExtractSubreg(getSubregIndexByMVT(VT, I), DL,
-                                               VT, SuperReg));
-
-  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
-  CurDAG->RemoveDeadNode(Node);
-}
-
-void RISCVDAGToDAGISel::selectVLXSEGMask(SDNode *Node, unsigned IntNo) {
-  SDLoc DL(Node);
-  unsigned NF = Node->getNumValues() - 1;
-  MVT VT = Node->getSimpleValueType(0);
-  unsigned ScalarSize = VT.getScalarSizeInBits();
-  MVT XLenVT = Subtarget->getXLenVT();
-  RISCVVLMUL LMUL = getLMUL(VT);
-  SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
+  unsigned CurOp = 2;
+  SmallVector<SDValue, 7> Operands;
+  if (IsMasked) {
+    SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
+                                 Node->op_begin() + CurOp + NF);
     SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
-  SDValue Operands[] = {
-      MaskedOff,
-      Node->getOperand(NF + 2), // Base pointer.
-      Node->getOperand(NF + 3), // Index.
-      Node->getOperand(NF + 4), // Mask.
-      Node->getOperand(NF + 5), // VL.
-      SEW,
-      Node->getOperand(0) // Chain.
-  };
+    Operands.push_back(MaskedOff);
+    CurOp += NF;
+  }
+  Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
+  Operands.push_back(Node->getOperand(CurOp++)); // Index.
+  MVT IndexVT = Operands.back()->getSimpleValueType(0);
+  if (IsMasked)
+    Operands.push_back(Node->getOperand(CurOp++)); // Mask.
+  Operands.push_back(Node->getOperand(CurOp++));   // VL.
+  Operands.push_back(SEW);
+  Operands.push_back(Node->getOperand(0)); // Chain.
 
-  MVT IndexVT = Node->getOperand(NF + 3)->getSimpleValueType(0);
   RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
       IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
       static_cast<unsigned>(IndexLMUL));
   SDNode *Load =
       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
   SDValue SuperReg = SDValue(Load, 0);
   for (unsigned I = 0; I < NF; ++I)
     ReplaceUses(SDValue(Node, I),
                 CurDAG->getTargetExtractSubreg(getSubregIndexByMVT(VT, I), DL,
                                                VT, SuperReg));
 
   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
   CurDAG->RemoveDeadNode(Node);
 }
 
-void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned IntNo,
+void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned IntNo, bool IsMasked,
                                     bool IsStrided) {
   SDLoc DL(Node);
   unsigned NF = Node->getNumOperands() - 4;
   if (IsStrided)
     NF--;
+  if (IsMasked)
+    NF--;
   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
   unsigned ScalarSize = VT.getScalarSizeInBits();
   MVT XLenVT = Subtarget->getXLenVT();
   RISCVVLMUL LMUL = getLMUL(VT);
   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
-  SmallVector<SDValue, 6> Operands;
+  SmallVector<SDValue, 7> Operands;
   Operands.push_back(StoreVal);
-  Operands.push_back(Node->getOperand(2 + NF)); // Base pointer.
-  if (IsStrided) {
-    Operands.push_back(Node->getOperand(3 + NF)); // Stride.
-    Operands.push_back(Node->getOperand(4 + NF)); // VL.
-  } else {
-    Operands.push_back(Node->getOperand(3 + NF)); // VL.
-  }
+  unsigned CurOp = 2 + NF;
+  Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
+  if (IsStrided)
+    Operands.push_back(Node->getOperand(CurOp++)); // Stride.
+  if (IsMasked)
+    Operands.push_back(Node->getOperand(CurOp++)); // Mask.
+  Operands.push_back(Node->getOperand(CurOp++));   // VL.
   Operands.push_back(SEW);
   Operands.push_back(Node->getOperand(0)); // Chain.
   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
   SDNode *Store =
       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
   ReplaceNode(Node, Store);
 }
 
-void RISCVDAGToDAGISel::selectVSSEGMask(SDNode *Node, unsigned IntNo,
-                                        bool IsStrided) {
+void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned IntNo,
+                                     bool IsMasked) {
   SDLoc DL(Node);
   unsigned NF = Node->getNumOperands() - 5;
-  if (IsStrided)
-    NF--;
+  if (IsMasked)
+    --NF;
   MVT VT = Node->getOperand(2)->getSimpleValueType(0);
   unsigned ScalarSize = VT.getScalarSizeInBits();
   MVT XLenVT = Subtarget->getXLenVT();
   RISCVVLMUL LMUL = getLMUL(VT);
   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
+  SmallVector<SDValue, 7> Operands;
   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
-  SmallVector<SDValue, 7> Operands;
   Operands.push_back(StoreVal);
-  Operands.push_back(Node->getOperand(2 + NF)); // Base pointer.
-  if (IsStrided) {
-    Operands.push_back(Node->getOperand(3 + NF)); // Stride.
-    Operands.push_back(Node->getOperand(4 + NF)); // Mask.
-    Operands.push_back(Node->getOperand(5 + NF)); // VL.
-  } else {
-    Operands.push_back(Node->getOperand(3 + NF)); // Mask.
-    Operands.push_back(Node->getOperand(4 + NF)); // VL.
-  }
+  unsigned CurOp = 2 + NF;
+  Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
+  Operands.push_back(Node->getOperand(CurOp++)); // Index.
+  MVT IndexVT = Operands.back()->getSimpleValueType(0);
+  if (IsMasked)
+    Operands.push_back(Node->getOperand(CurOp++)); // Mask.
+  Operands.push_back(Node->getOperand(CurOp++));   // VL.
   Operands.push_back(SEW);
   Operands.push_back(Node->getOperand(0)); // Chain.
-  const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
-      IntNo, ScalarSize, static_cast<unsigned>(LMUL),
-      static_cast<unsigned>(RISCVVLMUL::LMUL_1));
-  SDNode *Store =
-      CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
-  ReplaceNode(Node, Store);
-}
-
-void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned IntNo) {
-  SDLoc DL(Node);
-  unsigned NF = Node->getNumOperands() - 5;
-  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
-  unsigned ScalarSize = VT.getScalarSizeInBits();
-  MVT XLenVT = Subtarget->getXLenVT();
-  RISCVVLMUL LMUL = getLMUL(VT);
-  SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
-  SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
-  SDValue Operands[] = {
-      StoreVal,
-      Node->getOperand(2 + NF), // Base pointer.
-      Node->getOperand(3 + NF), // Index.
-      Node->getOperand(4 + NF), // VL.
-      SEW,
-      Node->getOperand(0) // Chain.
-  };
-
-  MVT IndexVT = Node->getOperand(3 + NF)->getSimpleValueType(0);
-  RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
-  unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
-  const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
-      IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
-      static_cast<unsigned>(IndexLMUL));
-  SDNode *Store =
-      CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
-  ReplaceNode(Node, Store);
-}
-
-void RISCVDAGToDAGISel::selectVSXSEGMask(SDNode *Node, unsigned IntNo) {
-  SDLoc DL(Node);
-  unsigned NF = Node->getNumOperands() - 6;
-  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
-  unsigned ScalarSize = VT.getScalarSizeInBits();
-  MVT XLenVT = Subtarget->getXLenVT();
-  RISCVVLMUL LMUL = getLMUL(VT);
-  SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
-  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
-  SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
-  SDValue Operands[] = {
-      StoreVal,
-      Node->getOperand(2 + NF), // Base pointer.
-      Node->getOperand(3 + NF), // Index.
-      Node->getOperand(4 + NF), // Mask.
-      Node->getOperand(5 + NF), // VL.
-      SEW,
-      Node->getOperand(0) // Chain.
-  };
 
-  MVT IndexVT = Node->getOperand(3 + NF)->getSimpleValueType(0);
   RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
       IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
       static_cast<unsigned>(IndexLMUL));
   SDNode *Store =
       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
   ReplaceNode(Node, Store);
case ISD::INTRINSIC_W_CHAIN: {
     }
     case Intrinsic::riscv_vlseg2:
     case Intrinsic::riscv_vlseg3:
     case Intrinsic::riscv_vlseg4:
     case Intrinsic::riscv_vlseg5:
     case Intrinsic::riscv_vlseg6:
     case Intrinsic::riscv_vlseg7:
     case Intrinsic::riscv_vlseg8: {
-      selectVLSEG(Node, IntNo, /*IsStrided=*/false);
+      selectVLSEG(Node, IntNo, /*IsMasked*/ false, /*IsStrided*/ false);
       return;
     }
     case Intrinsic::riscv_vlseg2_mask:
     case Intrinsic::riscv_vlseg3_mask:
     case Intrinsic::riscv_vlseg4_mask:
     case Intrinsic::riscv_vlseg5_mask:
     case Intrinsic::riscv_vlseg6_mask:
     case Intrinsic::riscv_vlseg7_mask:
     case Intrinsic::riscv_vlseg8_mask: {
-      selectVLSEGMask(Node, IntNo, /*IsStrided=*/false);
+      selectVLSEG(Node, IntNo, /*IsMasked*/ true, /*IsStrided*/ false);
       return;
     }
     case Intrinsic::riscv_vlsseg2:
     case Intrinsic::riscv_vlsseg3:
     case Intrinsic::riscv_vlsseg4:
     case Intrinsic::riscv_vlsseg5:
     case Intrinsic::riscv_vlsseg6:
     case Intrinsic::riscv_vlsseg7:
     case Intrinsic::riscv_vlsseg8: {
-      selectVLSEG(Node, IntNo, /*IsStrided=*/true);
+      selectVLSEG(Node, IntNo, /*IsMasked*/ false, /*IsStrided*/ true);
       return;
     }
     case Intrinsic::riscv_vlsseg2_mask:
     case Intrinsic::riscv_vlsseg3_mask:
     case Intrinsic::riscv_vlsseg4_mask:
     case Intrinsic::riscv_vlsseg5_mask:
     case Intrinsic::riscv_vlsseg6_mask:
     case Intrinsic::riscv_vlsseg7_mask:
     case Intrinsic::riscv_vlsseg8_mask: {
-      selectVLSEGMask(Node, IntNo, /*IsStrided=*/true);
+      selectVLSEG(Node, IntNo, /*IsMasked*/ true, /*IsStrided*/ true);
       return;
     }
     case Intrinsic::riscv_vloxseg2:
     case Intrinsic::riscv_vloxseg3:
     case Intrinsic::riscv_vloxseg4:
     case Intrinsic::riscv_vloxseg5:
     case Intrinsic::riscv_vloxseg6:
     case Intrinsic::riscv_vloxseg7:
     case Intrinsic::riscv_vloxseg8:
     case Intrinsic::riscv_vluxseg2:
     case Intrinsic::riscv_vluxseg3:
     case Intrinsic::riscv_vluxseg4:
     case Intrinsic::riscv_vluxseg5:
     case Intrinsic::riscv_vluxseg6:
     case Intrinsic::riscv_vluxseg7:
     case Intrinsic::riscv_vluxseg8: {
-      selectVLXSEG(Node, IntNo);
+      selectVLXSEG(Node, IntNo, /*IsMasked*/ false);
       return;
     }
     case Intrinsic::riscv_vloxseg2_mask:
     case Intrinsic::riscv_vloxseg3_mask:
     case Intrinsic::riscv_vloxseg4_mask:
     case Intrinsic::riscv_vloxseg5_mask:
     case Intrinsic::riscv_vloxseg6_mask:
     case Intrinsic::riscv_vloxseg7_mask:
     case Intrinsic::riscv_vloxseg8_mask:
     case Intrinsic::riscv_vluxseg2_mask:
     case Intrinsic::riscv_vluxseg3_mask:
     case Intrinsic::riscv_vluxseg4_mask:
     case Intrinsic::riscv_vluxseg5_mask:
     case Intrinsic::riscv_vluxseg6_mask:
     case Intrinsic::riscv_vluxseg7_mask:
     case Intrinsic::riscv_vluxseg8_mask: {
-      selectVLXSEGMask(Node, IntNo);
+      selectVLXSEG(Node, IntNo, /*IsMasked*/ true);
       return;
     }
     case Intrinsic::riscv_vlseg8ff:
     case Intrinsic::riscv_vlseg7ff:
     case Intrinsic::riscv_vlseg6ff:
     case Intrinsic::riscv_vlseg5ff:
     case Intrinsic::riscv_vlseg4ff:
     case Intrinsic::riscv_vlseg3ff:
     case Intrinsic::riscv_vlseg2ff: {
-      selectVLSEGFF(Node);
+      selectVLSEGFF(Node, /*IsMasked*/ false);
       return;
     }
     case Intrinsic::riscv_vlseg8ff_mask:
     case Intrinsic::riscv_vlseg7ff_mask:
     case Intrinsic::riscv_vlseg6ff_mask:
     case Intrinsic::riscv_vlseg5ff_mask:
     case Intrinsic::riscv_vlseg4ff_mask:
     case Intrinsic::riscv_vlseg3ff_mask:
     case Intrinsic::riscv_vlseg2ff_mask: {
-      selectVLSEGFFMask(Node);
+      selectVLSEGFF(Node, /*IsMasked*/ true);
       return;
     }
     }
     break;
   }
   case ISD::INTRINSIC_VOID: {
     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
     switch (IntNo) {
     case Intrinsic::riscv_vsseg2:
     case Intrinsic::riscv_vsseg3:
     case Intrinsic::riscv_vsseg4:
     case Intrinsic::riscv_vsseg5:
     case Intrinsic::riscv_vsseg6:
     case Intrinsic::riscv_vsseg7:
     case Intrinsic::riscv_vsseg8: {
-      selectVSSEG(Node, IntNo, /*IsStrided=*/false);
+      selectVSSEG(Node, IntNo, /*IsMasked*/ false, /*IsStrided*/ false);
       return;
     }
     case Intrinsic::riscv_vsseg2_mask:
     case Intrinsic::riscv_vsseg3_mask:
     case Intrinsic::riscv_vsseg4_mask:
     case Intrinsic::riscv_vsseg5_mask:
     case Intrinsic::riscv_vsseg6_mask:
     case Intrinsic::riscv_vsseg7_mask:
     case Intrinsic::riscv_vsseg8_mask: {
-      selectVSSEGMask(Node, IntNo, /*IsStrided=*/false);
+      selectVSSEG(Node, IntNo, /*IsMasked*/ true, /*IsStrided*/ false);
       return;
     }
     case Intrinsic::riscv_vssseg2:
     case Intrinsic::riscv_vssseg3:
     case Intrinsic::riscv_vssseg4:
     case Intrinsic::riscv_vssseg5:
     case Intrinsic::riscv_vssseg6:
     case Intrinsic::riscv_vssseg7:
     case Intrinsic::riscv_vssseg8: {
-      selectVSSEG(Node, IntNo, /*IsStrided=*/true);
+      selectVSSEG(Node, IntNo, /*IsMasked*/ false, /*IsStrided*/ true);
       return;
     }
     case Intrinsic::riscv_vssseg2_mask:
     case Intrinsic::riscv_vssseg3_mask:
     case Intrinsic::riscv_vssseg4_mask:
     case Intrinsic::riscv_vssseg5_mask:
     case Intrinsic::riscv_vssseg6_mask:
     case Intrinsic::riscv_vssseg7_mask:
     case Intrinsic::riscv_vssseg8_mask: {
-      selectVSSEGMask(Node, IntNo, /*IsStrided=*/true);
+      selectVSSEG(Node, IntNo, /*IsMasked*/ true, /*IsStrided*/ true);
       return;
     }
     case Intrinsic::riscv_vsoxseg2:
     case Intrinsic::riscv_vsoxseg3:
     case Intrinsic::riscv_vsoxseg4:
     case Intrinsic::riscv_vsoxseg5:
     case Intrinsic::riscv_vsoxseg6:
     case Intrinsic::riscv_vsoxseg7:
     case Intrinsic::riscv_vsoxseg8:
     case Intrinsic::riscv_vsuxseg2:
     case Intrinsic::riscv_vsuxseg3:
     case Intrinsic::riscv_vsuxseg4:
     case Intrinsic::riscv_vsuxseg5:
     case Intrinsic::riscv_vsuxseg6:
     case Intrinsic::riscv_vsuxseg7:
     case Intrinsic::riscv_vsuxseg8: {
-      selectVSXSEG(Node, IntNo);
+      selectVSXSEG(Node, IntNo, /*IsMasked*/ false);
       return;
     }
     case Intrinsic::riscv_vsoxseg2_mask:
     case Intrinsic::riscv_vsoxseg3_mask:
     case Intrinsic::riscv_vsoxseg4_mask:
     case Intrinsic::riscv_vsoxseg5_mask:
     case Intrinsic::riscv_vsoxseg6_mask:
     case Intrinsic::riscv_vsoxseg7_mask:
     case Intrinsic::riscv_vsoxseg8_mask:
     case Intrinsic::riscv_vsuxseg2_mask:
     case Intrinsic::riscv_vsuxseg3_mask:
     case Intrinsic::riscv_vsuxseg4_mask:
     case Intrinsic::riscv_vsuxseg5_mask:
     case Intrinsic::riscv_vsuxseg6_mask:
     case Intrinsic::riscv_vsuxseg7_mask:
     case Intrinsic::riscv_vsuxseg8_mask: {
-      selectVSXSEGMask(Node, IntNo);
+      selectVSXSEG(Node, IntNo, /*IsMasked*/ true);
       return;
     }
     }
     break;
   }
   case ISD::INSERT_SUBVECTOR: {
     // Bail when not a "cast" like insert_subvector.
     if (Node->getConstantOperandVal(2) != 0)