[RISCV] Preserve fixed-length VL on insert_vector_elt in more cases

[frasercrmck]

Summary

This patch fixes up one case where the fixed-length-vector VL was
dropped (falling back to VLMAX) when inserting vector elements, as the
code would lower via ISD::INSERT_VECTOR_ELT (at index 0) which loses the
fixed-length vector information.

To this end, a custom node, VMV_S_XF_VL, was introduced to carry the VL
operand through to the final instruction. This node wraps the RVV
vmv.s.x and vmv.s.f instructions, which were being selected by
insert_vector_elt anyway.

There should be no observable difference in scalable-vector codegen.

There is still one outstanding drop from fixed-length VL to VLMAX, when
an i64 element is inserted into a vector on RV32; the splat (which is
custom legalized) has no notion of the original fixed-length vector
type.

[frasercrmck]

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll

13

Still using VLMAX here, because we're splatting an i64 using the vector container type. I suppose we either need a custom node or must emit something in the fixed-length type and converting back to the scalable type. I tried that with build_vector and splat_vector but neither worked. build_vector expanded itself, and splat_vector doesn't work on fixed-length vectors (and I'm not sure we want to).

Do you have any ideas, @craig.topper?

[craig.topper]

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll

13

Nothing great beyond replicating the sequence in lowerSPLATVECTOR using RISCVISD::*_VL nodes.

I'm looking at type legalizing the vmv_v_x intrinsic for i64 on RV32 which has the same problem.

[craig.topper]

LGTM

[frasercrmck]

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll

13

Yeah, perhaps that's just what we'll have to do. As a free function to emit the sequence in place? Because I believe build_vector automatically legalizes itself in an unhelpful way, and splat_vector could potentially open up a can of worms?

llvm/lib/Target/RISCV/RISCVISelLowering.h

enum NodeType : unsigned {
   // for the VL value to be used for the operation.
   VMV_V_X_VL,
   // VFMV_V_F_VL matches the semantics of vfmv.v.f but includes an extra operand
   // for the VL value to be used for the operation.
   VFMV_V_F_VL,
   // VMV_X_S matches the semantics of vmv.x.s. The result is always XLenVT sign
   // extended from the vector element size.
   VMV_X_S,
+  // VMV_S_XF_VL matches the semantics of vmv.s.x/vmv.s.f, depending on the
+  // types of its operands. It carries a VL operand.
+  VMV_S_XF_VL,
   // Splats an i64 scalar to a vector type (with element type i64) where the
   // scalar is a sign-extended i32.
   SPLAT_VECTOR_I64,
   // Read VLENB CSR
   READ_VLENB,
   // Truncates a RVV integer vector by one power-of-two. Carries both an extra
   // mask and VL operand.
   TRUNCATE_VECTOR_VL,

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
   // Custom-legalize INSERT_VECTOR_ELT where XLEN>=SEW, so that the vector is
   // first slid down into position, the value is inserted into the first
   // position, and the vector is slid back up. We do this to simplify patterns.
   //   (slideup vec, (insertelt (slidedown impdef, vec, idx), val, 0), idx),
   if (Subtarget.is64Bit() || Val.getValueType() != MVT::i64) {
     if (isNullConstant(Idx))
-      return Op;
+      return DAG.getNode(RISCVISD::VMV_S_XF_VL, DL, ContainerVT, Vec, Val, VL);
     SDValue Slidedown =
         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
                     DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
     SDValue InsertElt0 =
-        DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT, Slidedown, Val,
-                    DAG.getConstant(0, DL, Subtarget.getXLenVT()));
-
+        DAG.getNode(RISCVISD::VMV_S_XF_VL, DL, ContainerVT, Slidedown, Val, VL);
     return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec, InsertElt0,
                        Idx, Mask, VL);
   }
 
   // Custom-legalize INSERT_VECTOR_ELT where XLEN<SEW, as the SEW element type
   // is illegal (currently only vXi64 RV32).
   // Since there is no easy way of getting a single element into a vector when
   // XLEN<SEW, we lower the operation to the following sequence:
#define NODE_NAME_CASE(NODE)                                                   \
   NODE_NAME_CASE(GREVI)
   NODE_NAME_CASE(GREVIW)
   NODE_NAME_CASE(GORCI)
   NODE_NAME_CASE(GORCIW)
   NODE_NAME_CASE(SHFLI)
   NODE_NAME_CASE(VMV_V_X_VL)
   NODE_NAME_CASE(VFMV_V_F_VL)
   NODE_NAME_CASE(VMV_X_S)
+  NODE_NAME_CASE(VMV_S_XF_VL)
   NODE_NAME_CASE(SPLAT_VECTOR_I64)
   NODE_NAME_CASE(READ_VLENB)
   NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
   NODE_NAME_CASE(VLEFF)
   NODE_NAME_CASE(VLEFF_MASK)
   NODE_NAME_CASE(VSLIDEUP_VL)
   NODE_NAME_CASE(VSLIDEDOWN_VL)
   NODE_NAME_CASE(VID_VL)

llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td

foreach fvti = AllFloatVectors in {
 
   def : Pat<(fvti.Vector (splat_vector (fvti.Scalar fpimm0))),
             (!cast<Instruction>("PseudoVMV_V_I_"#fvti.LMul.MX)
               0, fvti.AVL, fvti.SEW)>;
 }
 } // Predicates = [HasStdExtV, HasStdExtF]
 
 //===----------------------------------------------------------------------===//
-// Vector Element Inserts/Extracts
+// Vector Element Extracts
 //===----------------------------------------------------------------------===//
-
-// The built-in TableGen 'insertelt' node must return the same type as the
-// vector element type. On RISC-V, XLenVT is the only legal integer type, so
-// for integer inserts we use a custom node which inserts an XLenVT-typed
-// value.
-def riscv_insert_vector_elt
-    : SDNode<"ISD::INSERT_VECTOR_ELT",
-             SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisVT<2, XLenVT>,
-                                  SDTCisPtrTy<3>]>, []>;
-
-let Predicates = [HasStdExtV] in
-foreach vti = AllIntegerVectors in {
-  def : Pat<(vti.Vector (riscv_insert_vector_elt (vti.Vector vti.RegClass:$merge),
-                                                 vti.ScalarRegClass:$rs1, 0)),
-            (!cast<Instruction>("PseudoVMV_S_X_"#vti.LMul.MX)
-                vti.RegClass:$merge,
-                (vti.Scalar vti.ScalarRegClass:$rs1),
-                vti.AVL, vti.SEW)>;
-}
-
 let Predicates = [HasStdExtV, HasStdExtF] in
 foreach vti = AllFloatVectors in {
-  defvar MX = vti.LMul.MX;
   defvar vmv_f_s_inst = !cast<Instruction>(!strconcat("PseudoVFMV_",
                                                        vti.ScalarSuffix,
-                                                       "_S_", MX));
-  defvar vmv_s_f_inst = !cast<Instruction>(!strconcat("PseudoVFMV_S_",
-                                                       vti.ScalarSuffix,
-                                                       "_", vti.LMul.MX));
-  // Only pattern-match insert/extract-element operations where the index is
-  // 0. Any other index will have been custom-lowered to slide the vector
-  // correctly into place (and, in the case of insert, slide it back again
-  // afterwards).
+                                                       "_S_", vti.LMul.MX));
+  // Only pattern-match extract-element operations where the index is 0. Any
+  // other index will have been custom-lowered to slide the vector correctly
+  // into place.
   def : Pat<(vti.Scalar (extractelt (vti.Vector vti.RegClass:$rs2), 0)),
             (vmv_f_s_inst vti.RegClass:$rs2, vti.SEW)>;
-
-  def : Pat<(vti.Vector (insertelt (vti.Vector vti.RegClass:$merge),
-                                   vti.ScalarRegClass:$rs1, 0)),
-            (vmv_s_f_inst vti.RegClass:$merge,
-                    (vti.Scalar vti.ScalarRegClass:$rs1),
-                    vti.AVL, vti.SEW)>;
 }

llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td

 def riscv_vmv_v_x_vl : SDNode<"RISCVISD::VMV_V_X_VL",
                               SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<0>,
                                                    SDTCisVT<1, XLenVT>,
                                                    SDTCisVT<2, XLenVT>]>>;
 def riscv_vfmv_v_f_vl : SDNode<"RISCVISD::VFMV_V_F_VL",
                                SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisFP<0>,
                                                     SDTCisEltOfVec<1, 0>,
                                                     SDTCisVT<2, XLenVT>]>>;
+def riscv_vmv_s_xf_vl : SDNode<"RISCVISD::VMV_S_XF_VL",
+                               SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>,
+                                                    SDTCisVT<3, XLenVT>]>>;
 
 def riscv_vle_vl : SDNode<"RISCVISD::VLE_VL", SDT_RISCVVLE_VL,
                           [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 def riscv_vse_vl : SDNode<"RISCVISD::VSE_VL", SDT_RISCVVSE_VL,
                           [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 
 def riscv_add_vl   : SDNode<"RISCVISD::ADD_VL",   SDT_RISCVIntBinOp_VL, [SDNPCommutative]>;
 def riscv_sub_vl   : SDNode<"RISCVISD::SUB_VL",   SDT_RISCVIntBinOp_VL>;
foreach mti = AllMasks in {
   // Match the not idiom to the vnot.mm pseudo.
   def : Pat<(mti.Mask (riscv_vmnot_vl VR:$rs, (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVMNAND_MM_" # mti.LMul.MX)
                  VR:$rs, VR:$rs, GPR:$vl, mti.SEW)>;
 }
 
 } // Predicates = [HasStdExtV]
 
-// 17.4. Vector Register GAther Instruction
 let Predicates = [HasStdExtV] in {
-
+// 17.1. Integer Scalar Move Instructions
+// 17.4. Vector Register Gather Instruction
 foreach vti = AllIntegerVectors in {
+  def : Pat<(vti.Vector (riscv_vmv_s_xf_vl (vti.Vector vti.RegClass:$merge),
+                                           (XLenVT vti.ScalarRegClass:$rs1),
+                                           (XLenVT (VLOp GPR:$vl)))),
+            (!cast<Instruction>("PseudoVMV_S_X_"#vti.LMul.MX)
+                vti.RegClass:$merge,
+                (vti.Scalar vti.ScalarRegClass:$rs1), GPR:$vl, vti.SEW)>;
   def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1,
                                               (vti.Mask true_mask),
                                               (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVRGATHER_VX_"# vti.LMul.MX)
                  vti.RegClass:$rs2, GPR:$rs1, GPR:$vl, vti.SEW)>;
   def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, uimm5:$imm,
                                               (vti.Mask true_mask),
                                               (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVRGATHER_VI_"# vti.LMul.MX)
                  vti.RegClass:$rs2, uimm5:$imm, GPR:$vl, vti.SEW)>;
 }
 
 } // Predicates = [HasStdExtV]
 
 let Predicates = [HasStdExtV, HasStdExtF] in {
 
+// 17.2. Floating-Point Scalar Move Instructions
 foreach vti = AllFloatVectors in {
+  def : Pat<(vti.Vector (riscv_vmv_s_xf_vl (vti.Vector vti.RegClass:$merge),
+                                           vti.ScalarRegClass:$rs1,
+                                           (XLenVT (VLOp GPR:$vl)))),
+            (!cast<Instruction>("PseudoVFMV_S_"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                vti.RegClass:$merge,
+                (vti.Scalar vti.ScalarRegClass:$rs1), GPR:$vl, vti.SEW)>;
   def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1,
                                               (vti.Mask true_mask),
                                               (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVRGATHER_VX_"# vti.LMul.MX)
                  vti.RegClass:$rs2, GPR:$rs1, GPR:$vl, vti.SEW)>;
   def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, uimm5:$imm,
                                               (vti.Mask true_mask),
                                               (XLenVT (VLOp GPR:$vl)))),

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+experimental-v,+experimental-zfh,+f,+d -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV32
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+experimental-v,+experimental-zfh,+f,+d -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=RV64
 
 ; FIXME: This codegen needs to be improved. These tests previously asserted
 ; type legalizing the i64 type on RV32.
 
 define void @insertelt_v4i64(<4 x i64>* %x, i64 %y) {
 ; RV32-LABEL: insertelt_v4i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
 ; RV32-NEXT:    vle64.v v26, (a0)
 ; RV32-NEXT:    vsetvli a3, zero, e64,m2,ta,mu
 ; RV32-NEXT:    vmv.v.x v28, a2
 ; RV32-NEXT:    addi a2, zero, 32
 ; RV32-NEXT:    vsll.vx v28, v28, a2
 ; RV32-NEXT:    vmv.v.x v30, a1
 ; RV32-NEXT:    vsll.vx v30, v30, a2
 ; RV32-NEXT:    vsrl.vx v30, v30, a2
 ; RV32-NEXT:    vor.vv v28, v30, v28
 ; RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; RV32-NEXT:    vid.v v30
 ; RV32-NEXT:    vmseq.vi v0, v30, 3
 ; RV32-NEXT:    vmerge.vvm v26, v26, v28, v0
 ; RV32-NEXT:    vse64.v v26, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v4i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli a2, 4, e64,m2,ta,mu
 ; RV64-NEXT:    vle64.v v26, (a0)
 ; RV64-NEXT:    vslidedown.vi v28, v26, 3
-; RV64-NEXT:    vsetvli a2, zero, e64,m2,ta,mu
 ; RV64-NEXT:    vmv.s.x v28, a1
 ; RV64-NEXT:    vsetivli a1, 4, e64,m2,tu,mu
 ; RV64-NEXT:    vslideup.vi v26, v28, 3
 ; RV64-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; RV64-NEXT:    vse64.v v26, (a0)
 ; RV64-NEXT:    ret
   %a = load <4 x i64>, <4 x i64>* %x
   %b = insertelement <4 x i64> %a, i64 %y, i32 3

 }
 
 define void @insertelt_v16i8(<16 x i8>* %x, i8 %y) {
 ; RV32-LABEL: insertelt_v16i8:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
 ; RV32-NEXT:    vle8.v v25, (a0)
 ; RV32-NEXT:    vslidedown.vi v26, v25, 14
-; RV32-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; RV32-NEXT:    vmv.s.x v26, a1
 ; RV32-NEXT:    vsetivli a1, 16, e8,m1,tu,mu
 ; RV32-NEXT:    vslideup.vi v25, v26, 14
 ; RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
 ; RV32-NEXT:    vse8.v v25, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v16i8:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
 ; RV64-NEXT:    vle8.v v25, (a0)
 ; RV64-NEXT:    vslidedown.vi v26, v25, 14
-; RV64-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; RV64-NEXT:    vmv.s.x v26, a1
 ; RV64-NEXT:    vsetivli a1, 16, e8,m1,tu,mu
 ; RV64-NEXT:    vslideup.vi v25, v26, 14
 ; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
 ; RV64-NEXT:    vse8.v v25, (a0)
 ; RV64-NEXT:    ret
   %a = load <16 x i8>, <16 x i8>* %x
   %b = insertelement <16 x i8> %a, i8 %y, i32 14
   store <16 x i8> %b, <16 x i8>* %x
   ret void
 }
 
 define void @insertelt_v32i16(<32 x i16>* %x, i16 %y, i32 %idx) {
 ; RV32-LABEL: insertelt_v32i16:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi a3, zero, 32
 ; RV32-NEXT:    vsetvli a4, a3, e16,m4,ta,mu
 ; RV32-NEXT:    vle16.v v28, (a0)
 ; RV32-NEXT:    vslidedown.vx v8, v28, a2
-; RV32-NEXT:    vsetvli a4, zero, e16,m4,ta,mu
 ; RV32-NEXT:    vmv.s.x v8, a1
 ; RV32-NEXT:    vsetvli a1, a3, e16,m4,tu,mu
 ; RV32-NEXT:    vslideup.vx v28, v8, a2
 ; RV32-NEXT:    vsetvli a1, a3, e16,m4,ta,mu
 ; RV32-NEXT:    vse16.v v28, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v32i16:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi a3, zero, 32
 ; RV64-NEXT:    vsetvli a4, a3, e16,m4,ta,mu
 ; RV64-NEXT:    vle16.v v28, (a0)
 ; RV64-NEXT:    sext.w a2, a2
 ; RV64-NEXT:    vslidedown.vx v8, v28, a2
-; RV64-NEXT:    vsetvli a4, zero, e16,m4,ta,mu
 ; RV64-NEXT:    vmv.s.x v8, a1
 ; RV64-NEXT:    vsetvli a1, a3, e16,m4,tu,mu
 ; RV64-NEXT:    vslideup.vx v28, v8, a2
 ; RV64-NEXT:    vsetvli a1, a3, e16,m4,ta,mu
 ; RV64-NEXT:    vse16.v v28, (a0)
 ; RV64-NEXT:    ret
   %a = load <32 x i16>, <32 x i16>* %x
   %b = insertelement <32 x i16> %a, i16 %y, i32 %idx
   store <32 x i16> %b, <32 x i16>* %x
   ret void
 }
 
 define void @insertelt_v8f32(<8 x float>* %x, float %y, i32 %idx) {
 ; RV32-LABEL: insertelt_v8f32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
 ; RV32-NEXT:    vle32.v v26, (a0)
 ; RV32-NEXT:    vslidedown.vx v28, v26, a1
-; RV32-NEXT:    vsetvli a2, zero, e32,m2,ta,mu
 ; RV32-NEXT:    vfmv.s.f v28, fa0
 ; RV32-NEXT:    vsetivli a2, 8, e32,m2,tu,mu
 ; RV32-NEXT:    vslideup.vx v26, v28, a1
 ; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
 ; RV32-NEXT:    vse32.v v26, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v8f32:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
 ; RV64-NEXT:    vle32.v v26, (a0)
 ; RV64-NEXT:    sext.w a1, a1
 ; RV64-NEXT:    vslidedown.vx v28, v26, a1
-; RV64-NEXT:    vsetvli a2, zero, e32,m2,ta,mu
 ; RV64-NEXT:    vfmv.s.f v28, fa0
 ; RV64-NEXT:    vsetivli a2, 8, e32,m2,tu,mu
 ; RV64-NEXT:    vslideup.vx v26, v28, a1
 ; RV64-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
 ; RV64-NEXT:    vse32.v v26, (a0)
 ; RV64-NEXT:    ret
   %a = load <8 x float>, <8 x float>* %x
   %b = insertelement <8 x float> %a, float %y, i32 %idx
   store <8 x float> %b, <8 x float>* %x
   ret void
 }