[RISCV] Use non-strided load if VL=1 for optimized zero stride loads

[luke]

Summary

When optimizing vmv.s.x/vmv.v.x's of scalar loads, if VL is known to be
1 then we don't need to perform a stride of x0, and can just do a
regular load.

[luke]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2076–2081

Could VM_S_X_VL nodes be lowered to non-strided loads with VL=1? Since only the first lane needs to be written to.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2088–2099

Does this simplify to bool IsStrided = !isOneConstant(VL)?

[craig.topper]

LGTM other than the suggestion of how to simplify IsStrided calculation.

[luke]

Simplify IsStrided

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

case ISD::EXTRACT_SUBVECTOR: {
     SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
     ReplaceNode(Node, Extract.getNode());
     return;
   }
   case RISCVISD::VMV_S_X_VL:
   case RISCVISD::VFMV_S_F_VL:
   case RISCVISD::VMV_V_X_VL:
   case RISCVISD::VFMV_V_F_VL: {
-    // Only if we have optimized zero-stride vector load.
-    if (!Subtarget->hasOptimizedZeroStrideLoad())
-      break;
-
     // Try to match splat of a scalar load to a strided load with stride of x0.
     bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
                         Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
     if (!Node->getOperand(0).isUndef())
       break;
     SDValue Src = Node->getOperand(1);
     auto *Ld = dyn_cast<LoadSDNode>(Src);
     if (!Ld)
       break;
     EVT MemVT = Ld->getMemoryVT();
     // The memory VT should be the same size as the element type.
     if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
       break;
     if (!IsProfitableToFold(Src, Node, Node) ||
         !IsLegalToFold(Src, Node, Node, TM.getOptLevel()))
       break;
 
     SDValue VL;
     if (IsScalarMove) {
       // We could deal with more VL if we update the VSETVLI insert pass to
       // avoid introducing more VSETVLI.
       if (!isOneConstant(Node->getOperand(2)))
         break;
       selectVLOp(Node->getOperand(2), VL);
     } else
       selectVLOp(Node->getOperand(2), VL);
 
     unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
     SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
 
-    SDValue Operands[] = {Ld->getBasePtr(),
-                          CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW,
-                          Ld->getChain()};
+    // If VL=1, then we don't need to do a strided load and can just do a
+    // regular load.
+    bool IsStrided = !isOneConstant(VL);
+
+    // Only do a strided load if we have optimized zero-stride vector load.
+    if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
+      break;
+
+    SmallVector<SDValue> Operands = {Ld->getBasePtr()};
+    if (IsStrided)
+      Operands.push_back(CurDAG->getRegister(RISCV::X0, XLenVT));
+    Operands.append({VL, SEW, Ld->getChain()});
 
     RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
     const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
-        /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false,
+        /*IsMasked*/ false, /*IsTU*/ false, IsStrided, /*FF*/ false,
         Log2SEW, static_cast<unsigned>(LMUL));
     MachineSDNode *Load =
         CurDAG->getMachineNode(P->Pseudo, DL, {VT, MVT::Other}, Operands);
     // Update the chain.
     ReplaceUses(Src.getValue(1), SDValue(Load, 1));
     // Record the mem-refs
     CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()});
     // Replace the splat with the vlse.

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll

 ; RV32-LMULMAX4-NEXT:    vslideup.vi v0, v8, 1
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_v64i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI19_0)
 ; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI19_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX4-NEXT:    vle64.v v0, (a0)
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_v64i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, 748388
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
 ; RV32-LMULMAX8-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v8, a0
 ; RV32-LMULMAX8-NEXT:    lui a0, 748384
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, 1776
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v0, a0
 ; RV32-LMULMAX8-NEXT:    vslideup.vi v0, v8, 1
 ; RV32-LMULMAX8-NEXT:    ret
 ;
 ; RV64-LMULMAX8-LABEL: buildvec_mask_v64i1:
 ; RV64-LMULMAX8:       # %bb.0:
 ; RV64-LMULMAX8-NEXT:    lui a0, %hi(.LCPI19_0)
 ; RV64-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI19_0)
 ; RV64-LMULMAX8-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-LMULMAX8-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX8-NEXT:    vle64.v v0, (a0)
 ; RV64-LMULMAX8-NEXT:    ret
   ret <64 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1>
 }
 
 define <128 x i1> @buildvec_mask_v128i1() {
 ; RV32-LMULMAX1-LABEL: buildvec_mask_v128i1:
 ; RV32-LMULMAX1:       # %bb.0:
 ; RV32-LMULMAX1-NEXT:    li a0, 1776

 ; RV32-LMULMAX4-NEXT:    vslideup.vi v8, v9, 1
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_v128i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI20_0)
 ; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI20_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX4-NEXT:    vle64.v v0, (a0)
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI20_1)
 ; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI20_1)
-; RV64-LMULMAX4-NEXT:    vlse64.v v8, (a0), zero
+; RV64-LMULMAX4-NEXT:    vle64.v v8, (a0)
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_v128i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, 748388
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
 ; RV32-LMULMAX8-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v8, a0

 ; RV32-LMULMAX4-NEXT:    vlm.v v8, (a0)
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_optsize_v128i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_0)
 ; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX4-NEXT:    vle64.v v0, (a0)
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_1)
 ; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_1)
-; RV64-LMULMAX4-NEXT:    vlse64.v v8, (a0), zero
+; RV64-LMULMAX4-NEXT:    vle64.v v8, (a0)
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_optsize_v128i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, %hi(.LCPI21_0)
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI21_0)
 ; RV32-LMULMAX8-NEXT:    li a1, 128
 ; RV32-LMULMAX8-NEXT:    vsetvli zero, a1, e8, m8, ta, ma

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll

 ;
 ; RV64ZVE32F-LABEL: mgather_v1i8:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
 ; RV64ZVE32F-NEXT:    bnez a1, .LBB0_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vlse8.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vle8.v v8, (a0)
 ; RV64ZVE32F-NEXT:  .LBB0_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x i8> @llvm.masked.gather.v1i8.v1p0(<1 x ptr> %ptrs, i32 1, <1 x i1> %m, <1 x i8> %passthru)
   ret <1 x i8> %v
 }
 
 declare <2 x i8> @llvm.masked.gather.v2i8.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i8>)
 

 ;
 ; RV64ZVE32F-LABEL: mgather_v1i16:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
 ; RV64ZVE32F-NEXT:    bnez a1, .LBB13_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vle16.v v8, (a0)
 ; RV64ZVE32F-NEXT:  .LBB13_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x i16> @llvm.masked.gather.v1i16.v1p0(<1 x ptr> %ptrs, i32 2, <1 x i1> %m, <1 x i16> %passthru)
   ret <1 x i16> %v
 }
 
 declare <2 x i16> @llvm.masked.gather.v2i16.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i16>)
 

 ;
 ; RV64ZVE32F-LABEL: mgather_v1i32:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
 ; RV64ZVE32F-NEXT:    bnez a1, .LBB27_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vlse32.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vle32.v v8, (a0)
 ; RV64ZVE32F-NEXT:  .LBB27_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x i32> @llvm.masked.gather.v1i32.v1p0(<1 x ptr> %ptrs, i32 4, <1 x i1> %m, <1 x i32> %passthru)
   ret <1 x i32> %v
 }
 
 declare <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i32>)
 

 ;
 ; RV64ZVE32F-LABEL: mgather_v1f16:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
 ; RV64ZVE32F-NEXT:    bnez a1, .LBB58_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vle16.v v8, (a0)
 ; RV64ZVE32F-NEXT:  .LBB58_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x half> @llvm.masked.gather.v1f16.v1p0(<1 x ptr> %ptrs, i32 2, <1 x i1> %m, <1 x half> %passthru)
   ret <1 x half> %v
 }
 
 declare <2 x half> @llvm.masked.gather.v2f16.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x half>)
 

 ;
 ; RV64ZVE32F-LABEL: mgather_v1f32:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
 ; RV64ZVE32F-NEXT:    bnez a1, .LBB68_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vlse32.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vle32.v v8, (a0)
 ; RV64ZVE32F-NEXT:  .LBB68_2: # %else
 ; RV64ZVE32F-NEXT:    ret
   %v = call <1 x float> @llvm.masked.gather.v1f32.v1p0(<1 x ptr> %ptrs, i32 4, <1 x i1> %m, <1 x float> %passthru)
   ret <1 x float> %v
 }
 
 declare <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x float>)