[RISCV] Lower unmasked zero-stride vector load to (scalar load + splat)

[• pcwang-thead]

Summary

So we have the opportunity to fold splat into .vx instruction as what
D101138 has done. If failed, we can select zero-stride vector load
again.

[• pcwang-thead]

Fix test.

[• pcwang-thead]

Update tests.

[reames]

llvm/test/CodeGen/RISCV/rvv/fixed-vector-strided-load-store-asm.ll

273–274

This looks like a regression.

[• pcwang-thead]

llvm/test/CodeGen/RISCV/rvv/fixed-vector-strided-load-store-asm.ll

273–274

The diff result seems weird, but I think it's what we expected.
I added a new test gather_zero_stride_unfold (copied from gather_zero_stride) in which I changed %4 = add <32 x i8> %wide.load, %wide.masked.gather to %4 = udiv <32 x i8> %wide.masked.gather, %wide.load so that splat can't be folded since division is not commutative. So here are scalar load(lbu a5, 0(a1)) and vector splat(vmv.v.x v9, a5).

[reames]

LGTM, but please land the new test and the opcode change in a separate commit (using old llc), then rebase and land this over it. Will make the test diff less confusing in the change log.

llvm/test/CodeGen/RISCV/rvv/fixed-vector-strided-load-store-asm.ll

273–274

I got confused here by the diff. I was responding as if this was the optimized check, which it isn't. So ignore me here.

[• pcwang-thead]

Rebase on separate test.

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

case Intrinsic::riscv_masked_strided_load: {
     auto *Load = cast<MemIntrinsicSDNode>(Op);
     SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
     SDValue Ptr = Op.getOperand(3);
     SDValue Stride = Op.getOperand(4);
     SDValue Result, Chain;
 
     // TODO: We restrict this to unmasked loads currently in consideration of
     // the complexity of hanlding all falses masks.
-    if (IsUnmasked && isNullConstant(Stride) &&
-        !Subtarget.hasOptimizedZeroStrideLoad()) {
+    if (IsUnmasked && isNullConstant(Stride)) {
       MVT ScalarVT = ContainerVT.getVectorElementType();
       SDValue ScalarLoad =
           DAG.getExtLoad(ISD::ZEXTLOAD, DL, XLenVT, Load->getChain(), Ptr,
                          ScalarVT, Load->getMemOperand());
       Chain = ScalarLoad.getValue(1);
       Result = lowerScalarSplat(SDValue(), ScalarLoad, VL, ContainerVT, DL, DAG,
                                 Subtarget);
     } else {

llvm/test/CodeGen/RISCV/rvv/fixed-vector-strided-load-store-asm.ll

vector.body:                                      ; preds = %vector.body, %entry
   br i1 %6, label %for.cond.cleanup, label %vector.body
 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @gather_zero_stride(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
 ;
-; V-LABEL: gather_zero_stride:
+; CHECK-LABEL: gather_zero_stride:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li a2, 0
+; CHECK-NEXT:    li a3, 32
+; CHECK-NEXT:    li a4, 1024
+; CHECK-NEXT:  .LBB3_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    lbu a5, 0(a1)
+; CHECK-NEXT:    add a6, a0, a2
+; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a6)
+; CHECK-NEXT:    vadd.vx v8, v8, a5
+; CHECK-NEXT:    vse8.v v8, (a6)
+; CHECK-NEXT:    addi a2, a2, 32
+; CHECK-NEXT:    addi a1, a1, 160
+; CHECK-NEXT:    bne a2, a4, .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    ret
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.ind = phi <32 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]
+  %0 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
+  %1 = getelementptr inbounds i8, i8* %B, <32 x i64> %0
+  %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
+  %2 = getelementptr inbounds i8, i8* %A, i64 %index
+  %3 = bitcast i8* %2 to <32 x i8>*
+  %wide.load = load <32 x i8>, <32 x i8>* %3, align 1
+  %4 = add <32 x i8> %wide.load, %wide.masked.gather
+  %5 = bitcast i8* %2 to <32 x i8>*
+  store <32 x i8> %4, <32 x i8>* %5, align 1
+  %index.next = add nuw i64 %index, 32
+  %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+  %6 = icmp eq i64 %index.next, 1024
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
+
+define void @gather_zero_stride_unfold(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
+; V-LABEL: gather_zero_stride_unfold:
 ; V:       # %bb.0: # %entry
 ; V-NEXT:    li a2, 0
 ; V-NEXT:    li a3, 32
 ; V-NEXT:    li a4, 1024
-; V-NEXT:  .LBB3_1: # %vector.body
+; V-NEXT:  .LBB5_1: # %vector.body
 ; V-NEXT:    # =>This Inner Loop Header: Depth=1
 ; V-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; V-NEXT:    vlse8.v v8, (a1), zero
 ; V-NEXT:    add a5, a0, a2
 ; V-NEXT:    vle8.v v9, (a5)
-; V-NEXT:    vadd.vv v8, v9, v8
+; V-NEXT:    vdivu.vv v8, v8, v9
 ; V-NEXT:    vse8.v v8, (a5)
 ; V-NEXT:    addi a2, a2, 32
 ; V-NEXT:    addi a1, a1, 160
-; V-NEXT:    bne a2, a4, .LBB3_1
+; V-NEXT:    bne a2, a4, .LBB5_1
 ; V-NEXT:  # %bb.2: # %for.cond.cleanup
 ; V-NEXT:    ret
 ;
-; ZVE32F-LABEL: gather_zero_stride:
+; ZVE32F-LABEL: gather_zero_stride_unfold:
 ; ZVE32F:       # %bb.0: # %entry
 ; ZVE32F-NEXT:    li a2, 0
 ; ZVE32F-NEXT:    li a3, 32
 ; ZVE32F-NEXT:    li a4, 1024
-; ZVE32F-NEXT:  .LBB3_1: # %vector.body
+; ZVE32F-NEXT:  .LBB5_1: # %vector.body
 ; ZVE32F-NEXT:    # =>This Inner Loop Header: Depth=1
 ; ZVE32F-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; ZVE32F-NEXT:    vlse8.v v8, (a1), zero
 ; ZVE32F-NEXT:    add a5, a0, a2
 ; ZVE32F-NEXT:    vle8.v v9, (a5)
-; ZVE32F-NEXT:    vadd.vv v8, v9, v8
+; ZVE32F-NEXT:    vdivu.vv v8, v8, v9
 ; ZVE32F-NEXT:    vse8.v v8, (a5)
 ; ZVE32F-NEXT:    addi a2, a2, 32
 ; ZVE32F-NEXT:    addi a1, a1, 160
-; ZVE32F-NEXT:    bne a2, a4, .LBB3_1
+; ZVE32F-NEXT:    bne a2, a4, .LBB5_1
 ; ZVE32F-NEXT:  # %bb.2: # %for.cond.cleanup
 ; ZVE32F-NEXT:    ret
 ;
-; NOT-OPTIMIZED-LABEL: gather_zero_stride:
+; NOT-OPTIMIZED-LABEL: gather_zero_stride_unfold:
 ; NOT-OPTIMIZED:       # %bb.0: # %entry
 ; NOT-OPTIMIZED-NEXT:    li a2, 0
 ; NOT-OPTIMIZED-NEXT:    li a3, 32
 ; NOT-OPTIMIZED-NEXT:    li a4, 1024
-; NOT-OPTIMIZED-NEXT:  .LBB3_1: # %vector.body
+; NOT-OPTIMIZED-NEXT:  .LBB5_1: # %vector.body
 ; NOT-OPTIMIZED-NEXT:    # =>This Inner Loop Header: Depth=1
 ; NOT-OPTIMIZED-NEXT:    lbu a5, 0(a1)
-; NOT-OPTIMIZED-NEXT:    add a6, a0, a2
 ; NOT-OPTIMIZED-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
+; NOT-OPTIMIZED-NEXT:    add a6, a0, a2
 ; NOT-OPTIMIZED-NEXT:    vle8.v v8, (a6)
-; NOT-OPTIMIZED-NEXT:    vadd.vx v8, v8, a5
+; NOT-OPTIMIZED-NEXT:    vmv.v.x v9, a5
+; NOT-OPTIMIZED-NEXT:    vdivu.vv v8, v9, v8
 ; NOT-OPTIMIZED-NEXT:    vse8.v v8, (a6)
 ; NOT-OPTIMIZED-NEXT:    addi a2, a2, 32
 ; NOT-OPTIMIZED-NEXT:    addi a1, a1, 160
-; NOT-OPTIMIZED-NEXT:    bne a2, a4, .LBB3_1
+; NOT-OPTIMIZED-NEXT:    bne a2, a4, .LBB5_1
 ; NOT-OPTIMIZED-NEXT:  # %bb.2: # %for.cond.cleanup
 ; NOT-OPTIMIZED-NEXT:    ret
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
   %vec.ind = phi <32 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]
   %0 = mul nuw nsw <32 x i64> %vec.ind, <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>
   %1 = getelementptr inbounds i8, i8* %B, <32 x i64> %0
   %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
   %2 = getelementptr inbounds i8, i8* %A, i64 %index
   %3 = bitcast i8* %2 to <32 x i8>*
   %wide.load = load <32 x i8>, <32 x i8>* %3, align 1
-  %4 = add <32 x i8> %wide.load, %wide.masked.gather
+  %4 = udiv <32 x i8>  %wide.masked.gather, %wide.load
   %5 = bitcast i8* %2 to <32 x i8>*
   store <32 x i8> %4, <32 x i8>* %5, align 1
   %index.next = add nuw i64 %index, 32
   %vec.ind.next = add <32 x i64> %vec.ind, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
   %6 = icmp eq i64 %index.next, 1024
   br i1 %6, label %for.cond.cleanup, label %vector.body
 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 ;void scatter(signed char * __restrict  A, signed char * __restrict B) {
 ;  for (int i = 0; i < 1024; ++i)
 ;      A[i * 5] += B[i];
 ;}
 define void @scatter(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
 ;
 ; CHECK-LABEL: scatter:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    li a2, 0
 ; CHECK-NEXT:    li a3, 32
 ; CHECK-NEXT:    li a4, 5
 ; CHECK-NEXT:    li a5, 1024
-; CHECK-NEXT:  .LBB4_1: # %vector.body
+; CHECK-NEXT:  .LBB6_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add a6, a1, a2
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a6)
 ; CHECK-NEXT:    vlse8.v v9, (a0), a4
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vsse8.v v8, (a0), a4
 ; CHECK-NEXT:    addi a2, a2, 32
 ; CHECK-NEXT:    addi a0, a0, 160
-; CHECK-NEXT:    bne a2, a5, .LBB4_1
+; CHECK-NEXT:    bne a2, a5, .LBB6_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
   %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]

 ; V-NEXT:    li a2, 0
 ; V-NEXT:    li a3, 32
 ; V-NEXT:    lui a4, 983765
 ; V-NEXT:    addiw a4, a4, 873
 ; V-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; V-NEXT:    vmv.s.x v0, a4
 ; V-NEXT:    li a4, 5
 ; V-NEXT:    li a5, 1024
-; V-NEXT:  .LBB5_1: # %vector.body
+; V-NEXT:  .LBB7_1: # %vector.body
 ; V-NEXT:    # =>This Inner Loop Header: Depth=1
 ; V-NEXT:    add a6, a1, a2
 ; V-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
 ; V-NEXT:    vle8.v v9, (a6)
 ; V-NEXT:    vmv1r.v v10, v8
 ; V-NEXT:    vlse8.v v10, (a0), a4, v0.t
 ; V-NEXT:    vadd.vv v9, v10, v9
 ; V-NEXT:    vsse8.v v9, (a0), a4, v0.t
 ; V-NEXT:    addi a2, a2, 32
 ; V-NEXT:    addi a0, a0, 160
-; V-NEXT:    bne a2, a5, .LBB5_1
+; V-NEXT:    bne a2, a5, .LBB7_1
 ; V-NEXT:  # %bb.2: # %for.cond.cleanup
 ; V-NEXT:    ret
 ;
 ; ZVE32F-LABEL: scatter_masked:
 ; ZVE32F:       # %bb.0: # %entry
 ; ZVE32F-NEXT:    li a2, 0
 ; ZVE32F-NEXT:    li a3, 32
 ; ZVE32F-NEXT:    lui a4, 983765
 ; ZVE32F-NEXT:    addiw a4, a4, 873
 ; ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; ZVE32F-NEXT:    vmv.s.x v0, a4
 ; ZVE32F-NEXT:    li a4, 5
 ; ZVE32F-NEXT:    li a5, 1024
-; ZVE32F-NEXT:  .LBB5_1: # %vector.body
+; ZVE32F-NEXT:  .LBB7_1: # %vector.body
 ; ZVE32F-NEXT:    # =>This Inner Loop Header: Depth=1
 ; ZVE32F-NEXT:    add a6, a1, a2
 ; ZVE32F-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
 ; ZVE32F-NEXT:    vle8.v v9, (a6)
 ; ZVE32F-NEXT:    vmv1r.v v10, v8
 ; ZVE32F-NEXT:    vlse8.v v10, (a0), a4, v0.t
 ; ZVE32F-NEXT:    vadd.vv v9, v10, v9
 ; ZVE32F-NEXT:    vsse8.v v9, (a0), a4, v0.t
 ; ZVE32F-NEXT:    addi a2, a2, 32
 ; ZVE32F-NEXT:    addi a0, a0, 160
-; ZVE32F-NEXT:    bne a2, a5, .LBB5_1
+; ZVE32F-NEXT:    bne a2, a5, .LBB7_1
 ; ZVE32F-NEXT:  # %bb.2: # %for.cond.cleanup
 ; ZVE32F-NEXT:    ret
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
   %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]

 ; }
 define void @gather_pow2(i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
 ;
 ; CHECK-LABEL: gather_pow2:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    li a2, 1024
 ; CHECK-NEXT:    li a3, 16
 ; CHECK-NEXT:    li a4, 32
-; CHECK-NEXT:  .LBB6_1: # %vector.body
+; CHECK-NEXT:  .LBB8_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; CHECK-NEXT:    vlse32.v v8, (a1), a3
 ; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v9, (a0)
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
 ; CHECK-NEXT:    vse8.v v8, (a0)
 ; CHECK-NEXT:    addi a2, a2, -8
 ; CHECK-NEXT:    addi a0, a0, 32
 ; CHECK-NEXT:    addi a1, a1, 128
-; CHECK-NEXT:    bnez a2, .LBB6_1
+; CHECK-NEXT:    bnez a2, .LBB8_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
   %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]

 ;}
 define void @scatter_pow2(i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
 ;
 ; CHECK-LABEL: scatter_pow2:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    li a2, 1024
 ; CHECK-NEXT:    li a3, 32
 ; CHECK-NEXT:    li a4, 16
-; CHECK-NEXT:  .LBB7_1: # %vector.body
+; CHECK-NEXT:  .LBB9_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a1)
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; CHECK-NEXT:    vlse32.v v9, (a0), a4
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vsse32.v v8, (a0), a4
 ; CHECK-NEXT:    addi a2, a2, -8
 ; CHECK-NEXT:    addi a1, a1, 32
 ; CHECK-NEXT:    addi a0, a0, 128
-; CHECK-NEXT:    bnez a2, .LBB7_1
+; CHECK-NEXT:    bnez a2, .LBB9_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
   %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]

 define void @struct_gather(i32* noalias nocapture %A, %struct.foo* noalias nocapture readonly %B) {
 ;
 ; CHECK-LABEL: struct_gather:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi a1, a1, 132
 ; CHECK-NEXT:    li a2, 1024
 ; CHECK-NEXT:    li a3, 16
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
-; CHECK-NEXT:  .LBB8_1: # %vector.body
+; CHECK-NEXT:  .LBB10_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    addi a4, a1, -128
 ; CHECK-NEXT:    vlse32.v v8, (a4), a3
 ; CHECK-NEXT:    vlse32.v v9, (a1), a3
 ; CHECK-NEXT:    vle32.v v10, (a0)
 ; CHECK-NEXT:    addi a4, a0, 32
 ; CHECK-NEXT:    vle32.v v11, (a4)
 ; CHECK-NEXT:    vadd.vv v8, v10, v8
 ; CHECK-NEXT:    vadd.vv v9, v11, v9
 ; CHECK-NEXT:    vse32.v v8, (a0)
 ; CHECK-NEXT:    vse32.v v9, (a4)
 ; CHECK-NEXT:    addi a2, a2, -16
 ; CHECK-NEXT:    addi a0, a0, 64
 ; CHECK-NEXT:    addi a1, a1, 256
-; CHECK-NEXT:    bnez a2, .LBB8_1
+; CHECK-NEXT:    bnez a2, .LBB10_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
   %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]

 define void @gather_unroll(i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
 ;
 ; CHECK-LABEL: gather_unroll:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    li a2, 256
 ; CHECK-NEXT:    li a3, 64
 ; CHECK-NEXT:    li a4, 16
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
-; CHECK-NEXT:  .LBB9_1: # %vector.body
+; CHECK-NEXT:  .LBB11_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vlse32.v v8, (a1), a3
 ; CHECK-NEXT:    vlse32.v v9, (a0), a4
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vsse32.v v8, (a0), a4
 ; CHECK-NEXT:    addi a5, a1, 16
 ; CHECK-NEXT:    vlse32.v v8, (a5), a3
 ; CHECK-NEXT:    addi a5, a0, 4

 ; CHECK-NEXT:    vlse32.v v8, (a5), a3
 ; CHECK-NEXT:    addi a5, a0, 12
 ; CHECK-NEXT:    vlse32.v v9, (a5), a4
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vsse32.v v8, (a5), a4
 ; CHECK-NEXT:    addi a2, a2, -8
 ; CHECK-NEXT:    addi a1, a1, 512
 ; CHECK-NEXT:    addi a0, a0, 128
-; CHECK-NEXT:    bnez a2, .LBB9_1
+; CHECK-NEXT:    bnez a2, .LBB11_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
   %vec.ind = phi <8 x i64> [ <i64 0, i64 4, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>, %entry ], [ %vec.ind.next, %vector.body ]

 ; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.
 define void @gather_of_pointers(i32** noalias nocapture %0, i32** noalias nocapture readonly %1) {
 ;
 ; V-LABEL: gather_of_pointers:
 ; V:       # %bb.0:
 ; V-NEXT:    li a2, 1024
 ; V-NEXT:    li a3, 40
 ; V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; V-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
+; V-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; V-NEXT:    vlse64.v v8, (a1), a3
 ; V-NEXT:    addi a4, a1, 80
 ; V-NEXT:    vlse64.v v9, (a4), a3
 ; V-NEXT:    vse64.v v8, (a0)
 ; V-NEXT:    addi a4, a0, 16
 ; V-NEXT:    vse64.v v9, (a4)
 ; V-NEXT:    addi a2, a2, -4
 ; V-NEXT:    addi a0, a0, 32
 ; V-NEXT:    addi a1, a1, 160
-; V-NEXT:    bnez a2, .LBB10_1
+; V-NEXT:    bnez a2, .LBB12_1
 ; V-NEXT:  # %bb.2:
 ; V-NEXT:    ret
 ;
 ; ZVE32F-LABEL: gather_of_pointers:
 ; ZVE32F:       # %bb.0:
 ; ZVE32F-NEXT:    li a2, 0
 ; ZVE32F-NEXT:    li a3, 1
 ; ZVE32F-NEXT:    li a4, 1024
 ; ZVE32F-NEXT:    li a5, 40
-; ZVE32F-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
+; ZVE32F-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; ZVE32F-NEXT:    mul a6, a3, a5
 ; ZVE32F-NEXT:    add a6, a1, a6
 ; ZVE32F-NEXT:    mul a7, a2, a5
 ; ZVE32F-NEXT:    add a7, a1, a7
 ; ZVE32F-NEXT:    ld t0, 0(a6)
 ; ZVE32F-NEXT:    ld t1, 0(a7)
 ; ZVE32F-NEXT:    ld a6, 80(a6)
 ; ZVE32F-NEXT:    ld a7, 80(a7)
 ; ZVE32F-NEXT:    sd t0, 8(a0)
 ; ZVE32F-NEXT:    sd t1, 0(a0)
 ; ZVE32F-NEXT:    sd a6, 24(a0)
 ; ZVE32F-NEXT:    sd a7, 16(a0)
 ; ZVE32F-NEXT:    addi a2, a2, 4
 ; ZVE32F-NEXT:    addi a3, a3, 4
 ; ZVE32F-NEXT:    addi a4, a4, -4
 ; ZVE32F-NEXT:    addi a0, a0, 32
-; ZVE32F-NEXT:    bnez a4, .LBB10_1
+; ZVE32F-NEXT:    bnez a4, .LBB12_1
 ; ZVE32F-NEXT:  # %bb.2:
 ; ZVE32F-NEXT:    ret
   br label %3
 
 3:                                                ; preds = %3, %2
   %4 = phi i64 [ 0, %2 ], [ %17, %3 ]
   %5 = phi <2 x i64> [ <i64 0, i64 1>, %2 ], [ %18, %3 ]
   %6 = mul nuw nsw <2 x i64> %5, <i64 5, i64 5>

 ; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.
 define void @scatter_of_pointers(i32** noalias nocapture %0, i32** noalias nocapture readonly %1) {
 ;
 ; V-LABEL: scatter_of_pointers:
 ; V:       # %bb.0:
 ; V-NEXT:    li a2, 1024
 ; V-NEXT:    li a3, 40
 ; V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; V-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; V-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
 ; V-NEXT:    vle64.v v8, (a1)
 ; V-NEXT:    addi a4, a1, 16
 ; V-NEXT:    vle64.v v9, (a4)
 ; V-NEXT:    addi a4, a0, 80
 ; V-NEXT:    vsse64.v v8, (a0), a3
 ; V-NEXT:    vsse64.v v9, (a4), a3
 ; V-NEXT:    addi a2, a2, -4
 ; V-NEXT:    addi a1, a1, 32
 ; V-NEXT:    addi a0, a0, 160
-; V-NEXT:    bnez a2, .LBB11_1
+; V-NEXT:    bnez a2, .LBB13_1
 ; V-NEXT:  # %bb.2:
 ; V-NEXT:    ret
 ;
 ; ZVE32F-LABEL: scatter_of_pointers:
 ; ZVE32F:       # %bb.0:
 ; ZVE32F-NEXT:    li a2, 0
 ; ZVE32F-NEXT:    li a3, 1
 ; ZVE32F-NEXT:    li a4, 1024
 ; ZVE32F-NEXT:    li a5, 40
-; ZVE32F-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; ZVE32F-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
 ; ZVE32F-NEXT:    ld a6, 8(a1)
 ; ZVE32F-NEXT:    ld a7, 0(a1)
 ; ZVE32F-NEXT:    ld t0, 24(a1)
 ; ZVE32F-NEXT:    ld t1, 16(a1)
 ; ZVE32F-NEXT:    mul t2, a3, a5
 ; ZVE32F-NEXT:    add t2, a0, t2
 ; ZVE32F-NEXT:    mul t3, a2, a5
 ; ZVE32F-NEXT:    add t3, a0, t3
 ; ZVE32F-NEXT:    sd a7, 0(t3)
 ; ZVE32F-NEXT:    sd a6, 0(t2)
 ; ZVE32F-NEXT:    sd t1, 80(t3)
 ; ZVE32F-NEXT:    sd t0, 80(t2)
 ; ZVE32F-NEXT:    addi a2, a2, 4
 ; ZVE32F-NEXT:    addi a3, a3, 4
 ; ZVE32F-NEXT:    addi a4, a4, -4
 ; ZVE32F-NEXT:    addi a1, a1, 32
-; ZVE32F-NEXT:    bnez a4, .LBB11_1
+; ZVE32F-NEXT:    bnez a4, .LBB13_1
 ; ZVE32F-NEXT:  # %bb.2:
 ; ZVE32F-NEXT:    ret
   br label %3
 
 3:                                                ; preds = %3, %2
   %4 = phi i64 [ 0, %2 ], [ %17, %3 ]
   %5 = phi <2 x i64> [ <i64 0, i64 1>, %2 ], [ %18, %3 ]
   %6 = getelementptr inbounds i32*, i32** %1, i64 %4

 
 declare void @llvm.masked.scatter.v2p0i32.v2p0p0i32(<2 x i32*>, <2 x i32**>, i32 immarg, <2 x i1>)
 
 define void @strided_load_startval_add_with_splat(i8* noalias nocapture %0, i8* noalias nocapture readonly %1, i32 signext %2) {
 ;
 ; CHECK-LABEL: strided_load_startval_add_with_splat:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a3, 1024
-; CHECK-NEXT:    beq a2, a3, .LBB12_7
+; CHECK-NEXT:    beq a2, a3, .LBB14_7
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    li a3, 1023
 ; CHECK-NEXT:    subw a4, a3, a2
 ; CHECK-NEXT:    li a5, 31
 ; CHECK-NEXT:    mv a3, a2
-; CHECK-NEXT:    bltu a4, a5, .LBB12_5
+; CHECK-NEXT:    bltu a4, a5, .LBB14_5
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    slli a3, a4, 32
 ; CHECK-NEXT:    srli a3, a3, 32
 ; CHECK-NEXT:    addi a4, a3, 1
 ; CHECK-NEXT:    andi a5, a4, -32
 ; CHECK-NEXT:    add a3, a5, a2
 ; CHECK-NEXT:    slli a6, a2, 2
 ; CHECK-NEXT:    add a6, a6, a2
 ; CHECK-NEXT:    add a2, a0, a2
 ; CHECK-NEXT:    add a6, a1, a6
 ; CHECK-NEXT:    li a7, 32
 ; CHECK-NEXT:    li t0, 5
 ; CHECK-NEXT:    mv t1, a5
-; CHECK-NEXT:  .LBB12_3: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:  .LBB14_3: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vsetvli zero, a7, e8, m1, ta, ma
 ; CHECK-NEXT:    vlse8.v v8, (a6), t0
 ; CHECK-NEXT:    vle8.v v9, (a2)
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vse8.v v8, (a2)
 ; CHECK-NEXT:    addi t1, t1, -32
 ; CHECK-NEXT:    addi a2, a2, 32
 ; CHECK-NEXT:    addi a6, a6, 160
-; CHECK-NEXT:    bnez t1, .LBB12_3
+; CHECK-NEXT:    bnez t1, .LBB14_3
 ; CHECK-NEXT:  # %bb.4:
-; CHECK-NEXT:    beq a4, a5, .LBB12_7
-; CHECK-NEXT:  .LBB12_5:
+; CHECK-NEXT:    beq a4, a5, .LBB14_7
+; CHECK-NEXT:  .LBB14_5:
 ; CHECK-NEXT:    addiw a2, a3, -1024
 ; CHECK-NEXT:    add a0, a0, a3
 ; CHECK-NEXT:    slli a4, a3, 2
 ; CHECK-NEXT:    add a3, a4, a3
 ; CHECK-NEXT:    add a1, a1, a3
-; CHECK-NEXT:  .LBB12_6: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:  .LBB14_6: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lb a3, 0(a1)
 ; CHECK-NEXT:    lb a4, 0(a0)
 ; CHECK-NEXT:    addw a3, a4, a3
 ; CHECK-NEXT:    sb a3, 0(a0)
 ; CHECK-NEXT:    addiw a2, a2, 1
 ; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    addi a1, a1, 5
-; CHECK-NEXT:    bnez a2, .LBB12_6
-; CHECK-NEXT:  .LBB12_7:
+; CHECK-NEXT:    bnez a2, .LBB14_6
+; CHECK-NEXT:  .LBB14_7:
 ; CHECK-NEXT:    ret
   %4 = icmp eq i32 %2, 1024
   br i1 %4, label %36, label %5
 
 5:                                                ; preds = %3
   %6 = sext i32 %2 to i64
   %7 = sub i32 1023, %2
   %8 = zext i32 %7 to i64

 
 declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32 immarg, <16 x i1>, <16 x i8>)
 declare void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8>, <16 x i8*>, i32 immarg, <16 x i1>)
 
 define void @gather_no_scalar_remainder(i8* noalias nocapture noundef %arg, i8* noalias nocapture noundef readonly %arg1, i64 noundef %arg2) {
 ; CHECK-LABEL: gather_no_scalar_remainder:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    slli a2, a2, 4
-; CHECK-NEXT:    beqz a2, .LBB13_3
+; CHECK-NEXT:    beqz a2, .LBB15_3
 ; CHECK-NEXT:  # %bb.1: # %bb2
 ; CHECK-NEXT:    li a3, 5
 ; CHECK-NEXT:    vsetivli zero, 16, e8, mf2, ta, ma
-; CHECK-NEXT:  .LBB13_2: # %bb4
+; CHECK-NEXT:  .LBB15_2: # %bb4
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vlse8.v v8, (a1), a3
 ; CHECK-NEXT:    vle8.v v9, (a0)
 ; CHECK-NEXT:    vadd.vv v8, v9, v8
 ; CHECK-NEXT:    vse8.v v8, (a0)
 ; CHECK-NEXT:    addi a2, a2, -16
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    addi a1, a1, 80
-; CHECK-NEXT:    bnez a2, .LBB13_2
-; CHECK-NEXT:  .LBB13_3: # %bb16
+; CHECK-NEXT:    bnez a2, .LBB15_2
+; CHECK-NEXT:  .LBB15_3: # %bb16
 ; CHECK-NEXT:    ret
 bb:
   %i = shl i64 %arg2, 4
   %i3 = icmp eq i64 %i, 0
   br i1 %i3, label %bb16, label %bb2
 
 bb2:
   br label %bb4