[RISCV] Enable CGP to sink splat operands of Add/Sub/Mul/Shl/LShr/AShr

[craig.topper]

Summary

LICM may have pulled out a splat, but with .vx instructions we
can fold it into an operation.

This patch enables CGP to reverse the LICM transform and move the
splat back into the loop.

I've started with the commutable integer operations and shifts, but we can
extend this with more operations in future patches.

[frasercrmck]

Thanks for this patch! I've seen this problem locally but I hadn't found the time to investigate this; I didn't know there was already a handy utility to help sink these splats.

The code and principle definitely LGTM/SGTM, just I had questions about how else we may be able to go about it.

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1053

Do you think the IR pattern matching library would be appropriate to use here?

While I'm on that subject, do you think it's worth separating this logic out into a new pattern matcher (e.g. m_VectorSplat) if there isn't one already?

llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll

1

Might be worth pre-committing these tests just to help this patch shine.

[craig.topper]

Use PatternMatch.
Base the implementation on ARM instead of AArch64. This should give better options for future additions.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1053

I've switched to basing this on ARM's implementation which uses PatternMatch. I dropped the code for looking through a bitcast though. I'm uncomfortable using an m_VectorSplat because of the Ops.push_back(&Op->getOperandUse(0)). We need to know exactly what instructions we matched for that to be correct.

[frasercrmck]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1053

Yeah I think this is an improvement, thanks. Fair enough on m_VectorSplat; I hadn't fully understood exactly what we're supposed to return in this vector.

1065

Is it worth performing this check before the match? I'm assuming this early-exits more often and more cheaply than the match.

[craig.topper]

Move isSinker call earlier.

[craig.topper]

Add support shl/lshr/ashr as well.

[frasercrmck]

LGTM, thanks. I suppose we'll need to look into either better placement or hoisting of vsetvli ahead/out of the loop.

llvm/lib/Target/RISCV/RISCVISelLowering.h

public:
   bool isLegalICmpImmediate(int64_t Imm) const override;
   bool isLegalAddImmediate(int64_t Imm) const override;
   bool isTruncateFree(Type *SrcTy, Type *DstTy) const override;
   bool isTruncateFree(EVT SrcVT, EVT DstVT) const override;
   bool isZExtFree(SDValue Val, EVT VT2) const override;
   bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override;
   bool isCheapToSpeculateCttz() const override;
   bool isCheapToSpeculateCtlz() const override;
+  bool shouldSinkOperands(Instruction *I,
+                          SmallVectorImpl<Use *> &Ops) const override;
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
 
   bool softPromoteHalfType() const override { return true; }
 
   /// Return the register type for a given MVT, ensuring vectors are treated
   /// as a series of gpr sized integers.
   MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 

 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
   return Subtarget.hasStdExtZbb();
 }
 
 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
   return Subtarget.hasStdExtZbb();
 }
 
+/// Check if sinking \p I's operands to I's basic block is profitable, because
+/// the operands can be folded into a target instruction, e.g.
+/// splats of scalars can fold into vector instructions.
+bool RISCVTargetLowering::shouldSinkOperands(
+    Instruction *I, SmallVectorImpl<Use *> &Ops) const {
+  using namespace llvm::PatternMatch;
+
+  if (!I->getType()->isVectorTy() || !Subtarget.hasStdExtV())
+    return false;
+
+  auto IsSinker = [&](Instruction *I, int Operand) {
+    switch (I->getOpcode()) {
+    case Instruction::Add:
+    case Instruction::Sub:
+    case Instruction::Mul:
+      return true;
+    case Instruction::Shl:
+    case Instruction::LShr:
+    case Instruction::AShr:
+      return Operand == 1;
+    default:
+      return false;
+    }
+  };
+
+  for (auto OpIdx : enumerate(I->operands())) {
+    if (!IsSinker(I, OpIdx.index()))
+      continue;
+
+    Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
+    // Make sure we are not already sinking this operand
+    if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
+      continue;
+
+    // We are looking for a splat that can be sunk.
+    if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
+                             m_Undef(), m_ZeroMask())))
+      continue;
+
+    // All uses of the shuffle should be sunk to avoid duplicating it across gpr
+    // and vector registers
+    for (Use &U : Op->uses()) {
+      Instruction *Insn = cast<Instruction>(U.getUser());
+      if (!IsSinker(Insn, U.getOperandNo()))
+        return false;
+    }
+
+    Ops.push_back(&Op->getOperandUse(0));
+    Ops.push_back(&OpIdx.value());
+  }
+  return true;
+}
+
 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                        bool ForCodeSize) const {
   if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
     return false;
   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
     return false;
   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
     return false;

llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-v \
 ; RUN:     -riscv-v-vector-bits-min=128 | FileCheck %s
 
 define void @sink_splat_mul(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_mul:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 1024
+; CHECK-NEXT:    addi a2, zero, 1024
 ; CHECK-NEXT:  .LBB0_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v26, (a0)
-; CHECK-NEXT:    vmul.vv v26, v26, v25
-; CHECK-NEXT:    vse32.v v26, (a0)
-; CHECK-NEXT:    addi a1, a1, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vmul.vx v25, v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    addi a2, a2, -4
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bnez a1, .LBB0_1
+; CHECK-NEXT:    bnez a2, .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @sink_splat_add(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_add:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 1024
+; CHECK-NEXT:    addi a2, zero, 1024
 ; CHECK-NEXT:  .LBB1_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v26, (a0)
-; CHECK-NEXT:    vadd.vv v26, v26, v25
-; CHECK-NEXT:    vse32.v v26, (a0)
-; CHECK-NEXT:    addi a1, a1, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vadd.vx v25, v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    addi a2, a2, -4
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bnez a1, .LBB1_1
+; CHECK-NEXT:    bnez a2, .LBB1_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @sink_splat_sub(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_sub:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 1024
+; CHECK-NEXT:    addi a2, zero, 1024
 ; CHECK-NEXT:  .LBB2_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v26, (a0)
-; CHECK-NEXT:    vadd.vv v26, v26, v25
-; CHECK-NEXT:    vse32.v v26, (a0)
-; CHECK-NEXT:    addi a1, a1, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vadd.vx v25, v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    addi a2, a2, -4
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bnez a1, .LBB2_1
+; CHECK-NEXT:    bnez a2, .LBB2_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @sink_splat_rsub(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_rsub:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 1024
+; CHECK-NEXT:    addi a2, zero, 1024
 ; CHECK-NEXT:  .LBB3_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v26, (a0)
-; CHECK-NEXT:    vsub.vv v26, v25, v26
-; CHECK-NEXT:    vse32.v v26, (a0)
-; CHECK-NEXT:    addi a1, a1, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vrsub.vx v25, v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    addi a2, a2, -4
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bnez a1, .LBB3_1
+; CHECK-NEXT:    bnez a2, .LBB3_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @sink_splat_mul_scalable(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_mul_scalable:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    srli a7, a5, 1
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a7, a2, 1
 ; CHECK-NEXT:    addi a3, zero, 1024
 ; CHECK-NEXT:    bgeu a3, a7, .LBB4_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    mv t0, zero
 ; CHECK-NEXT:    j .LBB4_5
 ; CHECK-NEXT:  .LBB4_2: # %vector.ph
-; CHECK-NEXT:    mv a4, zero
+; CHECK-NEXT:    mv a5, zero
 ; CHECK-NEXT:    remu a6, a3, a7
-; CHECK-NEXT:    sub a3, a3, a6
-; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    sub t0, a3, a6
+; CHECK-NEXT:    slli a4, a2, 1
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:  .LBB4_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vl2re32.v v28, (a2)
-; CHECK-NEXT:    vmul.vv v28, v28, v26
-; CHECK-NEXT:    vs2r.v v28, (a2)
-; CHECK-NEXT:    add a4, a4, a7
-; CHECK-NEXT:    add a2, a2, a5
-; CHECK-NEXT:    bne a4, a3, .LBB4_3
+; CHECK-NEXT:    vl2re32.v v26, (a2)
+; CHECK-NEXT:    vsetvli a3, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vmul.vx v26, v26, a1
+; CHECK-NEXT:    vs2r.v v26, (a2)
+; CHECK-NEXT:    add a5, a5, a7
+; CHECK-NEXT:    add a2, a2, a4
+; CHECK-NEXT:    bne a5, t0, .LBB4_3
 ; CHECK-NEXT:  # %bb.4: # %middle.block
 ; CHECK-NEXT:    beqz a6, .LBB4_7
 ; CHECK-NEXT:  .LBB4_5: # %for.body.preheader
-; CHECK-NEXT:    addi a2, a3, -1024
-; CHECK-NEXT:    slli a3, a3, 2
+; CHECK-NEXT:    addi a2, t0, -1024
+; CHECK-NEXT:    slli a3, t0, 2
 ; CHECK-NEXT:    add a0, a0, a3
 ; CHECK-NEXT:  .LBB4_6: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lw a3, 0(a0)
 ; CHECK-NEXT:    mv a4, a2
 ; CHECK-NEXT:    mulw a2, a3, a1
 ; CHECK-NEXT:    sw a2, 0(a0)
 ; CHECK-NEXT:    addi a2, a4, 1
for.body:                                         ; preds = %for.body.preheader, %for.body
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %cmp.not = icmp eq i64 %indvars.iv.next, 1024
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
 
 define void @sink_splat_add_scalable(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_add_scalable:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    srli a7, a5, 1
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a7, a2, 1
 ; CHECK-NEXT:    addi a3, zero, 1024
 ; CHECK-NEXT:    bgeu a3, a7, .LBB5_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    mv t0, zero
 ; CHECK-NEXT:    j .LBB5_5
 ; CHECK-NEXT:  .LBB5_2: # %vector.ph
-; CHECK-NEXT:    mv a4, zero
+; CHECK-NEXT:    mv a5, zero
 ; CHECK-NEXT:    remu a6, a3, a7
-; CHECK-NEXT:    sub a3, a3, a6
-; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    sub t0, a3, a6
+; CHECK-NEXT:    slli a4, a2, 1
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:  .LBB5_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vl2re32.v v28, (a2)
-; CHECK-NEXT:    vadd.vv v28, v28, v26
-; CHECK-NEXT:    vs2r.v v28, (a2)
-; CHECK-NEXT:    add a4, a4, a7
-; CHECK-NEXT:    add a2, a2, a5
-; CHECK-NEXT:    bne a4, a3, .LBB5_3
+; CHECK-NEXT:    vl2re32.v v26, (a2)
+; CHECK-NEXT:    vsetvli a3, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vadd.vx v26, v26, a1
+; CHECK-NEXT:    vs2r.v v26, (a2)
+; CHECK-NEXT:    add a5, a5, a7
+; CHECK-NEXT:    add a2, a2, a4
+; CHECK-NEXT:    bne a5, t0, .LBB5_3
 ; CHECK-NEXT:  # %bb.4: # %middle.block
 ; CHECK-NEXT:    beqz a6, .LBB5_7
 ; CHECK-NEXT:  .LBB5_5: # %for.body.preheader
-; CHECK-NEXT:    addi a2, a3, -1024
-; CHECK-NEXT:    slli a3, a3, 2
+; CHECK-NEXT:    addi a2, t0, -1024
+; CHECK-NEXT:    slli a3, t0, 2
 ; CHECK-NEXT:    add a0, a0, a3
 ; CHECK-NEXT:  .LBB5_6: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lw a3, 0(a0)
 ; CHECK-NEXT:    mv a4, a2
 ; CHECK-NEXT:    addw a2, a3, a1
 ; CHECK-NEXT:    sw a2, 0(a0)
 ; CHECK-NEXT:    addi a2, a4, 1
for.body:                                         ; preds = %for.body.preheader, %for.body
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %cmp.not = icmp eq i64 %indvars.iv.next, 1024
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
 
 define void @sink_splat_sub_scalable(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_sub_scalable:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    srli a7, a5, 1
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a7, a2, 1
 ; CHECK-NEXT:    addi a3, zero, 1024
 ; CHECK-NEXT:    bgeu a3, a7, .LBB6_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    mv t0, zero
 ; CHECK-NEXT:    j .LBB6_5
 ; CHECK-NEXT:  .LBB6_2: # %vector.ph
-; CHECK-NEXT:    mv a4, zero
+; CHECK-NEXT:    mv a5, zero
 ; CHECK-NEXT:    remu a6, a3, a7
-; CHECK-NEXT:    sub a3, a3, a6
-; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    sub t0, a3, a6
+; CHECK-NEXT:    slli a4, a2, 1
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:  .LBB6_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vl2re32.v v28, (a2)
-; CHECK-NEXT:    vsub.vv v28, v28, v26
-; CHECK-NEXT:    vs2r.v v28, (a2)
-; CHECK-NEXT:    add a4, a4, a7
-; CHECK-NEXT:    add a2, a2, a5
-; CHECK-NEXT:    bne a4, a3, .LBB6_3
+; CHECK-NEXT:    vl2re32.v v26, (a2)
+; CHECK-NEXT:    vsetvli a3, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vsub.vx v26, v26, a1
+; CHECK-NEXT:    vs2r.v v26, (a2)
+; CHECK-NEXT:    add a5, a5, a7
+; CHECK-NEXT:    add a2, a2, a4
+; CHECK-NEXT:    bne a5, t0, .LBB6_3
 ; CHECK-NEXT:  # %bb.4: # %middle.block
 ; CHECK-NEXT:    beqz a6, .LBB6_7
 ; CHECK-NEXT:  .LBB6_5: # %for.body.preheader
-; CHECK-NEXT:    addi a2, a3, -1024
-; CHECK-NEXT:    slli a3, a3, 2
+; CHECK-NEXT:    addi a2, t0, -1024
+; CHECK-NEXT:    slli a3, t0, 2
 ; CHECK-NEXT:    add a0, a0, a3
 ; CHECK-NEXT:  .LBB6_6: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lw a3, 0(a0)
 ; CHECK-NEXT:    mv a4, a2
 ; CHECK-NEXT:    addw a2, a3, a1
 ; CHECK-NEXT:    sw a2, 0(a0)
 ; CHECK-NEXT:    addi a2, a4, 1
for.body:                                         ; preds = %for.body.preheader, %for.body
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %cmp.not = icmp eq i64 %indvars.iv.next, 1024
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
 
 define void @sink_splat_rsub_scalable(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_rsub_scalable:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    srli a7, a5, 1
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a7, a2, 1
 ; CHECK-NEXT:    addi a3, zero, 1024
 ; CHECK-NEXT:    bgeu a3, a7, .LBB7_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    mv t0, zero
 ; CHECK-NEXT:    j .LBB7_5
 ; CHECK-NEXT:  .LBB7_2: # %vector.ph
-; CHECK-NEXT:    mv a4, zero
+; CHECK-NEXT:    mv a5, zero
 ; CHECK-NEXT:    remu a6, a3, a7
-; CHECK-NEXT:    sub a3, a3, a6
-; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    sub t0, a3, a6
+; CHECK-NEXT:    slli a4, a2, 1
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:  .LBB7_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vl2re32.v v28, (a2)
-; CHECK-NEXT:    vsub.vv v28, v26, v28
-; CHECK-NEXT:    vs2r.v v28, (a2)
-; CHECK-NEXT:    add a4, a4, a7
-; CHECK-NEXT:    add a2, a2, a5
-; CHECK-NEXT:    bne a4, a3, .LBB7_3
+; CHECK-NEXT:    vl2re32.v v26, (a2)
+; CHECK-NEXT:    vsetvli a3, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vrsub.vx v26, v26, a1
+; CHECK-NEXT:    vs2r.v v26, (a2)
+; CHECK-NEXT:    add a5, a5, a7
+; CHECK-NEXT:    add a2, a2, a4
+; CHECK-NEXT:    bne a5, t0, .LBB7_3
 ; CHECK-NEXT:  # %bb.4: # %middle.block
 ; CHECK-NEXT:    beqz a6, .LBB7_7
 ; CHECK-NEXT:  .LBB7_5: # %for.body.preheader
-; CHECK-NEXT:    addi a2, a3, -1024
-; CHECK-NEXT:    slli a3, a3, 2
+; CHECK-NEXT:    addi a2, t0, -1024
+; CHECK-NEXT:    slli a3, t0, 2
 ; CHECK-NEXT:    add a0, a0, a3
 ; CHECK-NEXT:  .LBB7_6: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lw a3, 0(a0)
 ; CHECK-NEXT:    mv a4, a2
 ; CHECK-NEXT:    addw a2, a3, a1
 ; CHECK-NEXT:    sw a2, 0(a0)
 ; CHECK-NEXT:    addi a2, a4, 1
for.body:                                         ; preds = %for.body.preheader, %for.body
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %cmp.not = icmp eq i64 %indvars.iv.next, 1024
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
 
 define void @sink_splat_shl(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_shl:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 1024
+; CHECK-NEXT:    addi a2, zero, 1024
 ; CHECK-NEXT:  .LBB8_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v26, (a0)
-; CHECK-NEXT:    vsll.vv v26, v26, v25
-; CHECK-NEXT:    vse32.v v26, (a0)
-; CHECK-NEXT:    addi a1, a1, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vsll.vx v25, v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    addi a2, a2, -4
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bnez a1, .LBB8_1
+; CHECK-NEXT:    bnez a2, .LBB8_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @sink_splat_lshr(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_lshr:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 1024
+; CHECK-NEXT:    addi a2, zero, 1024
 ; CHECK-NEXT:  .LBB9_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v26, (a0)
-; CHECK-NEXT:    vsrl.vv v26, v26, v25
-; CHECK-NEXT:    vse32.v v26, (a0)
-; CHECK-NEXT:    addi a1, a1, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vsrl.vx v25, v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    addi a2, a2, -4
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bnez a1, .LBB9_1
+; CHECK-NEXT:    bnez a2, .LBB9_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @sink_splat_ashr(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_ashr:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 1024
+; CHECK-NEXT:    addi a2, zero, 1024
 ; CHECK-NEXT:  .LBB10_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v26, (a0)
-; CHECK-NEXT:    vsra.vv v26, v26, v25
-; CHECK-NEXT:    vse32.v v26, (a0)
-; CHECK-NEXT:    addi a1, a1, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vsra.vx v25, v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    addi a2, a2, -4
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bnez a1, .LBB10_1
+; CHECK-NEXT:    bnez a2, .LBB10_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
   %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @sink_splat_shl_scalable(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_shl_scalable:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    srli a7, a5, 1
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a7, a2, 1
 ; CHECK-NEXT:    addi a3, zero, 1024
 ; CHECK-NEXT:    bgeu a3, a7, .LBB11_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    mv t0, zero
 ; CHECK-NEXT:    j .LBB11_5
 ; CHECK-NEXT:  .LBB11_2: # %vector.ph
-; CHECK-NEXT:    mv a4, zero
+; CHECK-NEXT:    mv a5, zero
 ; CHECK-NEXT:    remu a6, a3, a7
-; CHECK-NEXT:    sub a3, a3, a6
-; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    sub t0, a3, a6
+; CHECK-NEXT:    slli a4, a2, 1
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:  .LBB11_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vl2re32.v v28, (a2)
-; CHECK-NEXT:    vsll.vv v28, v28, v26
-; CHECK-NEXT:    vs2r.v v28, (a2)
-; CHECK-NEXT:    add a4, a4, a7
-; CHECK-NEXT:    add a2, a2, a5
-; CHECK-NEXT:    bne a4, a3, .LBB11_3
+; CHECK-NEXT:    vl2re32.v v26, (a2)
+; CHECK-NEXT:    vsetvli a3, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vsll.vx v26, v26, a1
+; CHECK-NEXT:    vs2r.v v26, (a2)
+; CHECK-NEXT:    add a5, a5, a7
+; CHECK-NEXT:    add a2, a2, a4
+; CHECK-NEXT:    bne a5, t0, .LBB11_3
 ; CHECK-NEXT:  # %bb.4: # %middle.block
 ; CHECK-NEXT:    beqz a6, .LBB11_7
 ; CHECK-NEXT:  .LBB11_5: # %for.body.preheader
-; CHECK-NEXT:    addi a2, a3, -1024
-; CHECK-NEXT:    slli a3, a3, 2
+; CHECK-NEXT:    addi a2, t0, -1024
+; CHECK-NEXT:    slli a3, t0, 2
 ; CHECK-NEXT:    add a0, a0, a3
 ; CHECK-NEXT:  .LBB11_6: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lw a3, 0(a0)
 ; CHECK-NEXT:    mv a4, a2
 ; CHECK-NEXT:    sllw a2, a3, a1
 ; CHECK-NEXT:    sw a2, 0(a0)
 ; CHECK-NEXT:    addi a2, a4, 1
for.body:                                         ; preds = %for.body.preheader, %for.body
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %cmp.not = icmp eq i64 %indvars.iv.next, 1024
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
 
 define void @sink_splat_lshr_scalable(i32* nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_lshr_scalable:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    srli a7, a5, 1
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a7, a2, 1
 ; CHECK-NEXT:    addi a3, zero, 1024
 ; CHECK-NEXT:    bgeu a3, a7, .LBB12_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    mv t0, zero
 ; CHECK-NEXT:    j .LBB12_5
 ; CHECK-NEXT:  .LBB12_2: # %vector.ph
-; CHECK-NEXT:    mv a4, zero
+; CHECK-NEXT:    mv a5, zero
 ; CHECK-NEXT:    remu a6, a3, a7
-; CHECK-NEXT:    sub a3, a3, a6
-; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    sub t0, a3, a6
+; CHECK-NEXT:    slli a4, a2, 1
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:  .LBB12_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vl2re32.v v28, (a2)
-; CHECK-NEXT:    vsrl.vv v28, v28, v26
-; CHECK-NEXT:    vs2r.v v28, (a2)
-; CHECK-NEXT:    add a4, a4, a7
-; CHECK-NEXT:    add a2, a2, a5
-; CHECK-NEXT:    bne a4, a3, .LBB12_3
+; CHECK-NEXT:    vl2re32.v v26, (a2)
+; CHECK-NEXT:    vsetvli a3, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vsrl.vx v26, v26, a1
+; CHECK-NEXT:    vs2r.v v26, (a2)
+; CHECK-NEXT:    add a5, a5, a7
+; CHECK-NEXT:    add a2, a2, a4
+; CHECK-NEXT:    bne a5, t0, .LBB12_3
 ; CHECK-NEXT:  # %bb.4: # %middle.block
 ; CHECK-NEXT:    beqz a6, .LBB12_7
 ; CHECK-NEXT:  .LBB12_5: # %for.body.preheader
-; CHECK-NEXT:    addi a2, a3, -1024
-; CHECK-NEXT:    slli a3, a3, 2
+; CHECK-NEXT:    addi a2, t0, -1024
+; CHECK-NEXT:    slli a3, t0, 2
 ; CHECK-NEXT:    add a0, a0, a3
 ; CHECK-NEXT:  .LBB12_6: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lw a3, 0(a0)
 ; CHECK-NEXT:    mv a4, a2
 ; CHECK-NEXT:    srlw a2, a3, a1
 ; CHECK-NEXT:    sw a2, 0(a0)
 ; CHECK-NEXT:    addi a2, a4, 1
for.body:                                         ; preds = %for.body.preheader, %for.body
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %cmp.not = icmp eq i64 %indvars.iv.next, 1024
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
 
 define void @sink_splat_ashr_scalable(i32* nocapture %a) {
 ; CHECK-LABEL: sink_splat_ashr_scalable:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    csrr a4, vlenb
-; CHECK-NEXT:    srli a1, a4, 1
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    srli a7, a3, 1
 ; CHECK-NEXT:    addi a2, zero, 1024
-; CHECK-NEXT:    bgeu a2, a1, .LBB13_2
+; CHECK-NEXT:    bgeu a2, a7, .LBB13_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a2, zero
 ; CHECK-NEXT:    j .LBB13_5
 ; CHECK-NEXT:  .LBB13_2: # %vector.ph
-; CHECK-NEXT:    mv a5, zero
-; CHECK-NEXT:    remu a6, a2, a1
+; CHECK-NEXT:    mv a4, zero
+; CHECK-NEXT:    remu a6, a2, a7
 ; CHECK-NEXT:    sub a2, a2, a6
-; CHECK-NEXT:    vsetvli a3, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vmv.v.i v26, 2
-; CHECK-NEXT:    slli a4, a4, 1
+; CHECK-NEXT:    slli a5, a3, 1
 ; CHECK-NEXT:    mv a3, a0
 ; CHECK-NEXT:  .LBB13_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vl2re32.v v28, (a3)
-; CHECK-NEXT:    vsra.vv v28, v28, v26
-; CHECK-NEXT:    vs2r.v v28, (a3)
-; CHECK-NEXT:    add a5, a5, a1
-; CHECK-NEXT:    add a3, a3, a4
-; CHECK-NEXT:    bne a5, a2, .LBB13_3
+; CHECK-NEXT:    vl2re32.v v26, (a3)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vsra.vi v26, v26, 2
+; CHECK-NEXT:    vs2r.v v26, (a3)
+; CHECK-NEXT:    add a4, a4, a7
+; CHECK-NEXT:    add a3, a3, a5
+; CHECK-NEXT:    bne a4, a2, .LBB13_3
 ; CHECK-NEXT:  # %bb.4: # %middle.block
 ; CHECK-NEXT:    beqz a6, .LBB13_7
 ; CHECK-NEXT:  .LBB13_5: # %for.body.preheader
 ; CHECK-NEXT:    addi a1, a2, -1024
 ; CHECK-NEXT:    slli a2, a2, 2
 ; CHECK-NEXT:    add a0, a0, a2
 ; CHECK-NEXT:  .LBB13_6: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1