[RISCV] Fold ops into vmv.v.v as vmerge with all-ones mask

[luke]

Summary

A vmv.v.v shares the same encoding as a vmerge that isn't masked, so we can
also fold it into its operands if we treat it as a vmerge with an all-ones
mask. We take care here not to actually transform the existing vmv into a
vmerge, otherwise things like True.hasOneUse() become inaccurate. Instead this
just returns an equivalent list of operands.
This is an alternative to D153351.

[luke]

Rebase

[reames]

Generally much better than the alternative, but needs a bit of restructuring.

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

3242

Creating a new vmset here is bad if the caller ends up returning.

You can leave the Merge as an uninitialized SDValue, and lazily create the alls ones mask on demand if needed.

3266

This structure is really hard to read.

If you handle the vmerge case inside the helper, you can just unconditionally return the ops array.

[luke]

Rework to create the all ones mask ad-hoc

[reames]

First pass, minor comments only. Have to think about this deeper, and am probably going to get distracted in the next few minutes. Expect further comment later today.

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

3204

Formatting: Please use the same format as the lines above for all of the cases, both in this function and next one.

3277

Personally, the helper function is just confusing abstraction rather than helpful. I'd inline the initialization here, but up to you.

3426

Unrelated change, please land and rebase.

[reames]

LGTM w/previous comments applied.

(Went and read the vector manual very closely to make sure the overall claim in this review holds. It does.)

[luke]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

3204

I was surprised too, but amazingly this is the output of clang-format 🙃

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

case RISCV::TH_MULSH:
     // NOTE: We only handle the nodes that are selected with hasAllWUsers.
     ReplaceUses(N, N0.getNode());
     return true;
   }
 
   return false;
 }
 
-// Return true if we can make sure mask of N is all-ones mask.
-static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
+static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp) {
   // Check that we're using V0 as a mask register.
-  if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) ||
-      cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0)
+  if (!isa<RegisterSDNode>(MaskOp) ||
+      cast<RegisterSDNode>(MaskOp)->getReg() != RISCV::V0)
     return false;
 
   // The glued user defines V0.
-  const auto *Glued = N->getGluedNode();
+  const auto *Glued = GlueOp.getNode();
 
   if (!Glued || Glued->getOpcode() != ISD::CopyToReg)
     return false;
 
   // Check that we're defining V0 as a mask register.
   if (!isa<RegisterSDNode>(Glued->getOperand(1)) ||
       cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0)
     return false;
static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp) {
 
   // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
   // undefined behaviour if it's the wrong bitwidth, so we could choose to
   // assume that it's all-ones? Same applies to its VL.
   return MaskSetter->isMachineOpcode() &&
          IsVMSet(MaskSetter.getMachineOpcode());
 }
 
+// Return true if we can make sure mask of N is all-ones mask.
+static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
+  return usesAllOnesMask(N->getOperand(MaskOpIdx),
+                         N->getOperand(N->getNumOperands() - 1));
+}
+
 static bool isImplicitDef(SDValue V) {
   return V.isMachineOpcode() &&
          V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
 }
 
 // Optimize masked RVV pseudo instructions with a known all-ones mask to their
 // corresponding "unmasked" pseudo versions. The mask we're interested in will
 // take the form of a V0 physical register operand, with a glued
#endif
 
   SDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
   Result->setFlags(N->getFlags());
   ReplaceUses(N, Result);
 
   return true;
 }
 
+// A vmv.v.v is equivalent to a vmerge with an all-ones mask. If N is a
+// PseudoVMV_V_V, this places the operands for an equivalent PseudoVMERGE into
+// Ops.
+static bool getVMergeOpsFromVMv(SDNode *N, SelectionDAG *DAG,
+                                SmallVectorImpl<SDValue> &Ops) {
+  unsigned VmsetOpc;
+  switch (N->getMachineOpcode()) {
+  case RISCV::PseudoVMV_V_V_MF8:
+    VmsetOpc = RISCV::PseudoVMSET_M_B1;
+    break;
+  case RISCV::PseudoVMV_V_V_MF4:
+    VmsetOpc = RISCV::PseudoVMSET_M_B2;
+    break;
+  case RISCV::PseudoVMV_V_V_MF2:
+    VmsetOpc = RISCV::PseudoVMSET_M_B4;
+    break;
+  case RISCV::PseudoVMV_V_V_M1:
+    VmsetOpc = RISCV::PseudoVMSET_M_B8;
+    break;
+  case RISCV::PseudoVMV_V_V_M2:
+    VmsetOpc = RISCV::PseudoVMSET_M_B16;
+    break;
+  case RISCV::PseudoVMV_V_V_M4:
+    VmsetOpc = RISCV::PseudoVMSET_M_B32;
+    break;
+  case RISCV::PseudoVMV_V_V_M8:
+    VmsetOpc = RISCV::PseudoVMSET_M_B64;
+    break;
+  default:
+    return false;
+  }
+
+  SDLoc DL(N);
+  ElementCount EC = N->getValueType(0).getVectorElementCount();
+  MVT MaskVT = MVT::getVectorVT(MVT::i1, EC);
+  SDValue Merge = N->getOperand(0);
+  SDValue Src = N->getOperand(1);
+  SDValue VL = N->getOperand(2);
+  SDValue SEW = N->getOperand(3);
+  SDValue Policy = N->getOperand(4);
+
+  SDValue AllOnesMask =
+      SDValue(DAG->getMachineNode(VmsetOpc, DL, MaskVT, VL, SEW), 0);
+  SDValue MaskCopy = DAG->getCopyToReg(DAG->getEntryNode(), DL, RISCV::V0,
+                                       AllOnesMask, SDValue());
+  SDValue Mask = DAG->getRegister(RISCV::V0, MaskVT);
+
+  Ops = SmallVector<SDValue>(
+      {Merge, Merge, Src, Mask, VL, SEW, Policy, MaskCopy.getValue(1)});
+  return true;
+}
+
 // Try to fold away VMERGE_VVM instructions. We handle these cases:
 // -Masked TU VMERGE_VVM combined with an unmasked TA instruction instruction
 //  folds to a masked TU instruction. VMERGE_VVM must have have merge operand
 //  same as false operand.
 // -Masked TA VMERGE_VVM combined with an unmasked TA instruction fold to a
 //  masked TA instruction.
 // -Unmasked TU VMERGE_VVM combined with a masked MU TA instruction folds to
 //  masked TU instruction. Both instructions must have the same merge operand.
 //  VMERGE_VVM must have have merge operand same as false operand.
 // Note: The VMERGE_VVM forms above (TA, and TU) refer to the policy implied,
 // not the pseudo name.  That is, a TA VMERGE_VVM can be either the _TU pseudo
 // form with an IMPLICIT_DEF passthrough operand or the unsuffixed (TA) pseudo
 // form.
 bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
-
-  SDValue Merge = N->getOperand(0);
-  SDValue False = N->getOperand(1);
-  SDValue True = N->getOperand(2);
-  SDValue Mask = N->getOperand(3);
-  SDValue VL = N->getOperand(4);
+  SmallVector<SDValue, 8> VMergeOps;
+  // Handle vmv.v.v's, since they are equivalent to a vmerge with an all-ones
+  // mask.
+  if (!getVMergeOpsFromVMv(N, CurDAG, VMergeOps))
+    VMergeOps = SmallVector<SDValue>(N->ops());
+  SDValue Merge = VMergeOps[0];
+  SDValue False = VMergeOps[1];
+  SDValue True = VMergeOps[2];
+  SDValue Mask = VMergeOps[3];
+  SDValue VL = VMergeOps[4];
   // We always have a glue node for the mask at v0
+  SDValue Glue = VMergeOps[VMergeOps.size() - 1];
   assert(cast<RegisterSDNode>(Mask)->getReg() == RISCV::V0);
-  SDValue Glue = N->getOperand(N->getNumOperands() - 1);
   assert(Glue.getValueType() == MVT::Glue);
 
   // We require that either merge and false are the same, or that merge
   // is undefined.
   if (Merge != False && !isImplicitDef(Merge))
     return false;
 
   assert(True.getResNo() == 0 &&
          "Expect True is the first output of an instruction.");
 
if (IsMasked) {
     assert(HasTiedDest && "Expected tied dest");
     // The vmerge instruction must be TU.
     if (isImplicitDef(Merge))
       return false;
     // The vmerge instruction must have an all 1s mask since we're going to keep
     // the mask from the True instruction.
     // FIXME: Support mask agnostic True instruction which would have an
     // undef merge operand.
-    if (!usesAllOnesMask(N, /* MaskOpIdx */ 3))
+    if (!usesAllOnesMask(Mask, Glue))
       return false;
   }
 
   // Skip if True has side effect.
   // TODO: Support vleff and vlsegff.
   if (TII->get(TrueOpc).hasUnmodeledSideEffects())
     return false;
 
assert(RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) &&
          "Expected instructions with mask have policy operand.");
   assert(MaskedMCID.getOperandConstraint(MaskedMCID.getNumDefs(),
                                          MCOI::TIED_TO) == 0 &&
          "Expected instructions with mask have a tied dest.");
 #endif
 
   SDValue SEW = True.getOperand(TrueVLIndex + 1);
 
-  uint64_t Policy = isImplicitDef(N->getOperand(0)) ?
-    RISCVII::TAIL_AGNOSTIC : /*TUMU*/ 0;
+  uint64_t Policy = isImplicitDef(Merge) ? RISCVII::TAIL_AGNOSTIC : /*TUMU*/ 0;
   SDValue PolicyOp =
     CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT());
 
 
   SmallVector<SDValue, 8> Ops;
   Ops.push_back(False);
 
   const bool HasRoundingMode = RISCVII::hasRoundModeOp(TrueTSFlags);
auto IsVMerge = [](unsigned Opcode) {
              Opcode == RISCV::PseudoVMERGE_VVM_MF4 ||
              Opcode == RISCV::PseudoVMERGE_VVM_MF2 ||
              Opcode == RISCV::PseudoVMERGE_VVM_M1 ||
              Opcode == RISCV::PseudoVMERGE_VVM_M2 ||
              Opcode == RISCV::PseudoVMERGE_VVM_M4 ||
              Opcode == RISCV::PseudoVMERGE_VVM_M8;
     };
 
+    auto IsVMv = [](unsigned Opcode) {
+      return Opcode == RISCV::PseudoVMV_V_V_MF8 ||
+             Opcode == RISCV::PseudoVMV_V_V_MF4 ||
+             Opcode == RISCV::PseudoVMV_V_V_MF2 ||
+             Opcode == RISCV::PseudoVMV_V_V_M1 ||
+             Opcode == RISCV::PseudoVMV_V_V_M2 ||
+             Opcode == RISCV::PseudoVMV_V_V_M4 ||
+             Opcode == RISCV::PseudoVMV_V_V_M8;
+    };
+
     unsigned Opc = N->getMachineOpcode();
-    if (IsVMerge(Opc))
+    if (IsVMerge(Opc) || IsVMv(Opc))
       MadeChange |= performCombineVMergeAndVOps(N);
     if (IsVMerge(Opc) && N->getOperand(0) == N->getOperand(1))
       MadeChange |= performVMergeToVMv(N);
   }
   return MadeChange;
 }
 
 // This pass converts a legalized DAG into a RISCV-specific DAG, ready

llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll

; CHECK-NEXT:    ret
   ret <vscale x 4 x i32> %w
 }
 
 declare <vscale x 4 x i32> @llvm.riscv.vle.nxv4i32(<vscale x 4 x i32>, ptr, iXLen)
 
 define <vscale x 4 x i32> @foldable_load(<vscale x 4 x i32> %passthru, ptr %p) {
 ; CHECK-LABEL: foldable_load:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
-; CHECK-NEXT:    vle32.v v10, (a0)
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m2, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x i32> @llvm.riscv.vle.nxv4i32(<vscale x 4 x i32> poison, ptr %p, iXLen 4)
   %w = call <vscale x 4 x i32> @llvm.riscv.vmv.v.v.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, iXLen 2)
   ret <vscale x 4 x i32> %w
 }
 
 ; Can't fold this as the VLs aren't constant.
 define <vscale x 4 x i32> @unfoldable_load(<vscale x 4 x i32> %passthru, ptr %p, iXLen %vl1, iXLen %vl2) {

llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll

 }
 
 define <vscale x 6 x half> @extract_nxv6f16_nxv12f16_6(<vscale x 12 x half> %in) {
 ; CHECK-LABEL: extract_nxv6f16_nxv12f16_6:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v11, v10, a0
 ; CHECK-NEXT:    vslidedown.vx v8, v9, a0
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v9, v11
 ; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; CHECK-NEXT:    vslidedown.vx v9, v10, a0
 ; CHECK-NEXT:    ret
   %res = call <vscale x 6 x half> @llvm.vector.extract.nxv6f16.nxv12f16(<vscale x 12 x half> %in, i64 6)
   ret <vscale x 6 x half> %res
 }
 
 declare <vscale x 6 x half> @llvm.vector.extract.nxv6f16.nxv12f16(<vscale x 12 x half>, i64)
 
 declare <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv4i8(<vscale x 4 x i8> %vec, i64 %idx)

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector-shuffle.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mattr=+v -mtriple=riscv32 -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mattr=+v -mtriple=riscv64 -verify-machineinstrs < %s | FileCheck %s
 
 define <4 x i32> @insert_subvector_load_v4i32_v4i32(<4 x i32> %v1, ptr %p) {
 ; CHECK-LABEL: insert_subvector_load_v4i32_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vle32.v v9, (a0)
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    ret
   %v2 = load <4 x i32>, ptr %p
   %v3 = shufflevector <4 x i32> %v2, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v3
 }
 
 declare <4 x i32> @llvm.vp.load.v4i32(ptr, <4 x i1>, i32)
 define <4 x i32> @insert_subvector_vp_load_v4i32_v4i32(<4 x i32> %v1, ptr %p, <4 x i1> %mask) {
 ; CHECK-LABEL: insert_subvector_vp_load_v4i32_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vle32.v v9, (a0), v0.t
-; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, mu
+; CHECK-NEXT:    vle32.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %v2 = call <4 x i32> @llvm.vp.load.v4i32(ptr %p, <4 x i1> %mask, i32 4)
   %v3 = shufflevector <4 x i32> %v2, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v3
 }
 
 ; Can't fold this in because the load has a non-undef passthru that isn't equal to the vmv.v.v passtrhu
 declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)
; CHECK-NEXT:    ret
   %v3 = shufflevector <4 x i32> %v2, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v3
 }
 
 ; Can fold this in because the load has a non-undef passthru, but it's equal to the vmv.v.v passtrhu
 define <4 x i32> @insert_subvector_load_foldable_passthru_v4i32_v4i32(<4 x i32> %v1, ptr %p, <4 x i1> %mask) {
 ; CHECK-LABEL: insert_subvector_load_foldable_passthru_v4i32_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vle32.v v9, (a0), v0.t
-; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, mu
+; CHECK-NEXT:    vle32.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %v2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %p, i32 4, <4 x i1> %mask, <4 x i32> %v1)
   %v3 = shufflevector <4 x i32> %v2, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v3
 }
 
 define <4 x i32> @insert_subvector_add_v4i32_v4i32(<4 x i32> %v1, <4 x i32> %v2) {
 ; CHECK-LABEL: insert_subvector_add_v4i32_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vadd.vv v9, v9, v10
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    vadd.vv v8, v9, v10
 ; CHECK-NEXT:    ret
   %v3 = add <4 x i32> %v2, <i32 0, i32 1, i32 2, i32 3>
   %v4 = shufflevector <4 x i32> %v3, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v4
 }
 
 declare <4 x i32> @llvm.vp.add.v4i32(<4 x i32>, <4 x i32>, <4 x i1>, i32)
 define <4 x i32> @insert_subvector_vp_add_v4i32_v4i32(<4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) {
 ; CHECK-LABEL: insert_subvector_vp_add_v4i32_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vadd.vi v9, v9, 1, v0.t
-; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, mu
+; CHECK-NEXT:    vadd.vi v8, v9, 1, v0.t
 ; CHECK-NEXT:    ret
   %v3 = call <4 x i32> @llvm.vp.add.v4i32(<4 x i32> %v2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i1> %mask, i32 4)
   %v4 = shufflevector <4 x i32> %v3, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v4
 }
 
 define <4 x i32> @insert_subvector_load_v4i32_v2i32(<4 x i32> %v1, ptr %p) {
 ; CHECK-LABEL: insert_subvector_load_v4i32_v2i32:
; CHECK-NEXT:    ret
   %v4 = shufflevector <2 x i32> %v3, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %v5 = shufflevector <4 x i32> %v4, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v5
 }
 
 define <4 x i32> @insert_subvector_load_v4i32_v8i32(<4 x i32> %v1, ptr %p) {
 ; CHECK-LABEL: insert_subvector_load_v4i32_v8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vle32.v v9, (a0)
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    ret
   %v2 = load <8 x i32>, ptr %p
   %v3 = shufflevector <8 x i32> %v2, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %v4 = shufflevector <4 x i32> %v3, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v4
 }
 
 declare <8 x i32> @llvm.vp.load.v8i32(ptr, <8 x i1>, i32)
; CHECK-NEXT:    ret
   ret <4 x i32> %v4
 }
 
 define <4 x i32> @insert_subvector_add_v4i32_v8i32(<4 x i32> %v1, <8 x i32> %v2) {
 ; CHECK-LABEL: insert_subvector_add_v4i32_v8i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vadd.vv v9, v10, v9
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m1, tu, ma
-; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    vadd.vv v8, v10, v9
 ; CHECK-NEXT:    ret
   %v3 = add <8 x i32> %v2, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %v4 = shufflevector <8 x i32> %v3, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %v5 = shufflevector <4 x i32> %v4, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   ret <4 x i32> %v5
 }
 
 declare <8 x i32> @llvm.vp.add.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32)