[ARM] Attempt to distribute reductions

[dmgreen]

Summary

This adds a combine for adds of reductions, distributing them so that they occur sequentially to enable better use of accumulating VADDVA instructions. It combines:

add(X, add(vecreduce(Y), vecreduce(Z))) -> add(add(X, vecreduce(Y)), vecreduce(Z))

and

add(add(A, reduce(B)), add(C, reduce(D))) -> add(add(add(A, C), reduce(B)), reduce(D))

These together distribute the add's so that more reductions can be selected to VADDVA.

[NickGuy]

Apart from a personal gripe (which I feel shouldn't stop this being merged), LGTM

llvm/lib/Target/ARM/ARMISelLowering.cpp

13103–13104

If this IsVecReduce returns true, then the second will run needlessly. I'd prefer if the second invocation was conditional on the first failing.

llvm/lib/Target/ARM/ARMISelLowering.cpp

static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
   if (N0.getNode()->hasOneUse())
     if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
       return Result;
   return SDValue();
 }
 
 static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
                                    const ARMSubtarget *Subtarget) {
-  if (!Subtarget->hasMVEIntegerOps() || N->getValueType(0) != MVT::i64)
+  if (!Subtarget->hasMVEIntegerOps())
     return SDValue();
 
+  EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
+  SDLoc dl(N);
+
+  auto IsVecReduce = [](SDValue Op) {
+    switch (Op.getOpcode()) {
+    case ISD::VECREDUCE_ADD:
+    case ARMISD::VADDVs:
+    case ARMISD::VADDVu:
+    case ARMISD::VMLAVs:
+    case ARMISD::VMLAVu:
+      return true;
+    }
+    return false;
+  };
+
+  auto DistrubuteAddAddVecReduce = [&](SDValue N0, SDValue N1) {
+    // Distribute add(X, add(vecreduce(Y), vecreduce(Z))) ->
+    //   add(add(X, vecreduce(Y)), vecreduce(Z))
+    // to make better use of vaddva style instructions.
+    if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) &&
+        IsVecReduce(N1.getOperand(0)) && IsVecReduce(N1.getOperand(1))) {
+      SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0, N1.getOperand(0));
+      return DAG.getNode(ISD::ADD, dl, VT, Add0, N1.getOperand(1));
+    }
+    // And turn add(add(A, reduce(B)), add(C, reduce(D))) ->
+    //   add(add(add(A, C), reduce(B)), reduce(D))
+    if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD &&
+        N1.getOpcode() == ISD::ADD) {
+      unsigned N0RedOp = 0;
+      if (!IsVecReduce(N0.getOperand(N0RedOp))) {
+        N0RedOp = 1;
+        if (!IsVecReduce(N0.getOperand(N0RedOp)))
+          return SDValue();
+      }
+
+      unsigned N1RedOp = 0;
+      if (!IsVecReduce(N1.getOperand(N1RedOp)))
+        N1RedOp = 1;
+      if (!IsVecReduce(N1.getOperand(N1RedOp)))
+        return SDValue();
+
+      SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0.getOperand(1 - N0RedOp),
+                                 N1.getOperand(1 - N1RedOp));
+      SDValue Add1 =
+          DAG.getNode(ISD::ADD, dl, VT, Add0, N0.getOperand(N0RedOp));
+      return DAG.getNode(ISD::ADD, dl, VT, Add1, N1.getOperand(N1RedOp));
+    }
+    return SDValue();
+  };
+  if (SDValue R = DistrubuteAddAddVecReduce(N0, N1))
+    return R;
+  if (SDValue R = DistrubuteAddAddVecReduce(N1, N0))
+    return R;
+
+  if (VT != MVT::i64)
+    return SDValue();
 
   // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
   // will look like:
   //   t1: i32,i32 = ARMISD::VADDLVs x
   //   t2: i64 = build_pair t1, t1:1
   //   t3: i64 = add t2, y
   // Otherwise we try to push the add up above VADDLVAx, to potentially allow
   // the add to be simplified seperately.
   // We also need to check for sext / zext and commutitive adds.
   auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
                            SDValue NB) {
     if (NB->getOpcode() != ISD::BUILD_PAIR)
       return SDValue();
     SDValue VecRed = NB->getOperand(0);
     if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) ||
         VecRed.getResNo() != 0 ||
         NB->getOperand(1) != SDValue(VecRed.getNode(), 1))
       return SDValue();
 
-    SDLoc dl(N);
     if (VecRed->getOpcode() == OpcodeA) {
       // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y)
       SDValue Inp = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
                                 VecRed.getOperand(0), VecRed.getOperand(1));
       NA = DAG.getNode(ISD::ADD, dl, MVT::i64, Inp, NA);
     }
 
     SmallVector<SDValue, 4> Ops;

llvm/test/CodeGen/Thumb2/mve-vaddv.ll

entry:
   %t = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %s1)
   %r = add i32 %t, %x
   ret i32 %r
 }
 
 define arm_aapcs_vfpcc i32 @vaddva_v8i32_i32(<8 x i32> %s1, i32 %x) {
 ; CHECK-LABEL: vaddva_v8i32_i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vaddva.u32 r0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %t = call i32 @llvm.vector.reduce.add.i32.v8i32(<8 x i32> %s1)
   %r = add i32 %t, %x
   ret i32 %r
 }
 
 define arm_aapcs_vfpcc i16 @vaddva_v8i16_i16(<8 x i16> %s1, i16 %x) {
 ; CHECK-LABEL: vaddva_v8i16_i16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vaddva.u16 r0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %t = call i16 @llvm.vector.reduce.add.i16.v8i16(<8 x i16> %s1)
   %r = add i16 %t, %x
   ret i16 %r
 }
 
 define arm_aapcs_vfpcc i16 @vaddva_v16i16_i16(<16 x i16> %s1, i16 %x) {
 ; CHECK-LABEL: vaddva_v16i16_i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vaddv.u16 r2, q1
-; CHECK-NEXT:    vaddva.u16 r2, q0
-; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vaddva.u16 r0, q0
+; CHECK-NEXT:    vaddva.u16 r0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %t = call i16 @llvm.vector.reduce.add.i16.v16i16(<16 x i16> %s1)
   %r = add i16 %t, %x
   ret i16 %r
 }
 
 define arm_aapcs_vfpcc i8 @vaddva_v16i8_i8(<16 x i8> %s1, i8 %x) {
 ; CHECK-LABEL: vaddva_v16i8_i8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vaddva.u8 r0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %t = call i8 @llvm.vector.reduce.add.i8.v16i8(<16 x i8> %s1)
   %r = add i8 %t, %x
   ret i8 %r
 }
 
 define arm_aapcs_vfpcc i8 @vaddva_v32i8_i8(<32 x i8> %s1, i8 %x) {
 ; CHECK-LABEL: vaddva_v32i8_i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vaddv.u8 r2, q1
-; CHECK-NEXT:    vaddva.u8 r2, q0
-; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vaddva.u8 r0, q0
+; CHECK-NEXT:    vaddva.u8 r0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %t = call i8 @llvm.vector.reduce.add.i8.v32i8(<32 x i8> %s1)
   %r = add i8 %t, %x
   ret i8 %r
 }

llvm/test/CodeGen/Thumb2/mve-vecreduce-slp.ll

entry:
   %1 = load <8 x i32>, <8 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
   ret i32 %2
 }
 
 define i32 @addv16i32i32(i32* %x) {
 ; CHECK-LABEL: addv16i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r0, q1
-; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vaddv.u32 r2, q1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <16 x i32>*
   %1 = load <16 x i32>, <16 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
   ret i32 %2
 }
 
 define i32 @addv24i32i32(i32* %x) {
 ; CHECK-LABEL: addv24i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #80]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #64]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r0, q1
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <8 x i32>*
   %1 = load <8 x i32>, <8 x i32>* %0, align 4
   %arrayidx.8 = getelementptr inbounds i32, i32* %x, i32 8
   %2 = bitcast i32* %arrayidx.8 to <16 x i32>*
   %3 = load <16 x i32>, <16 x i32>* %2, align 4
   %4 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %3)
   %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
   %op.rdx = add nsw i32 %4, %5
   ret i32 %op.rdx
 }
 
 define i32 @addv32i32i32(i32* %x) {
 ; CHECK-LABEL: addv32i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #112]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #80]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #96]
-; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #64]
-; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    mov r1, r0
 ; CHECK-NEXT:    vaddv.u32 r0, q1
 ; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #96]
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #32]
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #48]
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #64]
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #112]
+; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <32 x i32>*
   %1 = load <32 x i32>, <32 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
   ret i32 %2
 }
 
 define i32 @addv64i32i32(i32* %x) {
 ; CHECK-LABEL: addv64i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #240]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #176]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #208]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #208]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddv.u32 r12, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #144]
-; CHECK-NEXT:    vaddva.u32 r12, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #224]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #224]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #96]
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    vaddv.u32 r12, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #160]
-; CHECK-NEXT:    vaddva.u32 r12, q0
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
-; CHECK-NEXT:    add r1, r2
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #192]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-NEXT:    add.w r3, r2, r12
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #128]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #160]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #144]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r0, q1
-; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    add r0, r3
-; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #192]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #240]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <64 x i32>*
   %1 = load <64 x i32>, <64 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %1)
   ret i32 %2
 }
 
entry:
   %2 = sext <8 x i16> %1 to <8 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv16i32i16(i16* %x) {
 ; CHECK-LABEL: addv16i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #24]
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
-; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vldrh.s32 q1, [r0, #16]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r0, q1
-; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    vaddv.u32 r2, q1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = sext <16 x i16> %1 to <16 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv24i32i16(i16* %x) {
 ; CHECK-LABEL: addv24i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #24]
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrh.s32 q1, [r0, #16]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
-; CHECK-NEXT:    add r2, r12
 ; CHECK-NEXT:    vaddva.s16 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = sext <16 x i16> %1 to <16 x i32>
   %arrayidx.16 = getelementptr inbounds i16, i16* %x, i32 16
   %3 = bitcast i16* %arrayidx.16 to <8 x i16>*
   %4 = load <8 x i16>, <8 x i16>* %3, align 2
   %5 = sext <8 x i16> %4 to <8 x i32>
   %6 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)
   %7 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %5)
   %op.rdx = add nsw i32 %6, %7
   ret i32 %op.rdx
 }
 
 define i32 @addv32i32i16(i16* %x) {
 ; CHECK-LABEL: addv32i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #56]
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrh.s32 q1, [r0, #40]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #48]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #32]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r0, q1
-; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #56]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
   %2 = sext <32 x i16> %1 to <32 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv64i32i16(i16* %x) {
 ; CHECK-LABEL: addv64i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #56]
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
-; CHECK-NEXT:    ldrsh.w r3, [r0, #122]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrh.s32 q1, [r0, #40]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
+; CHECK-NEXT:    ldrsh.w r1, [r0, #120]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #48]
+; CHECK-NEXT:    ldrsh.w r3, [r0, #122]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #48]
+; CHECK-NEXT:    ldrsh.w r12, [r0, #124]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddv.u32 r12, q1
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #32]
-; CHECK-NEXT:    vaddva.u32 r12, q0
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #88]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #72]
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    vaddv.u32 r12, q1
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #80]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #64]
-; CHECK-NEXT:    add r1, r2
-; CHECK-NEXT:    vaddv.u32 r2, q1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #80]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #72]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #56]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #88]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #96]
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    ldrsh.w r12, [r0, #124]
-; CHECK-NEXT:    add r2, r1
-; CHECK-NEXT:    ldrsh.w r1, [r0, #120]
 ; CHECK-NEXT:    vaddva.s16 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #112]
 ; CHECK-NEXT:    ldrsh.w r0, [r0, #126]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    add r1, r2
 ; CHECK-NEXT:    add r1, r3
 ; CHECK-NEXT:    add r1, r12
 ; CHECK-NEXT:    add r0, r1
entry:
   %7 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %5)
   %op.rdx = add nuw nsw i32 %6, %7
   ret i32 %op.rdx
 }
 
 define i32 @addv32i32i8(i8* %x) {
 ; CHECK-LABEL: addv32i32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #28]
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #12]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrb.u32 q1, [r0, #20]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #24]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #24]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #8]
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #16]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #12]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r0, q1
-; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #16]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #28]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <32 x i8>*
   %1 = load <32 x i8>, <32 x i8>* %0, align 1
   %2 = zext <32 x i8> %1 to <32 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv64i32i8(i8* %x) {
 ; CHECK-LABEL: addv64i32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #28]
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #12]
-; CHECK-NEXT:    ldrb.w r3, [r0, #61]
-; CHECK-NEXT:    vaddv.u32 r12, q1
 ; CHECK-NEXT:    vldrb.u32 q1, [r0, #20]
-; CHECK-NEXT:    vaddva.u32 r12, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
+; CHECK-NEXT:    ldrb.w r1, [r0, #60]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #24]
+; CHECK-NEXT:    ldrb.w r3, [r0, #61]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #24]
+; CHECK-NEXT:    ldrb.w r12, [r0, #62]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #8]
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddv.u32 r12, q1
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #16]
-; CHECK-NEXT:    vaddva.u32 r12, q0
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #12]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0]
-; CHECK-NEXT:    vaddv.u32 r2, q1
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #16]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #28]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u8 q0, [r0, #32]
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    ldrb.w r12, [r0, #62]
-; CHECK-NEXT:    add r2, r1
-; CHECK-NEXT:    ldrb.w r1, [r0, #60]
 ; CHECK-NEXT:    vaddva.u8 r2, q0
 ; CHECK-NEXT:    vldrb.u16 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #56]
 ; CHECK-NEXT:    ldrb.w r0, [r0, #63]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    add r1, r2
 ; CHECK-NEXT:    add r1, r3
entry:
   %5 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
   %op.rdx = add i16 %4, %5
   ret i16 %op.rdx
 }
 
 define signext i16 @addv32i16i16(i16* %x) {
 ; CHECK-LABEL: addv32i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #48]
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
-; CHECK-NEXT:    vaddv.u16 r2, q1
 ; CHECK-NEXT:    vldrh.u16 q1, [r0, #32]
-; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0]
-; CHECK-NEXT:    vaddv.u16 r0, q1
-; CHECK-NEXT:    vaddva.u16 r0, q0
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vaddv.u16 r2, q1
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
   %2 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %1)
   ret i16 %2
 }
 
 define signext i16 @addv64i16i16(i16* %x) {
 ; CHECK-LABEL: addv64i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #112]
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
-; CHECK-NEXT:    vaddv.u16 r12, q1
 ; CHECK-NEXT:    vldrh.u16 q1, [r0, #80]
-; CHECK-NEXT:    vaddva.u16 r12, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u16 r2, q1
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #96]
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #96]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
-; CHECK-NEXT:    add.w r1, r2, r12
-; CHECK-NEXT:    vaddv.u16 r2, q1
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #64]
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0]
-; CHECK-NEXT:    vaddv.u16 r0, q1
-; CHECK-NEXT:    vaddva.u16 r0, q0
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    add r0, r1
-; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #64]
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #112]
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <64 x i16>*
   %1 = load <64 x i16>, <64 x i16>* %0, align 2
   %2 = call i16 @llvm.vector.reduce.add.v64i16(<64 x i16> %1)
   ret i16 %2
 }
 
entry:
   %1 = load <32 x i8>, <32 x i8>* %0, align 1
   %2 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %1)
   ret i8 %2
 }
 
 define zeroext i8 @addv64i8i8(i8* %x) {
 ; CHECK-LABEL: addv64i8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u8 q1, [r0, #48]
-; CHECK-NEXT:    vldrb.u8 q0, [r0, #16]
-; CHECK-NEXT:    vaddv.u8 r2, q1
 ; CHECK-NEXT:    vldrb.u8 q1, [r0, #32]
-; CHECK-NEXT:    vaddva.u8 r2, q0
 ; CHECK-NEXT:    vldrb.u8 q0, [r0]
-; CHECK-NEXT:    vaddv.u8 r0, q1
-; CHECK-NEXT:    vaddva.u8 r0, q0
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    vaddv.u8 r2, q1
+; CHECK-NEXT:    vaddva.u8 r2, q0
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #16]
+; CHECK-NEXT:    vaddva.u8 r2, q0
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #48]
+; CHECK-NEXT:    vaddva.u8 r2, q0
+; CHECK-NEXT:    uxtb r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <64 x i8>*
   %1 = load <64 x i8>, <64 x i8>* %0, align 1
   %2 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> %1)
   ret i8 %2
 }
 
entry:
   %4 = mul nsw <8 x i32> %3, %1
   %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
   ret i32 %5
 }
 
 define i32 @mlav16i32i32(i32* %x, i32* %y) {
 ; CHECK-LABEL: mlav16i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #32]
 ; CHECK-NEXT:    vmlav.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    add.w r0, r2, r12
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
+; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
+; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <16 x i32>*
   %1 = load <16 x i32>, <16 x i32>* %0, align 4
   %2 = bitcast i32* %y to <16 x i32>*
   %3 = load <16 x i32>, <16 x i32>* %2, align 4
   %4 = mul nsw <16 x i32> %3, %1
   %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
   ret i32 %5
 }
 
 define i32 @mlav24i32i32(i32* %x, i32* %y) {
 ; CHECK-LABEL: mlav24i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #80]
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #64]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    vmlav.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #32]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #32]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
-; CHECK-NEXT:    add.w r3, r2, r12
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #80]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #80]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    adds r0, r3, r2
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <8 x i32>*
   %1 = load <8 x i32>, <8 x i32>* %0, align 4
   %2 = bitcast i32* %y to <8 x i32>*
   %3 = load <8 x i32>, <8 x i32>* %2, align 4
   %4 = mul nsw <8 x i32> %3, %1
   %arrayidx.8 = getelementptr inbounds i32, i32* %x, i32 8
   %arrayidx1.8 = getelementptr inbounds i32, i32* %y, i32 8
   %5 = bitcast i32* %arrayidx.8 to <16 x i32>*
   %6 = load <16 x i32>, <16 x i32>* %5, align 4
   %7 = bitcast i32* %arrayidx1.8 to <16 x i32>*
   %8 = load <16 x i32>, <16 x i32>* %7, align 4
   %9 = mul nsw <16 x i32> %8, %6
   %10 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %9)
   %11 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
   %op.rdx = add nsw i32 %10, %11
   ret i32 %op.rdx
 }
 
 define i32 @mlav32i32i32(i32* %x, i32* %y) {
 ; CHECK-LABEL: mlav32i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #112]
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #80]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    vmlav.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #96]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #96]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #96]
-; CHECK-NEXT:    add.w r3, r2, r12
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #32]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #32]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #64]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    add.w r0, r2, r12
-; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #64]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #64]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #112]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #112]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <32 x i32>*
   %1 = load <32 x i32>, <32 x i32>* %0, align 4
   %2 = bitcast i32* %y to <32 x i32>*
   %3 = load <32 x i32>, <32 x i32>* %2, align 4
   %4 = mul nsw <32 x i32> %3, %1
   %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
entry:
   %6 = mul nsw <8 x i32> %5, %2
   %7 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %6)
   ret i32 %7
 }
 
 define i32 @mlav16i32i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav16i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #16]
 ; CHECK-NEXT:    vmlav.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1]
 ; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    add.w r0, r2, r12
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
+; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = sext <16 x i16> %1 to <16 x i32>
   %3 = bitcast i16* %y to <16 x i16>*
   %4 = load <16 x i16>, <16 x i16>* %3, align 2
   %5 = sext <16 x i16> %4 to <16 x i32>
   %6 = mul nsw <16 x i32> %5, %2
   %7 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %6)
   ret i32 %7
 }
 
 define i32 @mlav24i32i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav24i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #40]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #40]
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #32]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    vmlav.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #16]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #16]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #24]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #40]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #40]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r2]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1]
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    vmlava.s16 r2, q1, q0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    vmlava.s16 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <8 x i16>*
   %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = sext <8 x i16> %1 to <8 x i32>
   %3 = bitcast i16* %y to <8 x i16>*
   %4 = load <8 x i16>, <8 x i16>* %3, align 2
   %5 = sext <8 x i16> %4 to <8 x i32>
entry:
   %15 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %6)
   %op.rdx = add nsw i32 %14, %15
   ret i32 %op.rdx
 }
 
 define i32 @mlav32i32i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav32i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #56]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #56]
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #40]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #40]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    vmlav.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #8]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #48]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #48]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #48]
-; CHECK-NEXT:    add.w r3, r2, r12
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #16]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #16]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #32]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #24]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    add.w r0, r2, r12
-; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #32]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #32]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #56]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #56]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
   %2 = sext <32 x i16> %1 to <32 x i32>
   %3 = bitcast i16* %y to <32 x i16>*
   %4 = load <32 x i16>, <32 x i16>* %3, align 2
   %5 = sext <32 x i16> %4 to <32 x i32>
entry:
   %15 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %6)
   %op.rdx = add nuw nsw i32 %14, %15
   ret i32 %op.rdx
 }
 
 define i32 @mlav32i32i8(i8* %x, i8* %y) {
 ; CHECK-LABEL: mlav32i32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #28]
-; CHECK-NEXT:    vldrb.u32 q1, [r1, #28]
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #12]
-; CHECK-NEXT:    vldrb.u32 q1, [r1, #12]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #20]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #20]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
+; CHECK-NEXT:    mov r2, r0
+; CHECK-NEXT:    vmlav.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #4]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #4]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #24]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #24]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #24]
-; CHECK-NEXT:    add.w r3, r2, r12
-; CHECK-NEXT:    vmlav.u32 r12, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #8]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #8]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #8]
-; CHECK-NEXT:    vmlava.u32 r12, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #16]
-; CHECK-NEXT:    vldrb.u32 q1, [r1, #16]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #12]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #12]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
-; CHECK-NEXT:    add.w r0, r2, r12
-; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #16]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #16]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #28]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #28]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <32 x i8>*
   %1 = load <32 x i8>, <32 x i8>* %0, align 1
   %2 = zext <32 x i8> %1 to <32 x i32>
   %3 = bitcast i8* %y to <32 x i8>*
   %4 = load <32 x i8>, <32 x i8>* %3, align 1
   %5 = zext <32 x i8> %4 to <32 x i32>
entry:
   %11 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %4)
   %op.rdx = add i16 %10, %11
   ret i16 %op.rdx
 }
 
 define signext i16 @mlav32i16i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav32i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
-; CHECK-NEXT:    vldrh.u16 q1, [r1, #48]
-; CHECK-NEXT:    vmlav.u16 r12, q1, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
-; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
-; CHECK-NEXT:    vmlava.u16 r12, q1, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1, #32]
 ; CHECK-NEXT:    vmlav.u16 r2, q1, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1]
 ; CHECK-NEXT:    vmlava.u16 r2, q1, q0
-; CHECK-NEXT:    add.w r0, r2, r12
-; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
+; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
+; CHECK-NEXT:    vmlava.u16 r2, q1, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
+; CHECK-NEXT:    vldrh.u16 q1, [r1, #48]
+; CHECK-NEXT:    vmlava.u16 r2, q1, q0
+; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
   %2 = bitcast i16* %y to <32 x i16>*
   %3 = load <32 x i16>, <32 x i16>* %2, align 2
   %4 = mul <32 x i16> %3, %1
   %5 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %4)