[ARM] Distribute reductions based on ascending load offset

[dmgreen]

Summary

This distributes reductions based on the relative offset of loads, if one is found from their operands. Given chains of reductions this will then sort them in ascending load order, which in turn can help simple prefetches latch on to increasing strides more easily.

[samtebbs]

Very nice idea and looks good to me other than an possible change I thought of.

llvm/lib/Target/ARM/ARMISelLowering.cpp

13160

Do you think there's a way to combine the checking if each operand is before the other one into a single call? Perhaps IsKnownLoadBefore could be renamed to something like GetFirstLoad, return true if one is before the other and take X and N0 as reference arguments? It might have a performance benefit over running the check twice but may be unnecessary. Your call.

llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h

BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset,
       : Base(Base), Index(Index), Offset(Offset),
         IsIndexSignExt(IsIndexSignExt) {}
 
   SDValue getBase() { return Base; }
   SDValue getBase() const { return Base; }
   SDValue getIndex() { return Index; }
   SDValue getIndex() const { return Index; }
   bool hasValidOffset() const { return Offset.hasValue(); }
+  int64_t getOffset() const { return *Offset; }
 
   // Returns true if `Other` and `*this` are both some offset from the same base
   // pointer. In that case, `Off` is set to the offset between `*this` and
   // `Other` (negative if `Other` is before `*this`).
   bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG,
                       int64_t &Off) const;
 
   bool equalBaseIndex(const BaseIndexOffset &Other,

llvm/lib/Target/ARM/ARMISelLowering.cpp

 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/Attributes.h"
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
 
   // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
   if (N0.getNode()->hasOneUse())
     if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
       return Result;
   return SDValue();
 }
 
-static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
-                                   const ARMSubtarget *Subtarget) {
-  if (!Subtarget->hasMVEIntegerOps())
-    return SDValue();
-
+static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG) {
   EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDLoc dl(N);
 
   auto IsVecReduce = [](SDValue Op) {
     switch (Op.getOpcode()) {
     case ISD::VECREDUCE_ADD:
auto DistrubuteAddAddVecReduce = [&](SDValue N0, SDValue N1) {
     }
     return SDValue();
   };
   if (SDValue R = DistrubuteAddAddVecReduce(N0, N1))
     return R;
   if (SDValue R = DistrubuteAddAddVecReduce(N1, N0))
     return R;
 
+  // Distribute add(vecreduce(load(Y)), vecreduce(load(Z)))
+  // Or add(add(X, vecreduce(load(Y))), vecreduce(load(Z)))
+  // by ascending load offsets. This can help cores prefetch if the order of
+  // loads is more predictable.
+  auto DistrubuteVecReduceLoad = [&](SDValue N0, SDValue N1, bool IsForward) {
+    // Check if two reductions are known to load data where one is before/after
+    // another. Return negative if N0 loads data before N1, positive if N1 is
+    // before N0 and 0 otherwise if nothing is known.
+    auto IsKnownOrderedLoad = [&](SDValue N0, SDValue N1) {
+      // Look through to the first operand of a MUL, for the VMLA case.
+      // Currently only looks at the first operand, in the hope they are equal.
+      if (N0.getOpcode() == ISD::MUL)
+        N0 = N0.getOperand(0);
+      if (N1.getOpcode() == ISD::MUL)
+        N1 = N1.getOperand(0);
+
+      // Return true if the two operands are loads to the same object and the
+      // offset of the first is known to be less than the offset of the second.
+      LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
+      LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
+      if (!Load0 || !Load1 || Load0->getChain() != Load1->getChain() ||
+          !Load0->isSimple() || !Load1->isSimple() || Load0->isIndexed() ||
+          Load1->isIndexed())
+        return 0;
+
+      auto BaseLocDecomp0 = BaseIndexOffset::match(Load0, DAG);
+      auto BaseLocDecomp1 = BaseIndexOffset::match(Load1, DAG);
+
+      if (!BaseLocDecomp0.getBase() ||
+          BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
+          !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
+        return 0;
+      if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
+        return -1;
+      if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
+        return 1;
+      return 0;
+    };
+
+    SDValue X;
+    if (N0.getOpcode() == ISD::ADD) {
+      if (IsVecReduce(N0.getOperand(0)) && IsVecReduce(N0.getOperand(1))) {
+        int IsBefore = IsKnownOrderedLoad(N0.getOperand(0).getOperand(0),
+                                         N0.getOperand(1).getOperand(0));
+        if (IsBefore < 0) {
+          X = N0.getOperand(0);
+          N0 = N0.getOperand(1);
+        } else if (IsBefore > 0) {
+          X = N0.getOperand(1);
+          N0 = N0.getOperand(0);
+        } else
+          return SDValue();
+      } else if (IsVecReduce(N0.getOperand(0))) {
+        X = N0.getOperand(1);
+        N0 = N0.getOperand(0);
+      } else if (IsVecReduce(N0.getOperand(1))) {
+        X = N0.getOperand(0);
+        N0 = N0.getOperand(1);
+      } else
+        return SDValue();
+    } else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
+               IsKnownOrderedLoad(N0.getOperand(0), N1.getOperand(0)) < 0) {
+      // Note this is backward to how you would expect. We create
+      // add(reduce(load + 16), reduce(load + 0)) so that the
+      // add(reduce(load+16), X) is combined into VADDVA(X, load+16)), leaving
+      // the X as VADDV(load + 0)
+      return DAG.getNode(ISD::ADD, dl, VT, N1, N0);
+    } else
+      return SDValue();
+
+    if (!IsVecReduce(N0) || !IsVecReduce(N1))
+      return SDValue();
+
+    if (IsKnownOrderedLoad(N1.getOperand(0), N0.getOperand(0)) >= 0)
+      return SDValue();
+
+    // Switch from add(add(X, N0), N1) to add(add(X, N1), N0)
+    SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, X, N1);
+    return DAG.getNode(ISD::ADD, dl, VT, Add0, N0);
+  };
+  if (SDValue R = DistrubuteVecReduceLoad(N0, N1, true))
+    return R;
+  if (SDValue R = DistrubuteVecReduceLoad(N1, N0, false))
+    return R;
+  return SDValue();
+}
+
+static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
+                                   const ARMSubtarget *Subtarget) {
+  if (!Subtarget->hasMVEIntegerOps())
+    return SDValue();
+
+  if (SDValue R = TryDistrubutionADDVecReduce(N, DAG))
+    return R;
+
+  EVT VT = N->getValueType(0);
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  SDLoc dl(N);
+
   if (VT != MVT::i64)
     return SDValue();
 
   // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
   // will look like:
   //   t1: i32,i32 = ARMISD::VADDLVs x
   //   t2: i64 = build_pair t1, t1:1
   //   t3: i64 = add t2, y

llvm/test/CodeGen/Thumb2/mve-vecreduce-slp.ll

entry:
   %1 = load <4 x i32>, <4 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
   ret i32 %2
 }
 
 define i32 @addv8i32i32(i32* %x) {
 ; CHECK-LABEL: addv8i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u32 r0, q1
 ; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <8 x i32>*
   %1 = load <8 x i32>, <8 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
   ret i32 %2
 }
 
 define i32 @addv16i32i32(i32* %x) {
 ; CHECK-LABEL: addv16i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <16 x i32>*
   %1 = load <16 x i32>, <16 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
   ret i32 %2
 }
 
 define i32 @addv24i32i32(i32* %x) {
 ; CHECK-LABEL: addv24i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <8 x i32>*
   %1 = load <8 x i32>, <8 x i32>* %0, align 4
   %arrayidx.8 = getelementptr inbounds i32, i32* %x, i32 8
   %2 = bitcast i32* %arrayidx.8 to <16 x i32>*
   %3 = load <16 x i32>, <16 x i32>* %2, align 4
   %4 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %3)
   %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
   %op.rdx = add nsw i32 %4, %5
   ret i32 %op.rdx
 }
 
 define i32 @addv32i32i32(i32* %x) {
 ; CHECK-LABEL: addv32i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
 ; CHECK-NEXT:    mov r1, r0
 ; CHECK-NEXT:    vaddv.u32 r0, q1
 ; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r1, #96]
-; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r1, #32]
 ; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r1, #48]
 ; CHECK-NEXT:    vaddva.u32 r0, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r1, #64]
 ; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #80]
+; CHECK-NEXT:    vaddva.u32 r0, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r1, #96]
+; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r1, #112]
 ; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <32 x i32>*
   %1 = load <32 x i32>, <32 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
   ret i32 %2
 }
 
 define i32 @addv64i32i32(i32* %x) {
 ; CHECK-LABEL: addv64i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #176]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #208]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #224]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #96]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #112]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #160]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #144]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #160]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #176]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #192]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #208]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #224]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #240]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <64 x i32>*
   %1 = load <64 x i32>, <64 x i32>* %0, align 4
   %2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %1)
entry:
   %2 = sext <8 x i16> %1 to <8 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv16i32i16(i16* %x) {
 ; CHECK-LABEL: addv16i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #16]
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    vldrh.s32 q1, [r0]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = sext <16 x i16> %1 to <16 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv24i32i16(i16* %x) {
 ; CHECK-LABEL: addv24i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #16]
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    vldrh.s32 q1, [r0]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.s16 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = sext <16 x i16> %1 to <16 x i32>
   %arrayidx.16 = getelementptr inbounds i16, i16* %x, i32 16
   %3 = bitcast i16* %arrayidx.16 to <8 x i16>*
   %4 = load <8 x i16>, <8 x i16>* %3, align 2
   %5 = sext <8 x i16> %4 to <8 x i32>
   %6 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)
   %7 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %5)
   %op.rdx = add nsw i32 %6, %7
   ret i32 %op.rdx
 }
 
 define i32 @addv32i32i16(i16* %x) {
 ; CHECK-LABEL: addv32i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #40]
+; CHECK-NEXT:    vldrh.s32 q1, [r0]
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #48]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #40]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #48]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #56]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
   %2 = sext <32 x i16> %1 to <32 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv64i32i16(i16* %x) {
 ; CHECK-LABEL: addv64i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q1, [r0, #40]
+; CHECK-NEXT:    vldrh.s32 q1, [r0]
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
 ; CHECK-NEXT:    ldrsh.w r1, [r0, #120]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    ldrsh.w r3, [r0, #122]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #48]
-; CHECK-NEXT:    ldrsh.w r12, [r0, #124]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
+; CHECK-NEXT:    ldrsh.w r12, [r0, #124]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #64]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #40]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #80]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #56]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #64]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #72]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #56]
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #80]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #88]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #96]
 ; CHECK-NEXT:    vaddva.s16 r2, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #112]
 ; CHECK-NEXT:    ldrsh.w r0, [r0, #126]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
entry:
   %2 = zext <16 x i8> %1 to <16 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv24i32i8(i8* %x) {
 ; CHECK-LABEL: addv24i32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u16 q1, [r0, #16]
-; CHECK-NEXT:    vldrb.u8 q0, [r0]
-; CHECK-NEXT:    vaddv.u16 r0, q1
-; CHECK-NEXT:    vaddva.u8 r0, q0
+; CHECK-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-NEXT:    vldrb.u16 q0, [r0, #16]
+; CHECK-NEXT:    vaddv.u8 r0, q1
+; CHECK-NEXT:    vaddva.u16 r0, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <16 x i8>*
   %1 = load <16 x i8>, <16 x i8>* %0, align 1
   %2 = zext <16 x i8> %1 to <16 x i32>
   %arrayidx.16 = getelementptr inbounds i8, i8* %x, i32 16
   %3 = bitcast i8* %arrayidx.16 to <8 x i8>*
   %4 = load <8 x i8>, <8 x i8>* %3, align 1
   %5 = zext <8 x i8> %4 to <8 x i32>
   %6 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)
   %7 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %5)
   %op.rdx = add nuw nsw i32 %6, %7
   ret i32 %op.rdx
 }
 
 define i32 @addv32i32i8(i8* %x) {
 ; CHECK-LABEL: addv32i32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #20]
+; CHECK-NEXT:    vldrb.u32 q1, [r0]
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #24]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #8]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #12]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #20]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #24]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #28]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <32 x i8>*
   %1 = load <32 x i8>, <32 x i8>* %0, align 1
   %2 = zext <32 x i8> %1 to <32 x i32>
   %3 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %2)
   ret i32 %3
 }
 
 define i32 @addv64i32i8(i8* %x) {
 ; CHECK-LABEL: addv64i32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q1, [r0, #20]
+; CHECK-NEXT:    vldrb.u32 q1, [r0]
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #4]
 ; CHECK-NEXT:    ldrb.w r1, [r0, #60]
 ; CHECK-NEXT:    vaddv.u32 r2, q1
 ; CHECK-NEXT:    ldrb.w r3, [r0, #61]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #24]
-; CHECK-NEXT:    ldrb.w r12, [r0, #62]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #8]
+; CHECK-NEXT:    ldrb.w r12, [r0, #62]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #12]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r0]
-; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #20]
+; CHECK-NEXT:    vaddva.u32 r2, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r0, #24]
+; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #28]
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    vldrb.u8 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u8 r2, q0
 ; CHECK-NEXT:    vldrb.u16 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r0, #56]
 ; CHECK-NEXT:    ldrb.w r0, [r0, #63]
entry:
   %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
   ret i16 %2
 }
 
 define signext i16 @addv16i16i16(i16* %x) {
 ; CHECK-LABEL: addv16i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #16]
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u16 r0, q1
 ; CHECK-NEXT:    vaddva.u16 r0, q0
 ; CHECK-NEXT:    sxth r0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %1)
   ret i16 %2
 }
 
 define signext i16 @addv24i16i16(i16* %x) {
 ; CHECK-LABEL: addv24i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #32]
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u16 r2, q1
 ; CHECK-NEXT:    vaddva.u16 r2, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <8 x i16>*
   %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %arrayidx.8 = getelementptr inbounds i16, i16* %x, i32 8
   %2 = bitcast i16* %arrayidx.8 to <16 x i16>*
   %3 = load <16 x i16>, <16 x i16>* %2, align 2
   %4 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %3)
   %5 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
   %op.rdx = add i16 %4, %5
   ret i16 %op.rdx
 }
 
 define signext i16 @addv32i16i16(i16* %x) {
 ; CHECK-LABEL: addv32i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #32]
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u16 r2, q1
 ; CHECK-NEXT:    vaddva.u16 r2, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
   %2 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %1)
   ret i16 %2
 }
 
 define signext i16 @addv64i16i16(i16* %x) {
 ; CHECK-LABEL: addv64i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q1, [r0, #80]
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u16 r2, q1
 ; CHECK-NEXT:    vaddva.u16 r2, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #96]
-; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
-; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #64]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #80]
+; CHECK-NEXT:    vaddva.u16 r2, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #96]
+; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #112]
 ; CHECK-NEXT:    vaddva.u16 r2, q0
 ; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <64 x i16>*
   %1 = load <64 x i16>, <64 x i16>* %0, align 2
   %2 = call i16 @llvm.vector.reduce.add.v64i16(<64 x i16> %1)
entry:
   %5 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %1)
   %op.rdx = add i8 %4, %5
   ret i8 %op.rdx
 }
 
 define zeroext i8 @addv32i8i8(i8* %x) {
 ; CHECK-LABEL: addv32i8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u8 q1, [r0, #16]
-; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u8 r0, q1
 ; CHECK-NEXT:    vaddva.u8 r0, q0
 ; CHECK-NEXT:    uxtb r0, r0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <32 x i8>*
   %1 = load <32 x i8>, <32 x i8>* %0, align 1
   %2 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %1)
   ret i8 %2
 }
 
 define zeroext i8 @addv64i8i8(i8* %x) {
 ; CHECK-LABEL: addv64i8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u8 q1, [r0, #32]
-; CHECK-NEXT:    vldrb.u8 q0, [r0]
+; CHECK-NEXT:    vldrb.u8 q1, [r0]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #16]
 ; CHECK-NEXT:    vaddv.u8 r2, q1
 ; CHECK-NEXT:    vaddva.u8 r2, q0
-; CHECK-NEXT:    vldrb.u8 q0, [r0, #16]
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #32]
 ; CHECK-NEXT:    vaddva.u8 r2, q0
 ; CHECK-NEXT:    vldrb.u8 q0, [r0, #48]
 ; CHECK-NEXT:    vaddva.u8 r2, q0
 ; CHECK-NEXT:    uxtb r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <64 x i8>*
   %1 = load <64 x i8>, <64 x i8>* %0, align 1
entry:
   %4 = mul nsw <4 x i32> %3, %1
   %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
   ret i32 %5
 }
 
 define i32 @mlav8i32i32(i32* %x, i32* %y) {
 ; CHECK-LABEL: mlav8i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vmlav.u32 r2, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u32 r2, q1, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <8 x i32>*
   %1 = load <8 x i32>, <8 x i32>* %0, align 4
   %2 = bitcast i32* %y to <8 x i32>*
   %3 = load <8 x i32>, <8 x i32>* %2, align 4
   %4 = mul nsw <8 x i32> %3, %1
   %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
   ret i32 %5
 }
 
 define i32 @mlav16i32i32(i32* %x, i32* %y) {
 ; CHECK-LABEL: mlav16i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #32]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    vmlav.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #32]
+; CHECK-NEXT:    vmlava.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
 ; CHECK-NEXT:    vmlava.u32 r2, q1, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <16 x i32>*
   %1 = load <16 x i32>, <16 x i32>* %0, align 4
   %2 = bitcast i32* %y to <16 x i32>*
   %3 = load <16 x i32>, <16 x i32>* %2, align 4
   %4 = mul nsw <16 x i32> %3, %1
   %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
   ret i32 %5
 }
 
 define i32 @mlav24i32i32(i32* %x, i32* %y) {
 ; CHECK-LABEL: mlav24i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #64]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    vmlav.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #32]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #32]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #48]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #64]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #64]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #80]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #80]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r2, #16]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <8 x i32>*
   %1 = load <8 x i32>, <8 x i32>* %0, align 4
   %2 = bitcast i32* %y to <8 x i32>*
   %3 = load <8 x i32>, <8 x i32>* %2, align 4
   %4 = mul nsw <8 x i32> %3, %1
   %arrayidx.8 = getelementptr inbounds i32, i32* %x, i32 8
   %arrayidx1.8 = getelementptr inbounds i32, i32* %y, i32 8
   %5 = bitcast i32* %arrayidx.8 to <16 x i32>*
   %6 = load <16 x i32>, <16 x i32>* %5, align 4
   %7 = bitcast i32* %arrayidx1.8 to <16 x i32>*
   %8 = load <16 x i32>, <16 x i32>* %7, align 4
   %9 = mul nsw <16 x i32> %8, %6
   %10 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %9)
   %11 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
   %op.rdx = add nsw i32 %10, %11
   ret i32 %op.rdx
 }
 
 define i32 @mlav32i32i32(i32* %x, i32* %y) {
 ; CHECK-LABEL: mlav32i32i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    vmlav.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r2, #96]
-; CHECK-NEXT:    vldrw.u32 q1, [r1, #96]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #32]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #32]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #48]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #48]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #64]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #64]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #80]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #80]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r2, #96]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #96]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2, #112]
 ; CHECK-NEXT:    vldrw.u32 q1, [r1, #112]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i32* %x to <32 x i32>*
   %1 = load <32 x i32>, <32 x i32>* %0, align 4
   %2 = bitcast i32* %y to <32 x i32>*
entry:
   %6 = mul nsw <8 x i32> %5, %2
   %7 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %6)
   ret i32 %7
 }
 
 define i32 @mlav16i32i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav16i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #16]
-; CHECK-NEXT:    vmlav.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    vmlav.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
 ; CHECK-NEXT:    vmlava.u32 r2, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r0, #16]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #16]
+; CHECK-NEXT:    vmlava.u32 r2, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #24]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
 ; CHECK-NEXT:    vmlava.u32 r2, q1, q0
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = sext <16 x i16> %1 to <16 x i32>
   %3 = bitcast i16* %y to <16 x i16>*
   %4 = load <16 x i16>, <16 x i16>* %3, align 2
   %5 = sext <16 x i16> %4 to <16 x i32>
   %6 = mul nsw <16 x i32> %5, %2
   %7 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %6)
   ret i32 %7
 }
 
 define i32 @mlav24i32i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav24i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #32]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #32]
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
 ; CHECK-NEXT:    mov r2, r0
-; CHECK-NEXT:    vmlav.u32 r0, q1, q0
+; CHECK-NEXT:    vmlav.s16 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #16]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #24]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #32]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #32]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #40]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #40]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r2]
-; CHECK-NEXT:    vldrh.u16 q1, [r1]
-; CHECK-NEXT:    vmlava.s16 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <8 x i16>*
   %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = sext <8 x i16> %1 to <8 x i32>
   %3 = bitcast i16* %y to <8 x i16>*
   %4 = load <8 x i16>, <8 x i16>* %3, align 2
   %5 = sext <8 x i16> %4 to <8 x i32>
entry:
   %15 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %6)
   %op.rdx = add nsw i32 %14, %15
   ret i32 %op.rdx
 }
 
 define i32 @mlav32i32i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav32i32i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #40]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #40]
+; CHECK-NEXT:    vldrh.s32 q0, [r0]
+; CHECK-NEXT:    vldrh.s32 q1, [r1]
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    vmlav.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #8]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r2, #48]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #48]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #16]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #24]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #24]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrh.s32 q0, [r2]
-; CHECK-NEXT:    vldrh.s32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #32]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #32]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #40]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #40]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrh.s32 q0, [r2, #48]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #48]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrh.s32 q0, [r2, #56]
 ; CHECK-NEXT:    vldrh.s32 q1, [r1, #56]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
   %2 = sext <32 x i16> %1 to <32 x i32>
entry:
   %15 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %6)
   %op.rdx = add nuw nsw i32 %14, %15
   ret i32 %op.rdx
 }
 
 define i32 @mlav32i32i8(i8* %x, i8* %y) {
 ; CHECK-LABEL: mlav32i32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #20]
-; CHECK-NEXT:    vldrb.u32 q1, [r1, #20]
+; CHECK-NEXT:    vldrb.u32 q0, [r0]
+; CHECK-NEXT:    vldrb.u32 q1, [r1]
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    vmlav.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r2, #4]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #4]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r2, #24]
-; CHECK-NEXT:    vldrb.u32 q1, [r1, #24]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r2, #8]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #8]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r2, #12]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #12]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
-; CHECK-NEXT:    vldrb.u32 q0, [r2]
-; CHECK-NEXT:    vldrb.u32 q1, [r1]
-; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r2, #16]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #20]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #20]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
+; CHECK-NEXT:    vldrb.u32 q0, [r2, #24]
+; CHECK-NEXT:    vldrb.u32 q1, [r1, #24]
+; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    vldrb.u32 q0, [r2, #28]
 ; CHECK-NEXT:    vldrb.u32 q1, [r1, #28]
 ; CHECK-NEXT:    vmlava.u32 r0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <32 x i8>*
   %1 = load <32 x i8>, <32 x i8>* %0, align 1
   %2 = zext <32 x i8> %1 to <32 x i32>
entry:
   %4 = mul <8 x i16> %3, %1
   %5 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %4)
   ret i16 %5
 }
 
 define signext i16 @mlav16i16i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav16i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
-; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
-; CHECK-NEXT:    vmlav.u16 r2, q1, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vmlav.u16 r2, q1, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
+; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u16 r2, q1, q0
 ; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <16 x i16>*
   %1 = load <16 x i16>, <16 x i16>* %0, align 2
   %2 = bitcast i16* %y to <16 x i16>*
   %3 = load <16 x i16>, <16 x i16>* %2, align 2
   %4 = mul <16 x i16> %3, %1
   %5 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %4)
   ret i16 %5
 }
 
 define signext i16 @mlav24i16i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav24i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
-; CHECK-NEXT:    vldrh.u16 q1, [r1, #32]
+; CHECK-NEXT:    vldrh.u16 q0, [r0]
+; CHECK-NEXT:    vldrh.u16 q1, [r1]
 ; CHECK-NEXT:    vmlav.u16 r2, q1, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u16 r2, q1, q0
-; CHECK-NEXT:    vldrh.u16 q0, [r0]
-; CHECK-NEXT:    vldrh.u16 q1, [r1]
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
+; CHECK-NEXT:    vldrh.u16 q1, [r1, #32]
 ; CHECK-NEXT:    vmlava.u16 r2, q1, q0
 ; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <8 x i16>*
   %1 = load <8 x i16>, <8 x i16>* %0, align 2
   %2 = bitcast i16* %y to <8 x i16>*
   %3 = load <8 x i16>, <8 x i16>* %2, align 2
entry:
   %11 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %4)
   %op.rdx = add i16 %10, %11
   ret i16 %op.rdx
 }
 
 define signext i16 @mlav32i16i16(i16* %x, i16* %y) {
 ; CHECK-LABEL: mlav32i16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
-; CHECK-NEXT:    vldrh.u16 q1, [r1, #32]
-; CHECK-NEXT:    vmlav.u16 r2, q1, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1]
-; CHECK-NEXT:    vmlava.u16 r2, q1, q0
+; CHECK-NEXT:    vmlav.u16 r2, q1, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u16 r2, q1, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #32]
+; CHECK-NEXT:    vldrh.u16 q1, [r1, #32]
+; CHECK-NEXT:    vmlava.u16 r2, q1, q0
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #48]
 ; CHECK-NEXT:    vldrh.u16 q1, [r1, #48]
 ; CHECK-NEXT:    vmlava.u16 r2, q1, q0
 ; CHECK-NEXT:    sxth r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i16* %x to <32 x i16>*
   %1 = load <32 x i16>, <32 x i16>* %0, align 2
entry:
   %11 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %4)
   %op.rdx = add i8 %10, %11
   ret i8 %op.rdx
 }
 
 define zeroext i8 @mlav32i8i8(i8* %x, i8* %y) {
 ; CHECK-LABEL: mlav32i8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.u8 q0, [r0, #16]
-; CHECK-NEXT:    vldrb.u8 q1, [r1, #16]
-; CHECK-NEXT:    vmlav.u8 r2, q1, q0
 ; CHECK-NEXT:    vldrb.u8 q0, [r0]
 ; CHECK-NEXT:    vldrb.u8 q1, [r1]
+; CHECK-NEXT:    vmlav.u8 r2, q1, q0
+; CHECK-NEXT:    vldrb.u8 q0, [r0, #16]
+; CHECK-NEXT:    vldrb.u8 q1, [r1, #16]
 ; CHECK-NEXT:    vmlava.u8 r2, q1, q0
 ; CHECK-NEXT:    uxtb r0, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i8* %x to <32 x i8>*
   %1 = load <32 x i8>, <32 x i8>* %0, align 1
   %2 = bitcast i8* %y to <32 x i8>*
   %3 = load <32 x i8>, <32 x i8>* %2, align 1