[ARM] Lower v16i8 -> i64 VMLA reductions.

[dmgreen]

Summary

MVE does not have a VMLALV instruction that can perform v16i8 -> i64 reductions, like it does for v8i16->i64 and v4i32->i64 reductions. That means that the pattern to create them will be spilt up by type legalization, creating a lot of instructions.

This extends the patterns for matching i64 reductions a little to handle the v16i8->i64 case. We need to turn them into a pair of v8i16->i64 VMLALVs that each perform half of the reduction and are summed together (so the later is a VMLALVA). The order of the lanes does not matter for the reduction so we generate a MVEEXT for the extension, that will either be folded into a extending load or can be optimized to a VREV/VMOVL. Some of the resulting codegen isn't optimal, but will be improved in a later patch.

[SjoerdMeijer]

llvm/lib/Target/ARM/ARMISelLowering.cpp

16070

Perhaps a silly question on the use of 'illegal'. Type v16i8 isn't an illegal type, it is just not supported for these VMLALV instructions. Thus, I was wondering if 'unsupported' would be better, to avoid possible confusion with type legalization in general if that makes sense?

[dmgreen]

llvm/lib/Target/ARM/ARMISelLowering.cpp

16070

I think both class as legalization. You both legalize the types and legalize the operations. A v4i32 bitreverse is illegal, for example, even if the v4i32 type is legal. We need to turn it into a v16i8 bitreverse and a VREV32.

I don't have a super strong opinion if you think it's best to change it - I think the two words essentially mean the same thing here :) But illegal sounds fine to me.

[SjoerdMeijer]

LGTM

llvm/lib/Target/ARM/ARMISelLowering.cpp

16070

No, it's fine by me too, just started to wonder about it....

llvm/lib/Target/ARM/ARMISelLowering.cpp

if (!ST->hasMVEIntegerOps())
     return SDValue();
 
   assert(N->getOpcode() == ISD::VECREDUCE_ADD);
   EVT ResVT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDLoc dl(N);
 
   // We are looking for something that will have illegal types if left alone,
-  // but that we can convert to a single instruction undef MVE. For example
+  // but that we can convert to a single instruction under MVE. For example
   // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
   // or
   // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
 
-  // Cases:
+  // The legal cases are:
   //   VADDV u/s 8/16/32
   //   VMLAV u/s 8/16/32
   //   VADDLV u/s 32
   //   VMLALV u/s 16/32
 
   // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
   // extend it and use v4i32 instead.
   auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
if (A.getValueType() == B.getValueType() &&
                      [&A](MVT Ty) { return A.getValueType() == Ty; })) {
       A = ExtendIfNeeded(A, ExtendCode);
       B = ExtendIfNeeded(B, ExtendCode);
       return true;
     }
     return false;
   };
   auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
+    // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64
+    // reductions. The operands are extended with MVEEXT, but as they are
+    // reductions the lane orders do not matter. MVEEXT may be combined with
+    // loads to produce two extending loads, or else they will be expanded to
+    // VREV/VMOVL.
+    EVT VT = Ops[0].getValueType();
+    if (VT == MVT::v16i8) {
+      assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
+             "Unexpected illegal long reduction opcode");
+      bool IsUnsigned = Opcode == ARMISD::VMLALVu;
+
+      SDValue Ext0 =
+          DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
+                      DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
+      SDValue Ext1 =
+          DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
+                      DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
+
+      SDValue MLA0 = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
+                                 Ext0, Ext1);
+      SDValue MLA1 =
+          DAG.getNode(IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, dl,
+                      DAG.getVTList(MVT::i32, MVT::i32), MLA0, MLA0.getValue(1),
+                      Ext0.getValue(1), Ext1.getValue(1));
+      return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, MLA1, MLA1.getValue(1));
+    }
     SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
     return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node,
                        SDValue(Node.getNode(), 1));
   };
 
   if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
     return DAG.getNode(ARMISD::VADDVs, dl, ResVT, A);
   if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
                        DAG.getNode(ARMISD::VADDVpu, dl, MVT::i32, A, Mask));
 
   SDValue A, B;
   if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
     return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B);
   if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
     return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
   if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND,
-              {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
+              {MVT::v16i8, MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
     return Create64bitNode(ARMISD::VMLALVs, {A, B});
   if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND,
-              {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
+              {MVT::v16i8, MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
     return Create64bitNode(ARMISD::VMLALVu, {A, B});
   if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
     return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
                        DAG.getNode(ARMISD::VMLAVs, dl, MVT::i32, A, B));
   if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B))
     return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
                        DAG.getNode(ARMISD::VMLAVu, dl, MVT::i32, A, B));
 

llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll

entry:
   %m = mul <16 x i8> %x, %y
   %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %m)
   ret i8 %z
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: add_v16i8_v16i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.u8 r1, q1[0]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    vmov.i64 q2, #0xff
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r1
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov r1, s16
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umull r0, r12, r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    smlabb r0, r3, r2, r0
-; CHECK-NEXT:    vmov.u8 r2, q1[3]
-; CHECK-NEXT:    vmov.u8 r3, q1[2]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[3]
-; CHECK-NEXT:    vmov q4[2], q4[0], r1, r3
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmov r1, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umull r1, r2, r1, r2
-; CHECK-NEXT:    adds r0, r0, r1
-; CHECK-NEXT:    adc.w r1, r12, r2
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[5]
-; CHECK-NEXT:    vmov.u8 r3, q1[4]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[5]
-; CHECK-NEXT:    vmov.u8 r2, q0[4]
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[7]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[7]
-; CHECK-NEXT:    vmov.u8 r2, q0[6]
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[9]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[8]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[9]
-; CHECK-NEXT:    vmov.u8 r2, q0[8]
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[11]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[10]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[11]
-; CHECK-NEXT:    vmov.u8 r2, q0[10]
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[13]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[12]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[13]
-; CHECK-NEXT:    vmov.u8 r2, q0[12]
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[15]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[14]
-; CHECK-NEXT:    vmov q1[2], q1[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[15]
-; CHECK-NEXT:    vmov.u8 r2, q0[14]
-; CHECK-NEXT:    vand q1, q1, q2
-; CHECK-NEXT:    vmov q0[2], q0[0], r2, r3
-; CHECK-NEXT:    vmov r12, s4
-; CHECK-NEXT:    vand q0, q0, q2
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s6
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    add r2, sp, #16
+; CHECK-NEXT:    mov r3, sp
+; CHECK-NEXT:    vstrw.32 q1, [r2]
+; CHECK-NEXT:    vstrw.32 q0, [r3]
+; CHECK-NEXT:    vldrb.u16 q0, [r2]
+; CHECK-NEXT:    vldrb.u16 q1, [r3]
+; CHECK-NEXT:    vmlalv.u16 r0, r1, q1, q0
+; CHECK-NEXT:    vldrb.u16 q0, [r2, #8]
+; CHECK-NEXT:    vldrb.u16 q1, [r3, #8]
+; CHECK-NEXT:    vmlalva.u16 r0, r1, q1, q0
+; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    bx lr
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
   %yy = zext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
   ret i64 %z
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: add_v16i8_v16i64_sext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.s8 r0, q1[1]
-; CHECK-NEXT:    vmov.s8 r1, q0[1]
-; CHECK-NEXT:    smull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.s8 r2, q1[0]
-; CHECK-NEXT:    vmov.s8 r3, q0[0]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[2]
-; CHECK-NEXT:    vmov.s8 r3, q0[2]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[3]
-; CHECK-NEXT:    vmov.s8 r3, q0[3]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[4]
-; CHECK-NEXT:    vmov.s8 r3, q0[4]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[5]
-; CHECK-NEXT:    vmov.s8 r3, q0[5]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[6]
-; CHECK-NEXT:    vmov.s8 r3, q0[6]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[7]
-; CHECK-NEXT:    vmov.s8 r3, q0[7]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[8]
-; CHECK-NEXT:    vmov.s8 r3, q0[8]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[9]
-; CHECK-NEXT:    vmov.s8 r3, q0[9]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[10]
-; CHECK-NEXT:    vmov.s8 r3, q0[10]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[11]
-; CHECK-NEXT:    vmov.s8 r3, q0[11]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[12]
-; CHECK-NEXT:    vmov.s8 r3, q0[12]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[13]
-; CHECK-NEXT:    vmov.s8 r3, q0[13]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[14]
-; CHECK-NEXT:    vmov.s8 r3, q0[14]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q1[15]
-; CHECK-NEXT:    vmov.s8 r3, q0[15]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    add r2, sp, #16
+; CHECK-NEXT:    mov r3, sp
+; CHECK-NEXT:    vstrw.32 q1, [r2]
+; CHECK-NEXT:    vstrw.32 q0, [r3]
+; CHECK-NEXT:    vldrb.s16 q0, [r2]
+; CHECK-NEXT:    vldrb.s16 q1, [r3]
+; CHECK-NEXT:    vmlalv.s16 r0, r1, q1, q0
+; CHECK-NEXT:    vldrb.s16 q0, [r2, #8]
+; CHECK-NEXT:    vldrb.s16 q1, [r3, #8]
+; CHECK-NEXT:    vmlalva.s16 r0, r1, q1, q0
+; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    bx lr
 entry:
   %xx = sext <16 x i8> %x to <16 x i64>
   %yy = sext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
   ret i64 %z
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext_load(<16 x i8> *%xp, <16 x i8> *%yp) {
 ; CHECK-LABEL: add_v16i8_v16i64_zext_load:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmov.i64 q0, #0xff
-; CHECK-NEXT:    vmov.u8 r1, q1[1]
-; CHECK-NEXT:    vmov.u8 r2, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r2, r1
-; CHECK-NEXT:    vand q3, q2, q0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    vmov.u8 r0, q2[1]
-; CHECK-NEXT:    vmov.u8 r2, q2[0]
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r0
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umull r0, r12, r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q2[2]
-; CHECK-NEXT:    smlabb r0, r3, r2, r0
-; CHECK-NEXT:    vmov.u8 r2, q1[3]
-; CHECK-NEXT:    vmov.u8 r3, q1[2]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q2[3]
-; CHECK-NEXT:    vmov q4[2], q4[0], r1, r3
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmov r1, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umull r1, r2, r1, r2
-; CHECK-NEXT:    adds r0, r0, r1
-; CHECK-NEXT:    adc.w r1, r12, r2
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[5]
-; CHECK-NEXT:    vmov.u8 r3, q1[4]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q2[5]
-; CHECK-NEXT:    vmov.u8 r2, q2[4]
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[7]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q2[7]
-; CHECK-NEXT:    vmov.u8 r2, q2[6]
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[9]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[8]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q2[9]
-; CHECK-NEXT:    vmov.u8 r2, q2[8]
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[11]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[10]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q2[11]
-; CHECK-NEXT:    vmov.u8 r2, q2[10]
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[13]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[12]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q2[13]
-; CHECK-NEXT:    vmov.u8 r2, q2[12]
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    vmov.u8 r2, q1[15]
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[14]
-; CHECK-NEXT:    vmov q1[2], q1[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q2[15]
-; CHECK-NEXT:    vmov.u8 r2, q2[14]
-; CHECK-NEXT:    vand q1, q1, q0
-; CHECK-NEXT:    vmov q2[2], q2[0], r2, r3
-; CHECK-NEXT:    vmov r12, s4
-; CHECK-NEXT:    vand q0, q2, q0
-; CHECK-NEXT:    vmov r2, s0
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    umlal r0, r1, r2, r12
-; CHECK-NEXT:    vmov r2, s6
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q1, [r0]
+; CHECK-NEXT:    vmlalv.u16 r2, r3, q1, q0
+; CHECK-NEXT:    vldrb.u16 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.u16 q1, [r0, #8]
+; CHECK-NEXT:    vmlalva.u16 r2, r3, q1, q0
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    bx lr
 entry:
   %x = load <16 x i8>, <16 x i8>* %xp
   %y = load <16 x i8>, <16 x i8>* %yp
   %xx = zext <16 x i8> %x to <16 x i64>
   %yy = zext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
   ret i64 %z
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext_load(<16 x i8> *%xp, <16 x i8> *%yp) {
 ; CHECK-LABEL: add_v16i8_v16i64_sext_load:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vmov.s8 r1, q0[1]
-; CHECK-NEXT:    vmov.s8 r0, q1[1]
-; CHECK-NEXT:    smull r0, r1, r0, r1
-; CHECK-NEXT:    vmov.s8 r2, q0[0]
-; CHECK-NEXT:    vmov.s8 r3, q1[0]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[2]
-; CHECK-NEXT:    vmov.s8 r3, q1[2]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[3]
-; CHECK-NEXT:    vmov.s8 r3, q1[3]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[4]
-; CHECK-NEXT:    vmov.s8 r3, q1[4]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[5]
-; CHECK-NEXT:    vmov.s8 r3, q1[5]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[6]
-; CHECK-NEXT:    vmov.s8 r3, q1[6]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[7]
-; CHECK-NEXT:    vmov.s8 r3, q1[7]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[8]
-; CHECK-NEXT:    vmov.s8 r3, q1[8]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[9]
-; CHECK-NEXT:    vmov.s8 r3, q1[9]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[10]
-; CHECK-NEXT:    vmov.s8 r3, q1[10]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[11]
-; CHECK-NEXT:    vmov.s8 r3, q1[11]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[12]
-; CHECK-NEXT:    vmov.s8 r3, q1[12]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[13]
-; CHECK-NEXT:    vmov.s8 r3, q1[13]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[14]
-; CHECK-NEXT:    vmov.s8 r3, q1[14]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
-; CHECK-NEXT:    vmov.s8 r2, q0[15]
-; CHECK-NEXT:    vmov.s8 r3, q1[15]
-; CHECK-NEXT:    smlal r0, r1, r3, r2
+; CHECK-NEXT:    vldrb.s16 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q1, [r0]
+; CHECK-NEXT:    vmlalv.s16 r2, r3, q1, q0
+; CHECK-NEXT:    vldrb.s16 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.s16 q1, [r0, #8]
+; CHECK-NEXT:    vmlalva.s16 r2, r3, q1, q0
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    bx lr
 entry:
   %x = load <16 x i8>, <16 x i8>* %xp
   %y = load <16 x i8>, <16 x i8>* %yp
   %xx = sext <16 x i8> %x to <16 x i64>
   %yy = sext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
entry:
   %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %m)
   %r = add i8 %z, %a
   ret i8 %r
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y, i64 %a) {
 ; CHECK-LABEL: add_v16i8_v16i64_acc_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.u8 r2, q1[1]
-; CHECK-NEXT:    vmov.u8 r3, q1[0]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
-; CHECK-NEXT:    vmov.i64 q2, #0xff
-; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r2, s16
-; CHECK-NEXT:    vmov.u8 r4, q0[2]
-; CHECK-NEXT:    vmov r3, s14
-; CHECK-NEXT:    umull lr, r12, r2, r12
-; CHECK-NEXT:    vmov r2, s18
-; CHECK-NEXT:    smlabb lr, r2, r3, lr
-; CHECK-NEXT:    vmov.u8 r3, q1[3]
-; CHECK-NEXT:    vmov.u8 r2, q1[2]
-; CHECK-NEXT:    vmov q3[2], q3[0], r2, r3
-; CHECK-NEXT:    vmov.u8 r3, q0[3]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, r3
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vmov r3, s16
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    adds.w r2, r2, lr
-; CHECK-NEXT:    vmov.u8 lr, q0[5]
-; CHECK-NEXT:    adc.w r3, r3, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    umlal r2, r3, r4, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[5]
-; CHECK-NEXT:    vmov.u8 r4, q1[4]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q0[4]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q0[7]
-; CHECK-NEXT:    umlal r2, r3, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r2, r2, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[6]
-; CHECK-NEXT:    adc.w r3, r3, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[7]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q0[6]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q0[9]
-; CHECK-NEXT:    umlal r2, r3, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r2, r2, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[8]
-; CHECK-NEXT:    adc.w r3, r3, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[9]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q0[8]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q0[11]
-; CHECK-NEXT:    umlal r2, r3, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r2, r2, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[10]
-; CHECK-NEXT:    adc.w r3, r3, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[11]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q0[10]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q0[13]
-; CHECK-NEXT:    umlal r2, r3, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r2, r2, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[12]
-; CHECK-NEXT:    adc.w r3, r3, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[13]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q0[12]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q0[15]
-; CHECK-NEXT:    umlal r2, r3, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r2, r2, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[14]
-; CHECK-NEXT:    adc.w r3, r3, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[15]
-; CHECK-NEXT:    vmov q1[2], q1[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q0[14]
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, lr
-; CHECK-NEXT:    vand q1, q1, q2
-; CHECK-NEXT:    vand q0, q0, q2
-; CHECK-NEXT:    vmov r12, s4
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    umlal r2, r3, r4, r12
-; CHECK-NEXT:    vmov r12, s6
-; CHECK-NEXT:    vmov r4, s2
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r2, r2, r4
-; CHECK-NEXT:    adc.w r3, r3, r12
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    add r4, sp, #16
+; CHECK-NEXT:    mov r3, sp
+; CHECK-NEXT:    vstrw.32 q1, [r4]
+; CHECK-NEXT:    vstrw.32 q0, [r3]
+; CHECK-NEXT:    vldrb.u16 q0, [r4]
+; CHECK-NEXT:    vldrb.u16 q1, [r3]
+; CHECK-NEXT:    vmlalv.u16 r2, r5, q1, q0
+; CHECK-NEXT:    vldrb.u16 q0, [r4, #8]
+; CHECK-NEXT:    vldrb.u16 q1, [r3, #8]
+; CHECK-NEXT:    vmlalva.u16 r2, r5, q1, q0
 ; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    adcs r1, r5
+; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
   %yy = zext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
   %r = add i64 %z, %a
   ret i64 %r
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, <16 x i8> %y, i64 %a) {
 ; CHECK-LABEL: add_v16i8_v16i64_acc_sext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    vmov.s8 r2, q1[1]
-; CHECK-NEXT:    vmov.s8 r3, q0[1]
-; CHECK-NEXT:    smull r3, lr, r3, r2
-; CHECK-NEXT:    vmov.s8 r12, q1[0]
-; CHECK-NEXT:    vmov.s8 r2, q0[0]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[2]
-; CHECK-NEXT:    vmov.s8 r2, q0[2]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[3]
-; CHECK-NEXT:    vmov.s8 r2, q0[3]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[4]
-; CHECK-NEXT:    vmov.s8 r2, q0[4]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[5]
-; CHECK-NEXT:    vmov.s8 r2, q0[5]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[6]
-; CHECK-NEXT:    vmov.s8 r2, q0[6]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[7]
-; CHECK-NEXT:    vmov.s8 r2, q0[7]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[8]
-; CHECK-NEXT:    vmov.s8 r2, q0[8]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[9]
-; CHECK-NEXT:    vmov.s8 r2, q0[9]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[10]
-; CHECK-NEXT:    vmov.s8 r2, q0[10]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[11]
-; CHECK-NEXT:    vmov.s8 r2, q0[11]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[12]
-; CHECK-NEXT:    vmov.s8 r2, q0[12]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[13]
-; CHECK-NEXT:    vmov.s8 r2, q0[13]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[14]
-; CHECK-NEXT:    vmov.s8 r2, q0[14]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    vmov.s8 r12, q1[15]
-; CHECK-NEXT:    vmov.s8 r2, q0[15]
-; CHECK-NEXT:    smlal r3, lr, r2, r12
-; CHECK-NEXT:    adds r0, r0, r3
-; CHECK-NEXT:    adc.w r1, r1, lr
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    add r4, sp, #16
+; CHECK-NEXT:    mov r3, sp
+; CHECK-NEXT:    vstrw.32 q1, [r4]
+; CHECK-NEXT:    vstrw.32 q0, [r3]
+; CHECK-NEXT:    vldrb.s16 q0, [r4]
+; CHECK-NEXT:    vldrb.s16 q1, [r3]
+; CHECK-NEXT:    vmlalv.s16 r2, r5, q1, q0
+; CHECK-NEXT:    vldrb.s16 q0, [r4, #8]
+; CHECK-NEXT:    vldrb.s16 q1, [r3, #8]
+; CHECK-NEXT:    vmlalva.s16 r2, r5, q1, q0
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r5
+; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %xx = sext <16 x i8> %x to <16 x i64>
   %yy = sext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
   %r = add i64 %z, %a
   ret i64 %r
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext_load(<16 x i8> *%xp, <16 x i8> *%yp, i64 %a) {
 ; CHECK-LABEL: add_v16i8_v16i64_acc_zext_load:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmov.i64 q0, #0xff
-; CHECK-NEXT:    vmov.u8 r12, q1[1]
-; CHECK-NEXT:    vmov.u8 r1, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r12
-; CHECK-NEXT:    vand q3, q2, q0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov.u8 r0, q2[1]
-; CHECK-NEXT:    vmov.u8 r1, q2[0]
-; CHECK-NEXT:    vmov q4[2], q4[0], r1, r0
-; CHECK-NEXT:    vmov r1, s14
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov.u8 r4, q2[2]
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    umull lr, r12, r0, r12
-; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    smlabb lr, r0, r1, lr
-; CHECK-NEXT:    vmov.u8 r1, q1[3]
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov q3[2], q3[0], r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q2[3]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, r1
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov r1, s16
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r0, r1, r1, r0
-; CHECK-NEXT:    adds.w r0, r0, lr
-; CHECK-NEXT:    vmov.u8 lr, q2[5]
-; CHECK-NEXT:    adc.w r1, r1, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    umlal r0, r1, r4, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[5]
-; CHECK-NEXT:    vmov.u8 r4, q1[4]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q2[4]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q2[7]
-; CHECK-NEXT:    umlal r0, r1, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r0, r0, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[6]
-; CHECK-NEXT:    adc.w r1, r1, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[7]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q2[6]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q2[9]
-; CHECK-NEXT:    umlal r0, r1, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r0, r0, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[8]
-; CHECK-NEXT:    adc.w r1, r1, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[9]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q2[8]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q2[11]
-; CHECK-NEXT:    umlal r0, r1, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r0, r0, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[10]
-; CHECK-NEXT:    adc.w r1, r1, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[11]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q2[10]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q2[13]
-; CHECK-NEXT:    umlal r0, r1, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r0, r0, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[12]
-; CHECK-NEXT:    adc.w r1, r1, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[13]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q2[12]
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q4, q4, q0
-; CHECK-NEXT:    vmov r12, s12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.u8 lr, q2[15]
-; CHECK-NEXT:    umlal r0, r1, r4, r12
-; CHECK-NEXT:    vmov r12, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r0, r0, r4
-; CHECK-NEXT:    vmov.u8 r4, q1[14]
-; CHECK-NEXT:    adc.w r1, r1, r12
-; CHECK-NEXT:    vmov.u8 r12, q1[15]
-; CHECK-NEXT:    vmov q1[2], q1[0], r4, r12
-; CHECK-NEXT:    vmov.u8 r4, q2[14]
-; CHECK-NEXT:    vmov q2[2], q2[0], r4, lr
-; CHECK-NEXT:    vand q1, q1, q0
-; CHECK-NEXT:    vand q0, q2, q0
-; CHECK-NEXT:    vmov r12, s4
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    umlal r0, r1, r4, r12
-; CHECK-NEXT:    vmov r12, s6
-; CHECK-NEXT:    vmov r4, s2
-; CHECK-NEXT:    umull r4, r12, r4, r12
-; CHECK-NEXT:    adds r0, r0, r4
-; CHECK-NEXT:    adc.w r1, r1, r12
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .save {r5, lr}
+; CHECK-NEXT:    push {r5, lr}
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q1, [r0]
+; CHECK-NEXT:    vmlalv.u16 r12, r5, q1, q0
+; CHECK-NEXT:    vldrb.u16 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.u16 q1, [r0, #8]
+; CHECK-NEXT:    vmlalva.u16 r12, r5, q1, q0
+; CHECK-NEXT:    adds.w r0, r12, r2
+; CHECK-NEXT:    adc.w r1, r5, r3
+; CHECK-NEXT:    pop {r5, pc}
 entry:
   %x = load <16 x i8>, <16 x i8>* %xp
   %y = load <16 x i8>, <16 x i8>* %yp
   %xx = zext <16 x i8> %x to <16 x i64>
   %yy = zext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
   %r = add i64 %z, %a
   ret i64 %r
 }
 
 define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext_load(<16 x i8> *%xp, <16 x i8> *%yp, i64 %a) {
 ; CHECK-LABEL: add_v16i8_v16i64_acc_sext_load:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vmov.s8 r1, q0[1]
-; CHECK-NEXT:    vmov.s8 r0, q1[1]
-; CHECK-NEXT:    smull r0, lr, r0, r1
-; CHECK-NEXT:    vmov.s8 r12, q0[0]
-; CHECK-NEXT:    vmov.s8 r1, q1[0]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[2]
-; CHECK-NEXT:    vmov.s8 r1, q1[2]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[3]
-; CHECK-NEXT:    vmov.s8 r1, q1[3]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[4]
-; CHECK-NEXT:    vmov.s8 r1, q1[4]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[5]
-; CHECK-NEXT:    vmov.s8 r1, q1[5]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[6]
-; CHECK-NEXT:    vmov.s8 r1, q1[6]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[7]
-; CHECK-NEXT:    vmov.s8 r1, q1[7]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[8]
-; CHECK-NEXT:    vmov.s8 r1, q1[8]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[9]
-; CHECK-NEXT:    vmov.s8 r1, q1[9]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[10]
-; CHECK-NEXT:    vmov.s8 r1, q1[10]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[11]
-; CHECK-NEXT:    vmov.s8 r1, q1[11]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[12]
-; CHECK-NEXT:    vmov.s8 r1, q1[12]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[13]
-; CHECK-NEXT:    vmov.s8 r1, q1[13]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[14]
-; CHECK-NEXT:    vmov.s8 r1, q1[14]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    vmov.s8 r12, q0[15]
-; CHECK-NEXT:    vmov.s8 r1, q1[15]
-; CHECK-NEXT:    smlal r0, lr, r1, r12
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r1, lr, r3
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .save {r5, lr}
+; CHECK-NEXT:    push {r5, lr}
+; CHECK-NEXT:    vldrb.s16 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q1, [r0]
+; CHECK-NEXT:    vmlalv.s16 r12, r5, q1, q0
+; CHECK-NEXT:    vldrb.s16 q0, [r1, #8]
+; CHECK-NEXT:    vldrb.s16 q1, [r0, #8]
+; CHECK-NEXT:    vmlalva.s16 r12, r5, q1, q0
+; CHECK-NEXT:    adds.w r0, r12, r2
+; CHECK-NEXT:    adc.w r1, r5, r3
+; CHECK-NEXT:    pop {r5, pc}
 entry:
   %x = load <16 x i8>, <16 x i8>* %xp
   %y = load <16 x i8>, <16 x i8>* %yp
   %xx = sext <16 x i8> %x to <16 x i64>
   %yy = sext <16 x i8> %y to <16 x i64>
   %m = mul <16 x i64> %xx, %yy
   %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)
   %r = add i64 %z, %a