[ARM] MVE hadd and rhadd

[dmgreen]

Summary

This uses the nodes from D106237 to add MVE HADD and RHADD lowering.

[samtebbs]

Nice one

llvm/lib/Target/ARM/ARMISelLowering.cpp

for (auto VT : IntTypes) {
     setOperationAction(ISD::BITREVERSE, VT, Legal);
     setOperationAction(ISD::BSWAP, VT, Legal);
     setOperationAction(ISD::SADDSAT, VT, Legal);
     setOperationAction(ISD::UADDSAT, VT, Legal);
     setOperationAction(ISD::SSUBSAT, VT, Legal);
     setOperationAction(ISD::USUBSAT, VT, Legal);
     setOperationAction(ISD::ABDS, VT, Legal);
     setOperationAction(ISD::ABDU, VT, Legal);
+    setOperationAction(ISD::AVGFLOORS, VT, Legal);
+    setOperationAction(ISD::AVGFLOORU, VT, Legal);
+    setOperationAction(ISD::AVGCEILS, VT, Legal);
+    setOperationAction(ISD::AVGCEILU, VT, Legal);
 
     // No native support for these.
     setOperationAction(ISD::UDIV, VT, Expand);
     setOperationAction(ISD::SDIV, VT, Expand);
     setOperationAction(ISD::UREM, VT, Expand);
     setOperationAction(ISD::SREM, VT, Expand);
     setOperationAction(ISD::UDIVREM, VT, Expand);
     setOperationAction(ISD::SDIVREM, VT, Expand);
static SDValue FlattenVectorShuffle(ShuffleVectorSDNode *N, SelectionDAG &DAG) {
   // Looking for binary operators that will have been folded from
   // truncates/extends.
   switch (Op.getOpcode()) {
   case ARMISD::VQDMULH:
   case ISD::MULHS:
   case ISD::MULHU:
   case ISD::ABDS:
   case ISD::ABDU:
+  case ISD::AVGFLOORS:
+  case ISD::AVGFLOORU:
+  case ISD::AVGCEILS:
+  case ISD::AVGCEILU:
     break;
   default:
     return SDValue();
   }
 
   ShuffleVectorSDNode *Op0 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(0));
   ShuffleVectorSDNode *Op1 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(1));
   if (!Op0 || !Op1 || !Op0->getOperand(1).isUndef() ||

llvm/lib/Target/ARM/ARMInstrMVE.td

def subnuw : PatFrag<(ops node:$lhs, node:$rhs),
   return N->getFlags().hasNoUnsignedWrap();
 }]>;
 
 def subnsw : PatFrag<(ops node:$lhs, node:$rhs),
                      (sub node:$lhs, node:$rhs), [{
   return N->getFlags().hasNoSignedWrap();
 }]>;
 
-multiclass MVE_VRHADD_m<MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int> {
+multiclass MVE_VRHADD_m<MVEVectorVTInfo VTI, SDNode Op,
+                      SDNode unpred_op, Intrinsic PredInt> {
   def "" : MVE_VRHADD_Base<VTI.Suffix, VTI.Unsigned, VTI.Size>;
   defvar Inst = !cast<Instruction>(NAME);
+  defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
 
   let Predicates = [HasMVEInt] in {
-    // Unpredicated rounding add-with-divide-by-two
+    // Unpredicated rounding add-with-divide-by-two intrinsic
     def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
                             (i32 VTI.Unsigned))),
               (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-
-    // Predicated add-with-divide-by-two
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
-                            (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
-                             (VTI.Vec MQPR:$inactive)))>;
   }
 }
 
-multiclass MVE_VRHADD<MVEVectorVTInfo VTI>
-  : MVE_VRHADD_m<VTI, int_arm_mve_vrhadd, int_arm_mve_rhadd_predicated>;
+multiclass MVE_VRHADD<MVEVectorVTInfo VTI, SDNode rhadd>
+  : MVE_VRHADD_m<VTI, rhadd, int_arm_mve_vrhadd, int_arm_mve_rhadd_predicated>;
 
-defm MVE_VRHADDs8  : MVE_VRHADD<MVE_v16s8>;
-defm MVE_VRHADDs16 : MVE_VRHADD<MVE_v8s16>;
-defm MVE_VRHADDs32 : MVE_VRHADD<MVE_v4s32>;
-defm MVE_VRHADDu8  : MVE_VRHADD<MVE_v16u8>;
-defm MVE_VRHADDu16 : MVE_VRHADD<MVE_v8u16>;
-defm MVE_VRHADDu32 : MVE_VRHADD<MVE_v4u32>;
+defm MVE_VRHADDs8  : MVE_VRHADD<MVE_v16s8, avgceils>;
+defm MVE_VRHADDs16 : MVE_VRHADD<MVE_v8s16, avgceils>;
+defm MVE_VRHADDs32 : MVE_VRHADD<MVE_v4s32, avgceils>;
+defm MVE_VRHADDu8  : MVE_VRHADD<MVE_v16u8, avgceilu>;
+defm MVE_VRHADDu16 : MVE_VRHADD<MVE_v8u16, avgceilu>;
+defm MVE_VRHADDu32 : MVE_VRHADD<MVE_v4u32, avgceilu>;
 
 // Rounding Halving Add perform the arithemtic operation with an extra bit of
 // precision, before performing the shift, to void clipping errors. We're not
 // modelling that here with these patterns, but we're using no wrap forms of
 // add to ensure that the extra bit of information is not needed for the
 // arithmetic or the rounding.
 let Predicates = [HasMVEInt] in {
   def : Pat<(v16i8 (ARMvshrsImm (addnsw (addnsw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),

 
 class MVE_VHADD_<string suffix, bit U, bits<2> size,
               list<dag> pattern=[]>
   : MVE_VHADDSUB<"vhadd", suffix, U, 0b0, size, pattern>;
 class MVE_VHSUB_<string suffix, bit U, bits<2> size,
               list<dag> pattern=[]>
   : MVE_VHADDSUB<"vhsub", suffix, U, 0b1, size, pattern>;
 
-multiclass MVE_VHADD_m<MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int, PatFrag add_op,
+multiclass MVE_VHADD_m<MVEVectorVTInfo VTI, SDNode Op,
+                      SDNode unpred_op, Intrinsic PredInt, PatFrag add_op,
                       SDNode shift_op> {
   def "" : MVE_VHADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
   defvar Inst = !cast<Instruction>(NAME);
+  defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
 
   let Predicates = [HasMVEInt] in {
     // Unpredicated add-and-divide-by-two
     def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned))),
               (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
 
     def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
               (Inst MQPR:$Qm, MQPR:$Qn)>;
-
-    // Predicated add-and-divide-by-two
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned),
-                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
-                             (VTI.Vec MQPR:$inactive)))>;
   }
 }
 
-multiclass MVE_VHADD<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op>
-  : MVE_VHADD_m<VTI, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, add_op,
+multiclass MVE_VHADD<MVEVectorVTInfo VTI, SDNode Op, PatFrag add_op, SDNode shift_op>
+  : MVE_VHADD_m<VTI, Op, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, add_op,
                 shift_op>;
 
 // Halving add/sub perform the arithemtic operation with an extra bit of
 // precision, before performing the shift, to void clipping errors. We're not
 // modelling that here with these patterns, but we're using no wrap forms of
 // add/sub to ensure that the extra bit of information is not needed.
-defm MVE_VHADDs8  : MVE_VHADD<MVE_v16s8, addnsw, ARMvshrsImm>;
-defm MVE_VHADDs16 : MVE_VHADD<MVE_v8s16, addnsw, ARMvshrsImm>;
-defm MVE_VHADDs32 : MVE_VHADD<MVE_v4s32, addnsw, ARMvshrsImm>;
-defm MVE_VHADDu8  : MVE_VHADD<MVE_v16u8, addnuw, ARMvshruImm>;
-defm MVE_VHADDu16 : MVE_VHADD<MVE_v8u16, addnuw, ARMvshruImm>;
-defm MVE_VHADDu32 : MVE_VHADD<MVE_v4u32, addnuw, ARMvshruImm>;
+defm MVE_VHADDs8  : MVE_VHADD<MVE_v16s8, avgfloors, addnsw, ARMvshrsImm>;
+defm MVE_VHADDs16 : MVE_VHADD<MVE_v8s16, avgfloors, addnsw, ARMvshrsImm>;
+defm MVE_VHADDs32 : MVE_VHADD<MVE_v4s32, avgfloors, addnsw, ARMvshrsImm>;
+defm MVE_VHADDu8  : MVE_VHADD<MVE_v16u8, avgflooru, addnuw, ARMvshruImm>;
+defm MVE_VHADDu16 : MVE_VHADD<MVE_v8u16, avgflooru, addnuw, ARMvshruImm>;
+defm MVE_VHADDu32 : MVE_VHADD<MVE_v4u32, avgflooru, addnuw, ARMvshruImm>;
 
 multiclass MVE_VHSUB_m<MVEVectorVTInfo VTI,
                       SDNode unpred_op, Intrinsic pred_int, PatFrag sub_op,
                       SDNode shift_op> {
   def "" : MVE_VHSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
class MVE_VxADDSUB_qr<string iname, string suffix,
   let Inst{21-20} = size;
   let Inst{16} = 0b0;
   let Inst{12} = subtract;
   let Inst{8} = 0b1;
   let Inst{5} = 0b0;
   let validForTailPredication = 1;
 }
 
-multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
-                             Intrinsic unpred_int, Intrinsic pred_int, PatFrag add_op,
-                             SDNode shift_op> {
+multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract, SDNode Op,
+                             Intrinsic unpred_int, Intrinsic pred_int, PatFrag add_op, PatFrag shift_op> {
   def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract, VTI.Size>;
+  defm : MVE_TwoOpPatternDup<VTI, Op, pred_int, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
   defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME),
                                   VTI, unpred_int, pred_int, 1, 1>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
     def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))), (i32 1))),
               (Inst MQPR:$Qm, rGPR:$Rn)>;
   }
 }
 
-multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> :
-  MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd, int_arm_mve_hadd_predicated,
-                    add_op, shift_op>;
+multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op, SDNode Op> :
+  MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, Op, int_arm_mve_vhadd,
+                    int_arm_mve_hadd_predicated, add_op, shift_op>;
 
 multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> :
-  MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub, int_arm_mve_hsub_predicated,
-                    add_op, shift_op>;
+  MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, null_frag, int_arm_mve_vhsub,
+                    int_arm_mve_hsub_predicated, add_op, shift_op>;
 
-defm MVE_VHADD_qr_s8  : MVE_VHADD_qr_m<MVE_v16s8, addnsw, ARMvshrsImm>;
-defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16, addnsw, ARMvshrsImm>;
-defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32, addnsw, ARMvshrsImm>;
-defm MVE_VHADD_qr_u8  : MVE_VHADD_qr_m<MVE_v16u8, addnuw, ARMvshruImm>;
-defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16, addnuw, ARMvshruImm>;
-defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32, addnuw, ARMvshruImm>;
+defm MVE_VHADD_qr_s8  : MVE_VHADD_qr_m<MVE_v16s8, addnsw, ARMvshrsImm, avgfloors>;
+defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16, addnsw, ARMvshrsImm, avgfloors>;
+defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32, addnsw, ARMvshrsImm, avgfloors>;
+defm MVE_VHADD_qr_u8  : MVE_VHADD_qr_m<MVE_v16u8, addnuw, ARMvshruImm, avgflooru>;
+defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16, addnuw, ARMvshruImm, avgflooru>;
+defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32, addnuw, ARMvshruImm, avgflooru>;
 
 defm MVE_VHSUB_qr_s8  : MVE_VHSUB_qr_m<MVE_v16s8, subnsw, ARMvshrsImm>;
 defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16, subnsw, ARMvshrsImm>;
 defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32, subnsw, ARMvshrsImm>;
 defm MVE_VHSUB_qr_u8  : MVE_VHSUB_qr_m<MVE_v16u8, subnuw, ARMvshruImm>;
 defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16, subnuw, ARMvshruImm>;
 defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32, subnuw, ARMvshruImm>;
 

llvm/test/CodeGen/Thumb2/mve-vhadd.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
 
 define arm_aapcs_vfpcc <4 x i32> @vhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
 ; CHECK-LABEL: vhadds_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d9}
-; CHECK-NEXT:    vpush {d9}
-; CHECK-NEXT:    vmov.f32 s8, s2
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.f32 s10, s3
-; CHECK-NEXT:    vmov.f32 s14, s1
-; CHECK-NEXT:    vmov.f32 s18, s5
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vmov.f32 s8, s6
-; CHECK-NEXT:    vmov.f32 s6, s7
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    asrs r1, r0, #31
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r1, r1, r2, asr #31
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    adds r2, r1, r3
-; CHECK-NEXT:    asr.w r12, r1, #31
-; CHECK-NEXT:    adc.w r1, r12, r3, asr #31
-; CHECK-NEXT:    lsrl r2, r1, #1
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    vmov q0[2], q0[0], r2, r0
-; CHECK-NEXT:    vmov r0, s10
-; CHECK-NEXT:    vmov r2, s6
-; CHECK-NEXT:    asrs r1, r0, #31
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r1, r1, r2, asr #31
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, s14
-; CHECK-NEXT:    adds r2, r1, r3
-; CHECK-NEXT:    asr.w r12, r1, #31
-; CHECK-NEXT:    adc.w r1, r12, r3, asr #31
-; CHECK-NEXT:    lsrl r2, r1, #1
-; CHECK-NEXT:    vmov q0[3], q0[1], r2, r0
-; CHECK-NEXT:    vpop {d9}
+; CHECK-NEXT:    vhadd.s32 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <4 x i32> %s0 to <4 x i64>
   %s1s = sext <4 x i32> %s1 to <4 x i64>
   %m = add <4 x i64> %s0s, %s1s
   %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>
   %s2 = trunc <4 x i64> %s to <4 x i32>
   ret <4 x i32> %s2
 }
 
 define arm_aapcs_vfpcc <4 x i32> @vhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
 ; CHECK-LABEL: vhaddu_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.f32 s8, s6
-; CHECK-NEXT:    vmov.i64 q4, #0xffffffff
-; CHECK-NEXT:    vmov.f32 s10, s7
-; CHECK-NEXT:    vmov.f32 s12, s2
-; CHECK-NEXT:    vand q2, q2, q4
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vand q3, q3, q4
-; CHECK-NEXT:    vmov r0, r1, d4
-; CHECK-NEXT:    vmov r2, r3, d6
-; CHECK-NEXT:    vmov.f32 s6, s5
-; CHECK-NEXT:    vmov.f32 s2, s1
-; CHECK-NEXT:    vand q1, q1, q4
-; CHECK-NEXT:    vand q4, q0, q4
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov r3, r2, d8
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, r12, d2
-; CHECK-NEXT:    adds r4, r3, r1
-; CHECK-NEXT:    adc.w r1, r2, r12
-; CHECK-NEXT:    vmov r2, r3, d7
-; CHECK-NEXT:    lsrl r4, r1, #1
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, r0
-; CHECK-NEXT:    vmov r0, r1, d5
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    vmov r3, r4, d9
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, r12, d3
-; CHECK-NEXT:    adds r2, r3, r1
-; CHECK-NEXT:    adc.w r1, r4, r12
-; CHECK-NEXT:    lsrl r2, r1, #1
-; CHECK-NEXT:    vmov q0[3], q0[1], r2, r0
-; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    vhadd.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <4 x i32> %s0 to <4 x i64>
   %s1s = zext <4 x i32> %s1 to <4 x i64>
   %m = add <4 x i64> %s0s, %s1s
   %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>
   %s2 = trunc <4 x i64> %s to <4 x i32>
   ret <4 x i32> %s2
 }
entry:
   %s = lshr <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
   %s2 = trunc <4 x i32> %s to <4 x i16>
   ret <4 x i16> %s2
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
 ; CHECK-LABEL: vhadds_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.s16 q2, q1
-; CHECK-NEXT:    vmovlt.s16 q3, q0
-; CHECK-NEXT:    vmovlb.s16 q1, q1
-; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vadd.i32 q2, q3, q2
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    vshr.u32 q2, q2, #1
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vhadd.s16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <8 x i16> %s0 to <8 x i32>
   %s1s = sext <8 x i16> %s1 to <8 x i32>
   %m = add <8 x i32> %s0s, %s1s
   %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %s2 = trunc <8 x i32> %s to <8 x i16>
   ret <8 x i16> %s2
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
 ; CHECK-LABEL: vhaddu_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.u16 q2, q1
-; CHECK-NEXT:    vmovlt.u16 q3, q0
-; CHECK-NEXT:    vmovlb.u16 q1, q1
-; CHECK-NEXT:    vmovlb.u16 q0, q0
-; CHECK-NEXT:    vadd.i32 q2, q3, q2
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    vshr.u32 q2, q2, #1
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vhadd.u16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <8 x i16> %s0 to <8 x i32>
   %s1s = zext <8 x i16> %s1 to <8 x i32>
   %m = add <8 x i32> %s0s, %s1s
   %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %s2 = trunc <8 x i32> %s to <8 x i16>
   ret <8 x i16> %s2
entry:
   %s = lshr <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %s2 = trunc <8 x i16> %s to <8 x i8>
   ret <8 x i8> %s2
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
 ; CHECK-LABEL: vhadds_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.s8 q2, q1
-; CHECK-NEXT:    vmovlt.s8 q3, q0
-; CHECK-NEXT:    vmovlb.s8 q1, q1
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vadd.i16 q2, q3, q2
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
-; CHECK-NEXT:    vshr.u16 q2, q2, #1
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vhadd.s8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <16 x i8> %s0 to <16 x i16>
   %s1s = sext <16 x i8> %s1 to <16 x i16>
   %m = add <16 x i16> %s0s, %s1s
   %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %s2 = trunc <16 x i16> %s to <16 x i8>
   ret <16 x i8> %s2
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
 ; CHECK-LABEL: vhaddu_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.u8 q2, q1
-; CHECK-NEXT:    vmovlt.u8 q3, q0
-; CHECK-NEXT:    vmovlb.u8 q1, q1
-; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    vadd.i16 q2, q3, q2
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
-; CHECK-NEXT:    vshr.u16 q2, q2, #1
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vhadd.u8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <16 x i8> %s0 to <16 x i16>
   %s1s = zext <16 x i8> %s1 to <16 x i16>
   %m = add <16 x i16> %s0s, %s1s
   %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %s2 = trunc <16 x i16> %s to <16 x i8>
   ret <16 x i8> %s2
 }
 
 define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
 ; CHECK-LABEL: vrhadds_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d9}
-; CHECK-NEXT:    vpush {d9}
-; CHECK-NEXT:    vmov.f32 s8, s2
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.f32 s10, s3
-; CHECK-NEXT:    vmov.f32 s14, s1
-; CHECK-NEXT:    vmov.f32 s18, s5
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vmov.f32 s8, s6
-; CHECK-NEXT:    vmov.f32 s6, s7
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    asrs r1, r0, #31
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r1, r1, r2, asr #31
-; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    adc r1, r1, #0
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    asrs r2, r1, #31
-; CHECK-NEXT:    adds r1, r1, r3
-; CHECK-NEXT:    adc.w r3, r2, r3, asr #31
-; CHECK-NEXT:    adds r2, r1, #1
-; CHECK-NEXT:    adc r1, r3, #0
-; CHECK-NEXT:    vmov r3, s18
-; CHECK-NEXT:    lsrl r2, r1, #1
-; CHECK-NEXT:    vmov q0[2], q0[0], r2, r0
-; CHECK-NEXT:    vmov r0, s10
-; CHECK-NEXT:    vmov r2, s6
-; CHECK-NEXT:    asrs r1, r0, #31
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r1, r1, r2, asr #31
-; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    adc r1, r1, #0
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, s14
-; CHECK-NEXT:    asrs r2, r1, #31
-; CHECK-NEXT:    adds r1, r1, r3
-; CHECK-NEXT:    adc.w r3, r2, r3, asr #31
-; CHECK-NEXT:    adds r2, r1, #1
-; CHECK-NEXT:    adc r1, r3, #0
-; CHECK-NEXT:    lsrl r2, r1, #1
-; CHECK-NEXT:    vmov q0[3], q0[1], r2, r0
-; CHECK-NEXT:    vpop {d9}
+; CHECK-NEXT:    vrhadd.s32 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <4 x i32> %s0 to <4 x i64>
   %s1s = sext <4 x i32> %s1 to <4 x i64>
   %add = add <4 x i64> %s0s, %s1s
   %add2 = add <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>
   %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
   %result = trunc <4 x i64> %s to <4 x i32>
   ret <4 x i32> %result
 }
 
 define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
 ; CHECK-LABEL: vrhaddu_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.f32 s8, s6
-; CHECK-NEXT:    vmov.i64 q4, #0xffffffff
-; CHECK-NEXT:    vmov.f32 s10, s7
-; CHECK-NEXT:    vmov.f32 s12, s2
-; CHECK-NEXT:    vand q2, q2, q4
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vand q3, q3, q4
-; CHECK-NEXT:    vmov r0, r1, d4
-; CHECK-NEXT:    vmov r2, r3, d6
-; CHECK-NEXT:    vmov.f32 s6, s5
-; CHECK-NEXT:    vmov.f32 s2, s1
-; CHECK-NEXT:    vand q1, q1, q4
-; CHECK-NEXT:    vand q4, q0, q4
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    adc r1, r1, #0
-; CHECK-NEXT:    vmov r3, r2, d8
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, r12, d2
-; CHECK-NEXT:    adds r1, r1, r3
-; CHECK-NEXT:    adc.w r3, r2, r12
-; CHECK-NEXT:    adds r2, r1, #1
-; CHECK-NEXT:    adc r1, r3, #0
-; CHECK-NEXT:    lsrl r2, r1, #1
-; CHECK-NEXT:    vmov q0[2], q0[0], r2, r0
-; CHECK-NEXT:    vmov r0, r1, d5
-; CHECK-NEXT:    vmov r2, r3, d7
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adcs r1, r3
-; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    adc r1, r1, #0
-; CHECK-NEXT:    vmov r3, r2, d9
-; CHECK-NEXT:    lsrl r0, r1, #1
-; CHECK-NEXT:    vmov r1, r12, d3
-; CHECK-NEXT:    adds r1, r1, r3
-; CHECK-NEXT:    adc.w r3, r2, r12
-; CHECK-NEXT:    adds r2, r1, #1
-; CHECK-NEXT:    adc r1, r3, #0
-; CHECK-NEXT:    lsrl r2, r1, #1
-; CHECK-NEXT:    vmov q0[3], q0[1], r2, r0
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <4 x i32> %s0 to <4 x i64>
   %s1s = zext <4 x i32> %s1 to <4 x i64>
   %add = add <4 x i64> %s0s, %s1s
   %add2 = add <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>
   %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
   %result = trunc <4 x i64> %s to <4 x i32>
entry:
   %s = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>
   %result = trunc <4 x i32> %s to <4 x i16>
   ret <4 x i16> %result
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
 ; CHECK-LABEL: vrhadds_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.s16 q2, q1
-; CHECK-NEXT:    vmovlt.s16 q3, q0
-; CHECK-NEXT:    vmovlb.s16 q1, q1
-; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vadd.i32 q2, q3, q2
-; CHECK-NEXT:    movs r0, #1
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    vadd.i32 q2, q2, r0
-; CHECK-NEXT:    vadd.i32 q0, q0, r0
-; CHECK-NEXT:    vshr.u32 q2, q2, #1
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vrhadd.s16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <8 x i16> %s0 to <8 x i32>
   %s1s = sext <8 x i16> %s1 to <8 x i32>
   %add = add <8 x i32> %s0s, %s1s
   %add2 = add <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %result = trunc <8 x i32> %s to <8 x i16>
   ret <8 x i16> %result
 }
 
 define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
 ; CHECK-LABEL: vrhaddu_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.u16 q2, q1
-; CHECK-NEXT:    vmovlt.u16 q3, q0
-; CHECK-NEXT:    vmovlb.u16 q1, q1
-; CHECK-NEXT:    vmovlb.u16 q0, q0
-; CHECK-NEXT:    vadd.i32 q2, q3, q2
-; CHECK-NEXT:    movs r0, #1
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    vadd.i32 q2, q2, r0
-; CHECK-NEXT:    vadd.i32 q0, q0, r0
-; CHECK-NEXT:    vshr.u32 q2, q2, #1
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vrhadd.u16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <8 x i16> %s0 to <8 x i32>
   %s1s = zext <8 x i16> %s1 to <8 x i32>
   %add = add <8 x i32> %s0s, %s1s
   %add2 = add <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %result = trunc <8 x i32> %s to <8 x i16>
entry:
   %s = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %result = trunc <8 x i16> %s to <8 x i8>
   ret <8 x i8> %result
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
 ; CHECK-LABEL: vrhadds_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.s8 q2, q1
-; CHECK-NEXT:    vmovlt.s8 q3, q0
-; CHECK-NEXT:    vmovlb.s8 q1, q1
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vadd.i16 q2, q3, q2
-; CHECK-NEXT:    movs r0, #1
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
-; CHECK-NEXT:    vadd.i16 q2, q2, r0
-; CHECK-NEXT:    vadd.i16 q0, q0, r0
-; CHECK-NEXT:    vshr.u16 q2, q2, #1
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vrhadd.s8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <16 x i8> %s0 to <16 x i16>
   %s1s = sext <16 x i8> %s1 to <16 x i16>
   %add = add <16 x i16> %s0s, %s1s
   %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %result = trunc <16 x i16> %s to <16 x i8>
   ret <16 x i8> %result
 }
 
 define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
 ; CHECK-LABEL: vrhaddu_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.u8 q2, q1
-; CHECK-NEXT:    vmovlt.u8 q3, q0
-; CHECK-NEXT:    vmovlb.u8 q1, q1
-; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    vadd.i16 q2, q3, q2
-; CHECK-NEXT:    movs r0, #1
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
-; CHECK-NEXT:    vadd.i16 q2, q2, r0
-; CHECK-NEXT:    vadd.i16 q0, q0, r0
-; CHECK-NEXT:    vshr.u16 q2, q2, #1
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vrhadd.u8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <16 x i8> %s0 to <16 x i16>
   %s1s = zext <16 x i8> %s1 to <16 x i16>
   %add = add <16 x i16> %s0s, %s1s
   %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %result = trunc <16 x i16> %s to <16 x i8>
   ret <16 x i8> %result
 }
 
 
 
 
 define void @vhadd_loop_s8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vhadd_loop_s8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #64
 ; CHECK-NEXT:  .LBB24_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.s16 q0, [r0, #8]
-; CHECK-NEXT:    vldrb.s16 q1, [r1, #8]
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
-; CHECK-NEXT:    vldrb.s16 q1, [r1], #16
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vstrb.16 q0, [r2, #8]
-; CHECK-NEXT:    vldrb.s16 q0, [r0], #16
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vstrb.16 q0, [r2], #16
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
+; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
+; CHECK-NEXT:    vhadd.s8 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB24_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 define void @vhadd_loop_s16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vhadd_loop_s16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #128
 ; CHECK-NEXT:  .LBB25_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
-; CHECK-NEXT:    vadd.i32 q0, q1, q0
-; CHECK-NEXT:    vldrh.s32 q1, [r1], #16
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vstrh.32 q0, [r2, #8]
-; CHECK-NEXT:    vldrh.s32 q0, [r0], #16
-; CHECK-NEXT:    vadd.i32 q0, q1, q0
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vstrh.32 q0, [r2], #16
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
+; CHECK-NEXT:    vhadd.s16 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB25_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @vhadd_loop_s32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vhadd_loop_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .vsave {d9}
-; CHECK-NEXT:    vpush {d9}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #256
 ; CHECK-NEXT:  .LBB26_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
-; CHECK-NEXT:    vmov.f32 s4, s2
-; CHECK-NEXT:    vmov.f32 s2, s3
-; CHECK-NEXT:    vmov.f32 s10, s1
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vmov.f32 s8, s6
-; CHECK-NEXT:    vmov.f32 s6, s7
-; CHECK-NEXT:    vmov.f32 s18, s5
-; CHECK-NEXT:    vmov r5, s8
-; CHECK-NEXT:    asrs r4, r3, #31
-; CHECK-NEXT:    adds.w r12, r3, r5
-; CHECK-NEXT:    adc.w r3, r4, r5, asr #31
-; CHECK-NEXT:    vmov r5, s4
-; CHECK-NEXT:    lsrl r12, r3, #1
-; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    adds r6, r3, r5
-; CHECK-NEXT:    asr.w r4, r3, #31
-; CHECK-NEXT:    adc.w r3, r4, r5, asr #31
-; CHECK-NEXT:    lsrl r6, r3, #1
-; CHECK-NEXT:    vmov r5, s6
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    vmov q3[2], q3[0], r6, r12
-; CHECK-NEXT:    adds r4, r3, r5
-; CHECK-NEXT:    asr.w r6, r3, #31
-; CHECK-NEXT:    adc.w r3, r6, r5, asr #31
-; CHECK-NEXT:    lsrl r4, r3, #1
-; CHECK-NEXT:    vmov r5, s18
-; CHECK-NEXT:    vmov r3, s10
-; CHECK-NEXT:    adds r6, r3, r5
-; CHECK-NEXT:    asr.w r12, r3, #31
-; CHECK-NEXT:    adc.w r3, r12, r5, asr #31
-; CHECK-NEXT:    lsrl r6, r3, #1
-; CHECK-NEXT:    vmov q3[3], q3[1], r6, r4
-; CHECK-NEXT:    vstrb.8 q3, [r2], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
+; CHECK-NEXT:    vhadd.s32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB26_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d9}
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
   %0 = getelementptr inbounds i32, i32* %x, i32 %index
   %1 = bitcast i32* %0 to <4 x i32>*
   %wide.load = load <4 x i32>, <4 x i32>* %1, align 4

 define void @vhadd_loop_u8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vhadd_loop_u8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #64
 ; CHECK-NEXT:  .LBB27_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.u16 q0, [r0, #8]
-; CHECK-NEXT:    vldrb.u16 q1, [r1, #8]
-; CHECK-NEXT:    vhadd.u16 q0, q1, q0
-; CHECK-NEXT:    vldrb.u16 q1, [r1], #16
-; CHECK-NEXT:    vstrb.16 q0, [r2, #8]
-; CHECK-NEXT:    vldrb.u16 q0, [r0], #16
-; CHECK-NEXT:    vhadd.u16 q0, q1, q0
-; CHECK-NEXT:    vstrb.16 q0, [r2], #16
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
+; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
+; CHECK-NEXT:    vhadd.u8 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB27_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 define void @vhadd_loop_u16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vhadd_loop_u16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #128
 ; CHECK-NEXT:  .LBB28_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u32 q0, [r0, #8]
-; CHECK-NEXT:    vldrh.u32 q1, [r1, #8]
-; CHECK-NEXT:    vhadd.u32 q0, q1, q0
-; CHECK-NEXT:    vldrh.u32 q1, [r1], #16
-; CHECK-NEXT:    vstrh.32 q0, [r2, #8]
-; CHECK-NEXT:    vldrh.u32 q0, [r0], #16
-; CHECK-NEXT:    vhadd.u32 q0, q1, q0
-; CHECK-NEXT:    vstrh.32 q0, [r2], #16
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
+; CHECK-NEXT:    vhadd.u16 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB28_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @vhadd_loop_u32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vhadd_loop_u32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #256
-; CHECK-NEXT:    vmov.i64 q0, #0xffffffff
 ; CHECK-NEXT:  .LBB29_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q3, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q4, [r1], #16
-; CHECK-NEXT:    vmov.f32 s4, s14
-; CHECK-NEXT:    vmov.f32 s6, s15
-; CHECK-NEXT:    vmov.f32 s8, s18
-; CHECK-NEXT:    vand q1, q1, q0
-; CHECK-NEXT:    vmov.f32 s10, s19
-; CHECK-NEXT:    vand q2, q2, q0
-; CHECK-NEXT:    vmov r3, r5, d2
-; CHECK-NEXT:    vmov r4, r6, d4
-; CHECK-NEXT:    vmov.f32 s14, s13
-; CHECK-NEXT:    vmov.f32 s18, s17
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q5, q4, q0
-; CHECK-NEXT:    adds.w r12, r4, r3
-; CHECK-NEXT:    adc.w r3, r6, r5
-; CHECK-NEXT:    vmov r5, r6, d10
-; CHECK-NEXT:    lsrl r12, r3, #1
-; CHECK-NEXT:    vmov r3, r7, d6
-; CHECK-NEXT:    adds r4, r5, r3
-; CHECK-NEXT:    adc.w r3, r6, r7
-; CHECK-NEXT:    vmov r6, r5, d5
-; CHECK-NEXT:    lsrl r4, r3, #1
-; CHECK-NEXT:    vmov r3, r7, d3
-; CHECK-NEXT:    vmov q4[2], q4[0], r4, r12
-; CHECK-NEXT:    adds r6, r6, r3
-; CHECK-NEXT:    adc.w r3, r5, r7
-; CHECK-NEXT:    vmov r5, r7, d11
-; CHECK-NEXT:    lsrl r6, r3, #1
-; CHECK-NEXT:    vmov r3, r12, d7
-; CHECK-NEXT:    adds r4, r5, r3
-; CHECK-NEXT:    adc.w r3, r7, r12
-; CHECK-NEXT:    lsrl r4, r3, #1
-; CHECK-NEXT:    vmov q4[3], q4[1], r4, r6
-; CHECK-NEXT:    vstrb.8 q4, [r2], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
+; CHECK-NEXT:    vhadd.u32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB29_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
   %0 = getelementptr inbounds i32, i32* %x, i32 %index
   %1 = bitcast i32* %0 to <4 x i32>*
   %wide.load = load <4 x i32>, <4 x i32>* %1, align 4

 }
 
 define void @vrhadd_loop_s8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vrhadd_loop_s8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #64
-; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:  .LBB30_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.u16 q0, [r1, #8]
-; CHECK-NEXT:    vldrb.u16 q1, [r0, #8]
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
-; CHECK-NEXT:    vldrb.u16 q1, [r0], #16
-; CHECK-NEXT:    vadd.i16 q0, q0, r3
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vstrb.16 q0, [r2, #8]
-; CHECK-NEXT:    vldrb.u16 q0, [r1], #16
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
-; CHECK-NEXT:    vadd.i16 q0, q0, r3
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vstrb.16 q0, [r2], #16
+; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
+; CHECK-NEXT:    vldrb.u8 q1, [r0], #16
+; CHECK-NEXT:    vrhadd.u8 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB30_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 }
 
 define void @vrhadd_loop_s16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vrhadd_loop_s16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #128
-; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:  .LBB31_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u32 q0, [r1, #8]
-; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
-; CHECK-NEXT:    vadd.i32 q0, q1, q0
-; CHECK-NEXT:    vldrh.u32 q1, [r0], #16
-; CHECK-NEXT:    vadd.i32 q0, q0, r3
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vstrh.32 q0, [r2, #8]
-; CHECK-NEXT:    vldrh.u32 q0, [r1], #16
-; CHECK-NEXT:    vadd.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r3
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vstrh.32 q0, [r2], #16
+; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vrhadd.u16 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB31_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @vrhadd_loop_s32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vrhadd_loop_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #256
-; CHECK-NEXT:    vmov.i64 q0, #0xffffffff
 ; CHECK-NEXT:  .LBB32_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q3, [r1], #16
-; CHECK-NEXT:    vldrw.u32 q4, [r0], #16
-; CHECK-NEXT:    vmov.f32 s4, s14
-; CHECK-NEXT:    vmov.f32 s6, s15
-; CHECK-NEXT:    vmov.f32 s8, s18
-; CHECK-NEXT:    vand q1, q1, q0
-; CHECK-NEXT:    vmov.f32 s10, s19
-; CHECK-NEXT:    vand q2, q2, q0
-; CHECK-NEXT:    vmov r3, r12, d2
-; CHECK-NEXT:    vmov r4, r5, d4
-; CHECK-NEXT:    vmov.f32 s14, s13
-; CHECK-NEXT:    vmov.f32 s18, s17
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q5, q4, q0
-; CHECK-NEXT:    adds r3, r3, r4
-; CHECK-NEXT:    adc.w r4, r5, r12
-; CHECK-NEXT:    adds.w r12, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    vmov r5, r6, d10
-; CHECK-NEXT:    lsrl r12, r3, #1
-; CHECK-NEXT:    vmov r3, r4, d6
-; CHECK-NEXT:    adds r3, r3, r5
-; CHECK-NEXT:    adcs r4, r6
-; CHECK-NEXT:    adds r6, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    vmov r5, r4, d5
-; CHECK-NEXT:    lsrl r6, r3, #1
-; CHECK-NEXT:    vmov q4[2], q4[0], r6, r12
-; CHECK-NEXT:    vmov r3, r6, d3
-; CHECK-NEXT:    adds r3, r3, r5
-; CHECK-NEXT:    adcs r4, r6
-; CHECK-NEXT:    adds.w r12, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    vmov r5, r6, d11
-; CHECK-NEXT:    lsrl r12, r3, #1
-; CHECK-NEXT:    vmov r3, r4, d7
-; CHECK-NEXT:    adds r3, r3, r5
-; CHECK-NEXT:    adcs r4, r6
-; CHECK-NEXT:    adds r6, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    lsrl r6, r3, #1
-; CHECK-NEXT:    vmov q4[3], q4[1], r6, r12
-; CHECK-NEXT:    vstrb.8 q4, [r2], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vrhadd.u32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB32_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
   %0 = getelementptr inbounds i32, i32* %x, i32 %index
   %1 = bitcast i32* %0 to <4 x i32>*
   %wide.load = load <4 x i32>, <4 x i32>* %1, align 4

 }
 
 define void @vrhadd_loop_u8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vrhadd_loop_u8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #64
-; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:  .LBB33_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.u16 q0, [r1, #8]
-; CHECK-NEXT:    vldrb.u16 q1, [r0, #8]
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
-; CHECK-NEXT:    vldrb.u16 q1, [r0], #16
-; CHECK-NEXT:    vadd.i16 q0, q0, r3
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vstrb.16 q0, [r2, #8]
-; CHECK-NEXT:    vldrb.u16 q0, [r1], #16
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
-; CHECK-NEXT:    vadd.i16 q0, q0, r3
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vstrb.16 q0, [r2], #16
+; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
+; CHECK-NEXT:    vldrb.u8 q1, [r0], #16
+; CHECK-NEXT:    vrhadd.u8 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB33_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 }
 
 define void @vrhadd_loop_u16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vrhadd_loop_u16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #128
-; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:  .LBB34_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u32 q0, [r1, #8]
-; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
-; CHECK-NEXT:    vadd.i32 q0, q1, q0
-; CHECK-NEXT:    vldrh.u32 q1, [r0], #16
-; CHECK-NEXT:    vadd.i32 q0, q0, r3
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vstrh.32 q0, [r2, #8]
-; CHECK-NEXT:    vldrh.u32 q0, [r1], #16
-; CHECK-NEXT:    vadd.i32 q0, q1, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, r3
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vstrh.32 q0, [r2], #16
+; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vrhadd.u16 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB34_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]

 
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
 
 define void @vrhadd_loop_u32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
 ; CHECK-LABEL: vrhadd_loop_u32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #256
-; CHECK-NEXT:    vmov.i64 q0, #0xffffffff
 ; CHECK-NEXT:  .LBB35_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q3, [r1], #16
-; CHECK-NEXT:    vldrw.u32 q4, [r0], #16
-; CHECK-NEXT:    vmov.f32 s4, s14
-; CHECK-NEXT:    vmov.f32 s6, s15
-; CHECK-NEXT:    vmov.f32 s8, s18
-; CHECK-NEXT:    vand q1, q1, q0
-; CHECK-NEXT:    vmov.f32 s10, s19
-; CHECK-NEXT:    vand q2, q2, q0
-; CHECK-NEXT:    vmov r3, r12, d2
-; CHECK-NEXT:    vmov r4, r5, d4
-; CHECK-NEXT:    vmov.f32 s14, s13
-; CHECK-NEXT:    vmov.f32 s18, s17
-; CHECK-NEXT:    vand q3, q3, q0
-; CHECK-NEXT:    vand q5, q4, q0
-; CHECK-NEXT:    adds r3, r3, r4
-; CHECK-NEXT:    adc.w r4, r5, r12
-; CHECK-NEXT:    adds.w r12, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    vmov r5, r6, d10
-; CHECK-NEXT:    lsrl r12, r3, #1
-; CHECK-NEXT:    vmov r3, r4, d6
-; CHECK-NEXT:    adds r3, r3, r5
-; CHECK-NEXT:    adcs r4, r6
-; CHECK-NEXT:    adds r6, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    vmov r5, r4, d5
-; CHECK-NEXT:    lsrl r6, r3, #1
-; CHECK-NEXT:    vmov q4[2], q4[0], r6, r12
-; CHECK-NEXT:    vmov r3, r6, d3
-; CHECK-NEXT:    adds r3, r3, r5
-; CHECK-NEXT:    adcs r4, r6
-; CHECK-NEXT:    adds.w r12, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    vmov r5, r6, d11
-; CHECK-NEXT:    lsrl r12, r3, #1
-; CHECK-NEXT:    vmov r3, r4, d7
-; CHECK-NEXT:    adds r3, r3, r5
-; CHECK-NEXT:    adcs r4, r6
-; CHECK-NEXT:    adds r6, r3, #1
-; CHECK-NEXT:    adc r3, r4, #0
-; CHECK-NEXT:    lsrl r6, r3, #1
-; CHECK-NEXT:    vmov q4[3], q4[1], r6, r12
-; CHECK-NEXT:    vstrb.8 q4, [r2], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vrhadd.u32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB35_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %vector.body
 
 vector.body:                                      ; preds = %vector.body, %entry
   %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
   %0 = getelementptr inbounds i32, i32* %x, i32 %index
   %1 = bitcast i32* %0 to <4 x i32>*
   %wide.load = load <4 x i32>, <4 x i32>* %1, align 4