[ARM] Fixup FP16 bitcasts

[dmgreen]

Summary

Under fp16 we have to lower via custom lowering a bitcast.

If the bitcast is between a i32 and a f32, we currently look for the pattern we want and if we don't find it we return SDValue(), which makes the underlying DAG legaliser expand the code to a stack load and store. The i32 to f32 bitcast should still be legal though, so the whole thing has been rewritten as a dag combine.

Seems to help in some of the MVE scalarised code too.

[dmgreen]

Update another test.

[efriedma]

llvm/lib/Target/ARM/ARMISelLowering.cpp

5740–5741

Can we make this optimization a TargetDAGCombine, instead of wedging it into legalization?

[dmgreen]

llvm/lib/Target/ARM/ARMISelLowering.cpp

5740–5741

Can you explain what you mean?

My understanding is that this is where the bug is. That we shouldn't be returning expand (SDValue()) if the node is actually legal. Trying to add something that sounds like custom bitcast lowering doesn't sound right to me.

But I may be wrong about any or all of that. Let me know.

[efriedma]

llvm/lib/Target/ARM/ARMISelLowering.cpp

5740–5741

An f32<->i32 bitcast is always legal; it shouldn't be marked "Custom" in the first place.

The transform here is to transform VMOVhr(bitcast(CopyFromReg)) -> CopyFromReg. That seems like a perfect candidate for a TargetDAGCombine.

[dmgreen]

llvm/lib/Target/ARM/ARMISelLowering.cpp

5740–5741

Ah. Rewrite the existing code? Sure, that sounds like something that should work. And like you said might be cleaner. Let me give it a go.

[efriedma]

LGTM

llvm/lib/Target/ARM/ARMISelLowering.cpp

if (!Subtarget->hasVFP2Base())
       setAllExpand(MVT::f32);
     if (!Subtarget->hasFP64())
       setAllExpand(MVT::f64);
   }
 
   if (Subtarget->hasFullFP16()) {
     addRegisterClass(MVT::f16, &ARM::HPRRegClass);
     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
-    setOperationAction(ISD::BITCAST, MVT::i32, Custom);
     setOperationAction(ISD::BITCAST, MVT::f16, Custom);
 
     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
   }
 
   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
   SDLoc dl(N);
   SDValue Op = N->getOperand(0);
 
   // This function is only supposed to be called for i64 types, either as the
   // source or destination of the bit convert.
   EVT SrcVT = Op.getValueType();
   EVT DstVT = N->getValueType(0);
   const bool HasFullFP16 = Subtarget->hasFullFP16();
 
-  if (SrcVT == MVT::f32 && DstVT == MVT::i32) {
-     // FullFP16: half values are passed in S-registers, and we don't
-     // need any of the bitcast and moves:
-     //
-     // t2: f32,ch = CopyFromReg t0, Register:f32 %0
-     //   t5: i32 = bitcast t2
-     // t18: f16 = ARMISD::VMOVhr t5
-     if (Op.getOpcode() != ISD::CopyFromReg ||
-         Op.getValueType() != MVT::f32)
-       return SDValue();
-
-     auto Move = N->use_begin();
-     if (Move->getOpcode() != ARMISD::VMOVhr)
-       return SDValue();
-
-     SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) };
-     SDValue Copy = DAG.getNode(ISD::CopyFromReg, SDLoc(Op), MVT::f16, Ops);
-     DAG.ReplaceAllUsesWith(*Move, &Copy);
-     return Copy;
-  }
-
   if (SrcVT == MVT::i16 && DstVT == MVT::f16) {
     if (!HasFullFP16)
       return SDValue();
     // SoftFP: read half-precision arguments:
     //
     // t2: i32,ch = ...
     //        t7: i16 = truncate t2 <~~~~ Op
     //      t8: f16 = bitcast t7    <~~~~ N
     //
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
   if (Op0.getOpcode() == ARMISD::VMOVRRD &&
       Op0.getNode() == Op1.getNode() &&
       Op0.getResNo() == 0 && Op1.getResNo() == 1)
     return DAG.getNode(ISD::BITCAST, SDLoc(N),
                        N->getValueType(0), Op0.getOperand(0));
   return SDValue();
 }
 
+static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
+  // FullFP16: half values are passed in S-registers, and we don't
+  // need any of the bitcast and moves:
+  //
+  //     t2: f32,ch = CopyFromReg t0, Register:f32 %0
+  //   t5: i32 = bitcast t2
+  // t18: f16 = ARMISD::VMOVhr t5
+  SDValue BC = N->getOperand(0);
+  if (BC->getOpcode() != ISD::BITCAST)
+    return SDValue();
+  SDValue Copy = BC->getOperand(0);
+  if (Copy.getValueType() != MVT::f32 || Copy->getOpcode() != ISD::CopyFromReg)
+    return SDValue();
+
+  SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1)};
+  SDValue NewCopy = DCI.DAG.getNode(ISD::CopyFromReg, SDLoc(N), MVT::f16, Ops);
+  return NewCopy;
+}
+
 /// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
 /// are normal, non-volatile loads.  If so, it is profitable to bitcast an
 /// i64 vector to have f64 elements, since the value can then be loaded
 /// directly into a VFP register.
 static bool hasNormalLoadOperand(SDNode *N) {
   unsigned NumElts = N->getValueType(0).getVectorNumElements();
   for (unsigned i = 0; i < NumElts; ++i) {
     SDNode *Elt = N->getOperand(i).getNode();
SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::BRCOND:
   case ISD::BR_CC:      return PerformHWLoopCombine(N, DCI, Subtarget);
   case ARMISD::ADDC:
   case ARMISD::SUBC:    return PerformAddcSubcCombine(N, DCI, Subtarget);
   case ARMISD::SUBE:    return PerformAddeSubeCombine(N, DCI, Subtarget);
   case ARMISD::BFI:     return PerformBFICombine(N, DCI);
   case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
   case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
+  case ARMISD::VMOVhr:  return PerformVMOVhrCombine(N, DCI);
   case ISD::STORE:      return PerformSTORECombine(N, DCI, Subtarget);
   case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
   case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
   case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
   case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI);
   case ARMISD::VDUP: return PerformVDUPCombine(N, DCI, Subtarget);
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT:

llvm/test/CodeGen/ARM/fp16-bitcast.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple thumbv8m.main-arm-unknown-eabi -mattr=+vfp4d16sp < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-VFPV4
 ; RUN: llc -mtriple thumbv8.1m.main-arm-unknown-eabi -mattr=+fullfp16 < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP16
 
 target triple = "thumbv8.1m.main-arm-unknown-eabi"
 
 define float @add(float %a, float %b) {
-; CHECK-VFPV4-LABEL: add:
-; CHECK-VFPV4:       @ %bb.0: @ %entry
-; CHECK-VFPV4-NEXT:    vmov s0, r1
-; CHECK-VFPV4-NEXT:    vmov s2, r0
-; CHECK-VFPV4-NEXT:    vadd.f32 s0, s2, s0
-; CHECK-VFPV4-NEXT:    vmov r0, s0
-; CHECK-VFPV4-NEXT:    bx lr
-;
-; CHECK-FP16-LABEL: add:
-; CHECK-FP16:       @ %bb.0: @ %entry
-; CHECK-FP16-NEXT:    .pad #4
-; CHECK-FP16-NEXT:    sub sp, #4
-; CHECK-FP16-NEXT:    vmov s0, r1
-; CHECK-FP16-NEXT:    vmov s2, r0
-; CHECK-FP16-NEXT:    vadd.f32 s0, s2, s0
-; CHECK-FP16-NEXT:    vstr s0, [sp]
-; CHECK-FP16-NEXT:    ldr r0, [sp], #4
-; CHECK-FP16-NEXT:    bx lr
+; CHECK-LABEL: add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov s0, r1
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
 entry:
   %add = fadd float %a, %b
   ret float %add
 }
 
 define i32 @addf16(i32 %a.coerce, i32 %b.coerce) {
 ; CHECK-VFPV4-LABEL: addf16:
 ; CHECK-VFPV4:       @ %bb.0: @ %entry

llvm/test/CodeGen/Thumb2/mve-div-expand.ll

entry:
   ret <4 x float> %out
 }
 
 define arm_aapcs_vfpcc <4 x float> @frem_f32(<4 x float> %in1, <4 x float> %in2) {
 ; CHECK-LABEL: frem_f32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r4, [sp]
-; CHECK-NEXT:    vstr s2, [sp, #16]
-; CHECK-NEXT:    vstr s6, [sp, #20]
-; CHECK-NEXT:    vstr s3, [sp, #24]
-; CHECK-NEXT:    vstr s7, [sp, #28]
-; CHECK-NEXT:    vstr s4, [sp, #4]
-; CHECK-NEXT:    vstr s1, [sp, #8]
-; CHECK-NEXT:    vstr s5, [sp, #12]
-; CHECK-NEXT:    ldrd r0, r1, [sp, #16]
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov q4, q1
+; CHECK-NEXT:    vmov q5, q0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov r1, s18
 ; CHECK-NEXT:    bl fmodf
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #24]
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vmov r0, s23
+; CHECK-NEXT:    vmov r1, s19
 ; CHECK-NEXT:    bl fmodf
-; CHECK-NEXT:    ldr r1, [sp, #12]
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r6, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    ldrd r6, r2, [sp, #4]
-; CHECK-NEXT:    vmov s18, r5
+; CHECK-NEXT:    vmov r5, s20
+; CHECK-NEXT:    vmov s18, r4
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s17, r0
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #32
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %out = frem <4 x float> %in1, %in2
   ret <4 x float> %out
 }
 
 
 define arm_aapcs_vfpcc <8 x half> @fdiv_f16(<8 x half> %in1, <8 x half> %in2) {

 
 define arm_aapcs_vfpcc <8 x half> @frem_f16(<8 x half> %in1, <8 x half> %in2) {
 ; CHECK-LABEL: frem_f16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    .pad #64
-; CHECK-NEXT:    sub sp, #64
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #56]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s20
-; CHECK-NEXT:    vstr s0, [sp, #60]
-; CHECK-NEXT:    ldrd r0, r1, [sp, #56]
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s16
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s20
-; CHECK-NEXT:    vstr s2, [sp, #48]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #52]
-; CHECK-NEXT:    ldrd r0, r1, [sp, #48]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q6[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #40]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
-; CHECK-NEXT:    vstr s0, [sp, #44]
 ; CHECK-NEXT:    vmov.16 q6[1], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #40]
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s17
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s21
-; CHECK-NEXT:    vstr s2, [sp, #32]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #36]
 ; CHECK-NEXT:    vmov.16 q6[2], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #32]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
-; CHECK-NEXT:    vstr s0, [sp, #28]
 ; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s18
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s22
-; CHECK-NEXT:    vstr s2, [sp, #16]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #16]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q6[5], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s19
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s23
-; CHECK-NEXT:    vstr s2, [sp]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q6[7], r0
 ; CHECK-NEXT:    vmov q0, q6
-; CHECK-NEXT:    add sp, #64
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %out = frem <8 x half> %in1, %in2
   ret <8 x half> %out
 }
 
 define arm_aapcs_vfpcc <2 x double> @fdiv_f64(<2 x double> %in1, <2 x double> %in2) {

llvm/test/CodeGen/Thumb2/mve-fmath.ll

 
 define arm_aapcs_vfpcc <4 x float> @cos_float32_t(<4 x float> %src) {
 ; CHECK-LABEL: cos_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vstmia sp, {s0, s1, s2, s3}
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl cosf
-; CHECK-NEXT:    ldrd r5, r1, [sp]
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r5, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov s18, r4
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s17, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.cos.v4f32(<4 x float> %src)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @cos_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: cos_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    ldr r0, [sp, #20]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vstr s0, [sp, #16]
-; CHECK-NEXT:    ldr r0, [sp, #16]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl cosf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q5[7], r0
 ; CHECK-NEXT:    vmov q0, q5
-; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.cos.v8f16(<8 x half> %src)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @cos_float64_t(<2 x double> %src) {

 
 define arm_aapcs_vfpcc <4 x float> @sin_float32_t(<4 x float> %src) {
 ; CHECK-LABEL: sin_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vstmia sp, {s0, s1, s2, s3}
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl sinf
-; CHECK-NEXT:    ldrd r5, r1, [sp]
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r5, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov s18, r4
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s17, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.sin.v4f32(<4 x float> %src)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @sin_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: sin_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    ldr r0, [sp, #20]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vstr s0, [sp, #16]
-; CHECK-NEXT:    ldr r0, [sp, #16]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl sinf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q5[7], r0
 ; CHECK-NEXT:    vmov q0, q5
-; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.sin.v8f16(<8 x half> %src)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @sin_float64_t(<2 x double> %src) {

 
 define arm_aapcs_vfpcc <4 x float> @exp_float32_t(<4 x float> %src) {
 ; CHECK-LABEL: exp_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vstmia sp, {s0, s1, s2, s3}
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl expf
-; CHECK-NEXT:    ldrd r5, r1, [sp]
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r5, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov s18, r4
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s17, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.exp.v4f32(<4 x float> %src)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @exp_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: exp_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    ldr r0, [sp, #20]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vstr s0, [sp, #16]
-; CHECK-NEXT:    ldr r0, [sp, #16]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl expf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q5[7], r0
 ; CHECK-NEXT:    vmov q0, q5
-; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.exp.v8f16(<8 x half> %src)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @exp_float64_t(<2 x double> %src) {

 
 define arm_aapcs_vfpcc <4 x float> @exp2_float32_t(<4 x float> %src) {
 ; CHECK-LABEL: exp2_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vstmia sp, {s0, s1, s2, s3}
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl exp2f
-; CHECK-NEXT:    ldrd r5, r1, [sp]
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r5, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov s18, r4
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s17, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.exp2.v4f32(<4 x float> %src)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @exp2_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: exp2_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    ldr r0, [sp, #20]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vstr s0, [sp, #16]
-; CHECK-NEXT:    ldr r0, [sp, #16]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl exp2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q5[7], r0
 ; CHECK-NEXT:    vmov q0, q5
-; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.exp2.v8f16(<8 x half> %src)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @exp2_float64_t(<2 x double> %src) {

 
 define arm_aapcs_vfpcc <4 x float> @log_float32_t(<4 x float> %src) {
 ; CHECK-LABEL: log_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vstmia sp, {s0, s1, s2, s3}
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl logf
-; CHECK-NEXT:    ldrd r5, r1, [sp]
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r5, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov s18, r4
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s17, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.log.v4f32(<4 x float> %src)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @log_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: log_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    ldr r0, [sp, #20]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vstr s0, [sp, #16]
-; CHECK-NEXT:    ldr r0, [sp, #16]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl logf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q5[7], r0
 ; CHECK-NEXT:    vmov q0, q5
-; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.log.v8f16(<8 x half> %src)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @log_float64_t(<2 x double> %src) {

 
 define arm_aapcs_vfpcc <4 x float> @log2_float32_t(<4 x float> %src) {
 ; CHECK-LABEL: log2_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vstmia sp, {s0, s1, s2, s3}
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl log2f
-; CHECK-NEXT:    ldrd r5, r1, [sp]
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r5, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov s18, r4
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s17, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.log2.v4f32(<4 x float> %src)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @log2_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: log2_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    ldr r0, [sp, #20]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vstr s0, [sp, #16]
-; CHECK-NEXT:    ldr r0, [sp, #16]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log2f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q5[7], r0
 ; CHECK-NEXT:    vmov q0, q5
-; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.log2.v8f16(<8 x half> %src)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @log2_float64_t(<2 x double> %src) {

 
 define arm_aapcs_vfpcc <4 x float> @log10_float32_t(<4 x float> %src) {
 ; CHECK-LABEL: log10_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vstmia sp, {s0, s1, s2, s3}
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s18
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s19
 ; CHECK-NEXT:    bl log10f
-; CHECK-NEXT:    ldrd r5, r1, [sp]
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r5, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    vmov s18, r4
+; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s17, r0
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.log10.v4f32(<4 x float> %src)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @log10_float16_t(<8 x half> %src) {
 ; CHECK-LABEL: log10_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #28]
-; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s16
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    ldr r0, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    ldr r0, [sp, #20]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s17
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vstr s0, [sp, #16]
-; CHECK-NEXT:    ldr r0, [sp, #16]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    ldr r0, [sp, #12]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s18
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    ldr r0, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    ldr r0, [sp, #4]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s19
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    bl log10f
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q5[7], r0
 ; CHECK-NEXT:    vmov q0, q5
-; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.log10.v8f16(<8 x half> %src)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @log10_float64_t(<2 x double> %src) {
entry:
   ret <2 x double> %0
 }
 
 define arm_aapcs_vfpcc <4 x float> @pow_float32_t(<4 x float> %src1, <4 x float> %src2) {
 ; CHECK-LABEL: pow_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vstr s0, [sp]
-; CHECK-NEXT:    ldr r4, [sp]
-; CHECK-NEXT:    vstr s2, [sp, #16]
-; CHECK-NEXT:    vstr s6, [sp, #20]
-; CHECK-NEXT:    vstr s3, [sp, #24]
-; CHECK-NEXT:    vstr s7, [sp, #28]
-; CHECK-NEXT:    vstr s4, [sp, #4]
-; CHECK-NEXT:    vstr s1, [sp, #8]
-; CHECK-NEXT:    vstr s5, [sp, #12]
-; CHECK-NEXT:    ldrd r0, r1, [sp, #16]
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov q4, q1
+; CHECK-NEXT:    vmov q5, q0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    vmov r1, s18
 ; CHECK-NEXT:    bl powf
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #24]
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    vmov r0, s23
+; CHECK-NEXT:    vmov r1, s19
 ; CHECK-NEXT:    bl powf
-; CHECK-NEXT:    ldr r1, [sp, #12]
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vmov r1, s17
+; CHECK-NEXT:    vmov r6, s16
 ; CHECK-NEXT:    vmov s19, r0
-; CHECK-NEXT:    ldrd r6, r2, [sp, #4]
-; CHECK-NEXT:    vmov s18, r5
+; CHECK-NEXT:    vmov r5, s20
+; CHECK-NEXT:    vmov s18, r4
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s17, r0
-; CHECK-NEXT:    mov r0, r4
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s16, r0
 ; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    add sp, #32
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %0 = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %src1, <4 x float> %src2)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @pow_float16_t(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-LABEL: pow_float16_t:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    .pad #64
-; CHECK-NEXT:    sub sp, #64
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s16
-; CHECK-NEXT:    vstr s0, [sp, #56]
+; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s20
-; CHECK-NEXT:    vstr s0, [sp, #60]
-; CHECK-NEXT:    ldrd r0, r1, [sp, #56]
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s16
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s20
-; CHECK-NEXT:    vstr s2, [sp, #48]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #52]
-; CHECK-NEXT:    ldrd r0, r1, [sp, #48]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmov.16 q6[0], r4
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s17
-; CHECK-NEXT:    vstr s0, [sp, #40]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
-; CHECK-NEXT:    vstr s0, [sp, #44]
 ; CHECK-NEXT:    vmov.16 q6[1], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #40]
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s21
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s17
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s21
-; CHECK-NEXT:    vstr s2, [sp, #32]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #36]
 ; CHECK-NEXT:    vmov.16 q6[2], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #32]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s18
-; CHECK-NEXT:    vstr s0, [sp, #24]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
-; CHECK-NEXT:    vstr s0, [sp, #28]
 ; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #24]
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s22
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s18
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s22
-; CHECK-NEXT:    vstr s2, [sp, #16]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #20]
 ; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #16]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s19
-; CHECK-NEXT:    vstr s0, [sp, #8]
-; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
-; CHECK-NEXT:    vstr s0, [sp, #12]
 ; CHECK-NEXT:    vmov.16 q6[5], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp, #8]
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s23
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vmovx.f16 s2, s19
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s23
-; CHECK-NEXT:    vstr s2, [sp]
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [sp, #4]
 ; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    ldrd r0, r1, [sp]
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    vmov s0, r0
 ; CHECK-NEXT:    vcvtb.f16.f32 s0, s0
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q6[7], r0
 ; CHECK-NEXT:    vmov q0, q6
-; CHECK-NEXT:    add sp, #64
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = call fast <8 x half> @llvm.pow.v8f16(<8 x half> %src1, <8 x half> %src2)
   ret <8 x half> %0
 }
 
 define arm_aapcs_vfpcc <2 x double> @pow_float64_t(<2 x double> %src1, <2 x double> %src2) {

 entry:
   %0 = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %src1, <2 x double> %src2)
   ret <2 x double> %0
 }
 
 define arm_aapcs_vfpcc <4 x float> @copysign_float32_t(<4 x float> %src1, <4 x float> %src2) {
 ; CHECK-LABEL: copysign_float32_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    vstr s5, [sp, #8]
-; CHECK-NEXT:    ldr.w r12, [sp, #8]
-; CHECK-NEXT:    vstr s6, [sp, #16]
-; CHECK-NEXT:    ldr.w lr, [sp, #16]
-; CHECK-NEXT:    vstr s7, [sp, #24]
-; CHECK-NEXT:    lsr.w r2, r12, #31
-; CHECK-NEXT:    ldr r6, [sp, #24]
-; CHECK-NEXT:    vstr s3, [sp, #28]
-; CHECK-NEXT:    ldr r3, [sp, #28]
-; CHECK-NEXT:    vstr s4, [sp]
-; CHECK-NEXT:    ldr r0, [sp]
-; CHECK-NEXT:    vstr s0, [sp, #4]
-; CHECK-NEXT:    ldr r1, [sp, #4]
-; CHECK-NEXT:    vstr s1, [sp, #12]
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov lr, s6
+; CHECK-NEXT:    vmov r12, s7
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r4, s4
+; CHECK-NEXT:    vmov r5, s0
 ; CHECK-NEXT:    lsrs r0, r0, #31
-; CHECK-NEXT:    vstr s2, [sp, #20]
+; CHECK-NEXT:    bfi r3, r0, #31, #1
+; CHECK-NEXT:    lsr.w r0, lr, #31
+; CHECK-NEXT:    bfi r2, r0, #31, #1
+; CHECK-NEXT:    lsr.w r0, r12, #31
 ; CHECK-NEXT:    bfi r1, r0, #31, #1
-; CHECK-NEXT:    ldr r4, [sp, #12]
-; CHECK-NEXT:    ldr r5, [sp, #20]
-; CHECK-NEXT:    bfi r4, r2, #31, #1
-; CHECK-NEXT:    lsr.w r2, lr, #31
-; CHECK-NEXT:    bfi r5, r2, #31, #1
-; CHECK-NEXT:    lsrs r2, r6, #31
-; CHECK-NEXT:    bfi r3, r2, #31, #1
-; CHECK-NEXT:    vmov s3, r3
-; CHECK-NEXT:    vmov s2, r5
-; CHECK-NEXT:    vmov s1, r4
-; CHECK-NEXT:    vmov s0, r1
-; CHECK-NEXT:    add sp, #32
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    vmov s3, r1
+; CHECK-NEXT:    lsrs r0, r4, #31
+; CHECK-NEXT:    vmov s2, r2
+; CHECK-NEXT:    bfi r5, r0, #31, #1
+; CHECK-NEXT:    vmov s1, r3
+; CHECK-NEXT:    vmov s0, r5
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call fast <4 x float> @llvm.copysign.v4f32(<4 x float> %src1, <4 x float> %src2)
   ret <4 x float> %0
 }
 
 define arm_aapcs_vfpcc <8 x half> @copysign_float16_t(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-LABEL: copysign_float16_t:
 ; CHECK:       @ %bb.0: @ %entry

llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll

entry:
   ret void
 }
 
 define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%src) {
 ; CHECK-LABEL: foo_v4f32_v4f16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrh.s32 q0, [r1]
 ; CHECK-NEXT:    mov.w lr, #0
 ; CHECK-NEXT:    @ implicit-def: $q1
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    vmrs r3, p0
 ; CHECK-NEXT:    and r1, r3, #1
 ; CHECK-NEXT:    rsb.w r12, r1, #0
 ; CHECK-NEXT:    ubfx r1, r3, #4, #1

 ; CHECK-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-NEXT:    rsbs r2, r2, #0
 ; CHECK-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    ittt ne
-; CHECK-NEXT:    vstrne s0, [sp, #12]
-; CHECK-NEXT:    ldrne r2, [sp, #12]
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    vmovne r2, s0
 ; CHECK-NEXT:    strne r2, [r0]
 ; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    ittt mi
-; CHECK-NEXT:    vstrmi s1, [sp, #8]
-; CHECK-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    vmovmi r2, s1
 ; CHECK-NEXT:    strmi r2, [r0, #4]
 ; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    ittt mi
-; CHECK-NEXT:    vstrmi s2, [sp, #4]
-; CHECK-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    vmovmi r2, s2
 ; CHECK-NEXT:    strmi r2, [r0, #8]
 ; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    ittt mi
-; CHECK-NEXT:    vstrmi s3, [sp]
-; CHECK-NEXT:    ldrmi r1, [sp]
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    vmovmi r1, s3
 ; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i16>, <4 x i16>* %mask, align 2
   %1 = icmp sgt <4 x i16> %0, zeroinitializer
   %2 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %src, i32 2, <4 x i1> %1, <4 x half> undef)
   %3 = fpext <4 x half> %2 to <4 x float>
   call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %3, <4 x float>* %dest, i32 2, <4 x i1> %1)
   ret void
 }
 
 define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%src) {
 ; CHECK-LABEL: foo_v4f32_v4f16_unaligned:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrh.s32 q0, [r1]
 ; CHECK-NEXT:    mov.w lr, #0
 ; CHECK-NEXT:    @ implicit-def: $q1
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    vmrs r3, p0
 ; CHECK-NEXT:    and r1, r3, #1
 ; CHECK-NEXT:    rsb.w r12, r1, #0
 ; CHECK-NEXT:    ubfx r1, r3, #4, #1

 ; CHECK-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-NEXT:    rsbs r2, r2, #0
 ; CHECK-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    ittt ne
-; CHECK-NEXT:    vstrne s0, [sp, #12]
-; CHECK-NEXT:    ldrne r2, [sp, #12]
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    vmovne r2, s0
 ; CHECK-NEXT:    strne r2, [r0]
 ; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    ittt mi
-; CHECK-NEXT:    vstrmi s1, [sp, #8]
-; CHECK-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    vmovmi r2, s1
 ; CHECK-NEXT:    strmi r2, [r0, #4]
 ; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    ittt mi
-; CHECK-NEXT:    vstrmi s2, [sp, #4]
-; CHECK-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    vmovmi r2, s2
 ; CHECK-NEXT:    strmi r2, [r0, #8]
 ; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    ittt mi
-; CHECK-NEXT:    vstrmi s3, [sp]
-; CHECK-NEXT:    ldrmi r1, [sp]
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    vmovmi r1, s3
 ; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i16>, <4 x i16>* %mask, align 2
   %1 = icmp sgt <4 x i16> %0, zeroinitializer
   %2 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %src, i32 2, <4 x i1> %1, <4 x half> undef)
   %3 = fpext <4 x half> %2 to <4 x float>
   call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %3, <4 x float>* %dest, i32 1, <4 x i1> %1)
   ret void

llvm/test/CodeGen/Thumb2/mve-masked-store.ll

entry:
   %c = icmp ugt <4 x i32> %b, zeroinitializer
   call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %a, <4 x float>* %dest, i32 4, <4 x i1> %c)
   ret void
 }
 
 define arm_aapcs_vfpcc void @masked_v4f32_align1(<4 x float> *%dest, <4 x float> %a, <4 x i32> %b) {
 ; CHECK-LE-LABEL: masked_v4f32_align1:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #20
-; CHECK-LE-NEXT:    sub sp, #20
+; CHECK-LE-NEXT:    .pad #4
+; CHECK-LE-NEXT:    sub sp, #4
 ; CHECK-LE-NEXT:    vcmp.i32 ne, q1, zr
 ; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    vmrs r2, p0
 ; CHECK-LE-NEXT:    and r3, r2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
 ; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
 ; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    ittt ne
-; CHECK-LE-NEXT:    vstrne s0, [sp, #12]
-; CHECK-LE-NEXT:    ldrne r2, [sp, #12]
+; CHECK-LE-NEXT:    itt ne
+; CHECK-LE-NEXT:    vmovne r2, s0
 ; CHECK-LE-NEXT:    strne r2, [r0]
 ; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    ittt mi
-; CHECK-LE-NEXT:    vstrmi s1, [sp, #8]
-; CHECK-LE-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s1
 ; CHECK-LE-NEXT:    strmi r2, [r0, #4]
 ; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    ittt mi
-; CHECK-LE-NEXT:    vstrmi s2, [sp, #4]
-; CHECK-LE-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r2, s2
 ; CHECK-LE-NEXT:    strmi r2, [r0, #8]
 ; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    ittt mi
-; CHECK-LE-NEXT:    vstrmi s3, [sp]
-; CHECK-LE-NEXT:    ldrmi r1, [sp]
+; CHECK-LE-NEXT:    itt mi
+; CHECK-LE-NEXT:    vmovmi r1, s3
 ; CHECK-LE-NEXT:    strmi r1, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #20
+; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_align1:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #20
-; CHECK-BE-NEXT:    sub sp, #20
+; CHECK-BE-NEXT:    .pad #4
+; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    vcmp.i32 ne, q2, zr
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
 ; CHECK-BE-NEXT:    vmrs r2, p0
 ; CHECK-BE-NEXT:    and r3, r2, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
 ; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    ittt ne
-; CHECK-BE-NEXT:    vstrne s4, [sp, #12]
-; CHECK-BE-NEXT:    ldrne r2, [sp, #12]
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r2, s4
 ; CHECK-BE-NEXT:    strne r2, [r0]
 ; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    ittt mi
-; CHECK-BE-NEXT:    vstrmi s5, [sp, #8]
-; CHECK-BE-NEXT:    ldrmi r2, [sp, #8]
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    ittt mi
-; CHECK-BE-NEXT:    vstrmi s6, [sp, #4]
-; CHECK-BE-NEXT:    ldrmi r2, [sp, #4]
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    ittt mi
-; CHECK-BE-NEXT:    vstrmi s7, [sp]
-; CHECK-BE-NEXT:    ldrmi r1, [sp]
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r1, s7
 ; CHECK-BE-NEXT:    strmi r1, [r0, #12]
-; CHECK-BE-NEXT:    add sp, #20
+; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp ugt <4 x i32> %b, zeroinitializer
   call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %a, <4 x float>* %dest, i32 1, <4 x i1> %c)
   ret void
 }
 
 define i8* @masked_v4f32_pre(i8* %y, i8* %x, <4 x i32> %a) {

llvm/test/CodeGen/Thumb2/mve-phireg.ll

 @a = external dso_local global i32, align 4
 @b = dso_local local_unnamed_addr global i32 ptrtoint (i32* @a to i32), align 4
 @c = dso_local global i32 2, align 4
 @d = dso_local global i32 2, align 4
 
 define dso_local i32 @e() #0 {
 ; CHECK-LABEL: e:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    .pad #440
-; CHECK-NEXT:    sub sp, #440
-; CHECK-NEXT:    vldr s20, .LCPI1_0
+; CHECK-NEXT:    .pad #392
+; CHECK-NEXT:    sub sp, #392
 ; CHECK-NEXT:    movw r9, :lower16:.L_MergedGlobals
-; CHECK-NEXT:    vldr s23, .LCPI1_1
+; CHECK-NEXT:    vldr s0, .LCPI1_0
 ; CHECK-NEXT:    movt r9, :upper16:.L_MergedGlobals
-; CHECK-NEXT:    mov.w r8, #4
+; CHECK-NEXT:    vldr s3, .LCPI1_1
 ; CHECK-NEXT:    mov r5, r9
-; CHECK-NEXT:    strh.w r8, [sp, #438]
-; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    vstr s23, [sp, #48]
 ; CHECK-NEXT:    mov r7, r9
-; CHECK-NEXT:    vstr s23, [sp, #40]
+; CHECK-NEXT:    ldr r1, [r5, #8]!
+; CHECK-NEXT:    vmov r6, s3
+; CHECK-NEXT:    ldr r0, [r7, #4]!
 ; CHECK-NEXT:    movw r4, :lower16:e
-; CHECK-NEXT:    ldr r1, [r5, #4]!
+; CHECK-NEXT:    vmov.32 q4[0], r5
 ; CHECK-NEXT:    movt r4, :upper16:e
-; CHECK-NEXT:    str r6, [sp, #76]
-; CHECK-NEXT:    vmov s5, r4
-; CHECK-NEXT:    vmov.32 q7[0], r5
-; CHECK-NEXT:    ldr r0, [r7, #8]!
-; CHECK-NEXT:    vmov q0, q7
-; CHECK-NEXT:    ldr r2, [sp, #48]
-; CHECK-NEXT:    vmov.32 q0[1], r5
-; CHECK-NEXT:    vmov s21, r5
-; CHECK-NEXT:    vmov.32 q0[2], r2
-; CHECK-NEXT:    ldr r2, [sp, #40]
-; CHECK-NEXT:    vdup.32 q2, r5
-; CHECK-NEXT:    vmov.32 q4[0], r7
-; CHECK-NEXT:    vmov q6, q4
-; CHECK-NEXT:    vstrw.32 q2, [sp] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 q6[1], r2
-; CHECK-NEXT:    vldrw.u32 q3, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s22, s21
-; CHECK-NEXT:    vmov.32 q6[2], r7
-; CHECK-NEXT:    vmov.f32 s4, s20
-; CHECK-NEXT:    vstrw.32 q2, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT:    vmov.f32 s6, s21
+; CHECK-NEXT:    vmov q1, q4
+; CHECK-NEXT:    vmov s1, r7
+; CHECK-NEXT:    vmov.32 q1[1], r6
+; CHECK-NEXT:    mov.w r10, #0
+; CHECK-NEXT:    vmov.32 q1[2], r5
+; CHECK-NEXT:    vmov.32 q5[0], r7
+; CHECK-NEXT:    vmov.32 q1[3], r4
+; CHECK-NEXT:    strd r0, r10, [sp, #24]
+; CHECK-NEXT:    vstrw.32 q1, [sp, #76]
+; CHECK-NEXT:    vmov q1, q5
+; CHECK-NEXT:    vmov s9, r4
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vdup.32 q6, r7
+; CHECK-NEXT:    vmov.f32 s2, s1
+; CHECK-NEXT:    vmov.f32 s8, s0
+; CHECK-NEXT:    vmov.32 q1[2], r6
+; CHECK-NEXT:    vmov q3, q6
+; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    vmov.f32 s10, s1
+; CHECK-NEXT:    mov.w r8, #4
+; CHECK-NEXT:    vmov.32 q1[3], r4
 ; CHECK-NEXT:    vmov.32 q3[0], r4
-; CHECK-NEXT:    vmov.32 q2[1], r4
-; CHECK-NEXT:    vmov.32 q6[3], r4
-; CHECK-NEXT:    vmov.32 q0[3], r4
-; CHECK-NEXT:    vmov.f32 s7, s23
-; CHECK-NEXT:    str r1, [sp, #72]
-; CHECK-NEXT:    vstrw.32 q6, [sp, #124]
+; CHECK-NEXT:    vmov.32 q7[1], r4
 ; CHECK-NEXT:    str r1, [r0]
+; CHECK-NEXT:    vmov.f32 s11, s3
 ; CHECK-NEXT:    movs r1, #64
+; CHECK-NEXT:    strh.w r8, [sp, #390]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #44]
 ; CHECK-NEXT:    str r0, [r0]
-; CHECK-NEXT:    vstrw.32 q5, [sp, #92]
-; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    vstrw.32 q2, [r0]
+; CHECK-NEXT:    vstrw.32 q7, [r0]
 ; CHECK-NEXT:    vstrw.32 q3, [r0]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bl __aeabi_memclr4
-; CHECK-NEXT:    vstr s23, [sp, #44]
-; CHECK-NEXT:    vmov.32 q7[1], r7
-; CHECK-NEXT:    ldr r0, [sp, #44]
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 q5[1], r5
 ; CHECK-NEXT:    vmov.32 q4[1], r4
-; CHECK-NEXT:    vmov.32 q7[2], r5
-; CHECK-NEXT:    vmov.32 q4[2], r5
-; CHECK-NEXT:    vmov.32 q0[0], r6
-; CHECK-NEXT:    vmov.32 q4[3], r7
-; CHECK-NEXT:    vmov.32 q7[3], r0
+; CHECK-NEXT:    vmov.32 q5[2], r7
+; CHECK-NEXT:    vmov.32 q4[2], r7
+; CHECK-NEXT:    vmov.32 q5[3], r6
+; CHECK-NEXT:    vmov.32 q6[0], r10
+; CHECK-NEXT:    vmov.32 q4[3], r5
+; CHECK-NEXT:    str.w r10, [r9]
 ; CHECK-NEXT:    vstrw.32 q4, [r0]
-; CHECK-NEXT:    str.w r6, [r9]
-; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    vstrw.32 q7, [r0]
-; CHECK-NEXT:    str.w r8, [sp, #356]
+; CHECK-NEXT:    vstrw.32 q6, [r0]
+; CHECK-NEXT:    vstrw.32 q5, [r0]
+; CHECK-NEXT:    str.w r8, [sp, #308]
 ; CHECK-NEXT:  .LBB1_1: @ %for.cond
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    b .LBB1_1
 ; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:  .LCPI1_0:
 ; CHECK-NEXT:    .long 0x00000004 @ float 5.60519386E-45
 ; CHECK-NEXT:  .LCPI1_1: