[ARM] Favour post inc for MVE loops

[dmgreen]

Summary

We were previously not necessarily favouring postinc for the MVE loads and stores, leading to extra code prior to the loop to set up the preinc. MVE in general can benefit from postinc (as we don't have unrolled loops), and certain instructions like the VLD2's only offset post-inc versions.

[dmgreen]

Update code to what I was intending.

[SjoerdMeijer]

LGTM

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

67

nit: "modify unroll"? Perhaps clarify this a bit.

80

Is there a test for optsize?

[samparker]

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

1253

nice one. would you mind adding a test to show we no longer unroll loops with just mve intrinsics?

[dmgreen]

Sorry for the long delay here. This wasn't making things much better in the tests I was trying (it was a bit up-and-down). I've adjusted it now to include shouldFavorPostInc for MVE subtargets, not just disable shouldFavorBackedgeIndex. That should get the costmodel more correct in LSR, where the AddRec is now free because it can just be the postinc. I also removed the old "containsVectors(L)" check as adding something that is O(n) to the inner parts of LSR, something that is already O(something large), was probably a bad idea.

This means that it's just based on subtarget. I was hoping I could add Type here and use that, but the only type we have in LSR is the type of the SCEV, not the type of the memory being loaded (i.e just a pointer, not a vector). My benchmarking shows this to be an improvement (even if a bit of an unreliable one). More so with D71194. The argument is that most simple loops are vectorized, not unrolled, so favouring post-inc is a slightly better alternative in general when we have MVE.

llvm/lib/Target/ARM/ARMTargetTransformInfo.h

explicit ARMTTIImpl(const ARMBaseTargetMachine *TM, const Function &F)
       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
 
   bool enableInterleavedAccessVectorization() { return true; }
 
-  bool shouldFavorBackedgeIndex(const Loop *L) const {
-    if (L->getHeader()->getParent()->hasOptSize())
-      return false;
-    return ST->isMClass() && ST->isThumb2() && L->getNumBlocks() == 1;
-  }
+  bool shouldFavorBackedgeIndex(const Loop *L) const;
 
   /// Floating-point computation using ARMv8 AArch32 Advanced
   /// SIMD instructions remains unchanged from ARMv7. Only AArch64 SIMD
   /// and Arm MVE are IEEE-754 compliant.
   bool isFPVectorizationPotentiallyUnsafe() {
     return !ST->isTargetDarwin() && !ST->hasMVEFloatOps();
   }
 

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

bool MatchExact = (CallerBits & ~InlineFeatureWhitelist) ==
                     (CalleeBits & ~InlineFeatureWhitelist);
   // For features in the whitelist, the callee's features must be a subset of
   // the callers'.
   bool MatchSubset = ((CallerBits & CalleeBits) & InlineFeatureWhitelist) ==
                      (CalleeBits & InlineFeatureWhitelist);
   return MatchExact && MatchSubset;
 }
 
+// Check if the loop contains any vector code, in which case we modify unroll
+// and favor backedge behaviour.
+static bool containsVectors(const Loop *L) {
+  for (auto *BB : L->getBlocks()) {
+    for (auto &I : *BB)
+      if (I.getType()->isVectorTy())
+        return true;
+  }
+
+  return false;
+}
+
+bool ARMTTIImpl::shouldFavorBackedgeIndex(const Loop *L) const {
+  if (L->getHeader()->getParent()->hasOptSize())
+    return false;
+  if (ST->hasMVEIntegerOps() && containsVectors(L))
+    return false;
+  return ST->isMClass() && ST->isThumb2() && L->getNumBlocks() == 1;
+}
+
 int ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty) {
   assert(Ty->isIntegerTy());
 
  unsigned Bits = Ty->getPrimitiveSizeInBits();
  if (Bits == 0 || Imm.getActiveBits() >= 64)
    return 4;
 
   int64_t SImmVal = Imm.getSExtValue();
void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   if (ExitingBlocks.size() > 2)
     return;
 
   // Limit the CFG of the loop body for targets with a branch predictor.
   // Allowing 4 blocks permits if-then-else diamonds in the body.
   if (ST->hasBranchPredictor() && L->getNumBlocks() > 4)
     return;
 
+  // Don't unroll vectorised loop. MVE does not benefit from it as much as
+  // scalar code.
+  if (ST->hasMVEIntegerOps() && containsVectors(L))
+    return;
+
   // Scan the loop: don't unroll loops with calls as this could prevent
   // inlining.
   unsigned Cost = 0;
   for (auto *BB : L->getBlocks()) {
     for (auto &I : *BB) {
       if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
         ImmutableCallSite CS(&I);
         if (const Function *F = CS.getCalledFunction()) {
           if (!isLoweredToCall(F))
             continue;
         }
         return;
       }
-      // Don't unroll vectorised loop. MVE does not benefit from it as much as
-      // scalar code.
-      if (I.getType()->isVectorTy())
-        return;
 
       SmallVector<const Value*, 4> Operands(I.value_op_begin(),
                                             I.value_op_end());
       Cost += getUserCost(&I, Operands);
     }
   }
 
   LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll

 ; CHECK-NEXT:    vstr s0, [r9, #16]
 ; CHECK-NEXT:    bne .LBB0_9
 ; CHECK-NEXT:  .LBB0_10: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:  .LBB0_11: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r6, #1
 ; CHECK-NEXT:    sub.w r7, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #16
-; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB0_12: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
-; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vldrw.u32 q0, [r5], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r4], #16
 ; CHECK-NEXT:    vmul.f32 q0, q1, q0
-; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q0, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB0_12
 ; CHECK-NEXT:  @ %bb.13: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    bne.w .LBB0_4
 ; CHECK-NEXT:    b .LBB0_10
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

 ; CHECK-NEXT:    vstr s0, [r9, #16]
 ; CHECK-NEXT:    bne .LBB1_9
 ; CHECK-NEXT:  .LBB1_10: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:  .LBB1_11: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r6, #1
 ; CHECK-NEXT:    sub.w r7, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #16
-; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB1_12: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
-; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vldrw.u32 q0, [r5], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r4], #16
 ; CHECK-NEXT:    vadd.f32 q0, q1, q0
-; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q0, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB1_12
 ; CHECK-NEXT:  @ %bb.13: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    bne.w .LBB1_4
 ; CHECK-NEXT:    b .LBB1_10
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

 ; CHECK-NEXT:    vstr s0, [r9, #16]
 ; CHECK-NEXT:    bne .LBB2_9
 ; CHECK-NEXT:  .LBB2_10: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:  .LBB2_11: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r6, #1
 ; CHECK-NEXT:    sub.w r7, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #16
-; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB2_12: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
-; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vldrw.u32 q0, [r5], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r4], #16
 ; CHECK-NEXT:    vsub.f32 q0, q1, q0
-; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q0, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB2_12
 ; CHECK-NEXT:  @ %bb.13: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    bne.w .LBB2_4
 ; CHECK-NEXT:    b .LBB2_10
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

 ; CHECK-NEXT:    itt hi
 ; CHECK-NEXT:    addhi.w r7, r2, r3, lsl #2
 ; CHECK-NEXT:    cmphi r7, r0
 ; CHECK-NEXT:    bhi .LBB3_6
 ; CHECK-NEXT:  @ %bb.3: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r6, #1
 ; CHECK-NEXT:    sub.w r7, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #16
-; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB3_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
-; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vldrw.u32 q0, [r5], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r4], #16
 ; CHECK-NEXT:    vcvt.f32.s32 q0, q0
 ; CHECK-NEXT:    vmul.f32 q0, q1, q0
-; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q0, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB3_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    bne .LBB3_7
 ; CHECK-NEXT:    b .LBB3_13
 ; CHECK-NEXT:  .LBB3_6:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:  .LBB3_7: @ %for.body.preheader16

 ; CHECK-NEXT:    bhi .LBB4_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB4_6
 ; CHECK-NEXT:  .LBB4_3: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r5, #1
 ; CHECK-NEXT:    sub.w r6, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
-; CHECK-NEXT:    sub.w r5, r1, #16
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB4_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [r4, #16]!
-; CHECK-NEXT:    vldrw.u32 q1, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q0, [r4], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r5], #16
 ; CHECK-NEXT:    vmul.i32 q0, q1, q0
 ; CHECK-NEXT:    vcvt.f32.s32 q0, q0
-; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q0, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB4_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r4, r5, r6, pc}
 ; CHECK-NEXT:  .LBB4_6: @ %for.body.preheader11
 ; CHECK-NEXT:    sub.w lr, r3, r12
 ; CHECK-NEXT:    mvn r3, #3
for.body:                                         ; preds = %for.body.preheader11, %for.body
   %inc = add nuw i32 %i.09, 1
   %exitcond = icmp eq i32 %inc, %N
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
 define arm_aapcs_vfpcc void @half_half_mul(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
 ; CHECK-LABEL: half_half_mul:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq .LBB5_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB5_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB5_6
 ; CHECK-NEXT:  .LBB5_3: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r5, #1
 ; CHECK-NEXT:    sub.w r6, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
-; CHECK-NEXT:    sub.w r5, r1, #8
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB5_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r8, [r5, #8]!
-; CHECK-NEXT:    ldr r7, [r4, #8]!
-; CHECK-NEXT:    vmov.32 q1[0], r8
-; CHECK-NEXT:    vmov.32 q0[0], r7
-; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r9, [r4]
+; CHECK-NEXT:    ldr r7, [r5]
 ; CHECK-NEXT:    ldr.w r8, [r4, #4]
-; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[0], r9
+; CHECK-NEXT:    ldr.w r10, [r5, #4]
+; CHECK-NEXT:    vmov.32 q1[0], r7
 ; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    adds r4, #8
+; CHECK-NEXT:    vmov.32 q1[1], r10
+; CHECK-NEXT:    adds r5, #8
 ; CHECK-NEXT:    vmul.f16 q0, q0, q1
 ; CHECK-NEXT:    vmovx.f16 s6, s1
 ; CHECK-NEXT:    vmovx.f16 s4, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
 ; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
 ; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q2, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB5_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    beq .LBB5_8
 ; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader11
 ; CHECK-NEXT:    sub.w lr, r3, r12
 ; CHECK-NEXT:    sub.w r3, r12, #1
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
 ; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
 ; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
 ; CHECK-NEXT:  .LBB5_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s0, [r1, #2]
 ; CHECK-NEXT:    vldr.16 s2, [r0, #2]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    adds r1, #2
 ; CHECK-NEXT:    vmul.f16 s0, s2, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [r2, #4]
 ; CHECK-NEXT:    adds r2, #4
 ; CHECK-NEXT:    le lr, .LBB5_7
 ; CHECK-NEXT:  .LBB5_8: @ %for.cond.cleanup
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %N, 4
   br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
 
for.body:                                         ; preds = %for.body.preheader11, %for.body
   %inc = add nuw i32 %i.09, 1
   %exitcond = icmp eq i32 %inc, %N
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
 define arm_aapcs_vfpcc void @half_half_add(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
 ; CHECK-LABEL: half_half_add:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq .LBB6_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB6_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB6_6
 ; CHECK-NEXT:  .LBB6_3: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r5, #1
 ; CHECK-NEXT:    sub.w r6, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
-; CHECK-NEXT:    sub.w r5, r1, #8
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB6_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r8, [r5, #8]!
-; CHECK-NEXT:    ldr r7, [r4, #8]!
-; CHECK-NEXT:    vmov.32 q1[0], r8
-; CHECK-NEXT:    vmov.32 q0[0], r7
-; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r9, [r4]
+; CHECK-NEXT:    ldr r7, [r5]
 ; CHECK-NEXT:    ldr.w r8, [r4, #4]
-; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[0], r9
+; CHECK-NEXT:    ldr.w r10, [r5, #4]
+; CHECK-NEXT:    vmov.32 q1[0], r7
 ; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    adds r4, #8
+; CHECK-NEXT:    vmov.32 q1[1], r10
+; CHECK-NEXT:    adds r5, #8
 ; CHECK-NEXT:    vadd.f16 q0, q0, q1
 ; CHECK-NEXT:    vmovx.f16 s6, s1
 ; CHECK-NEXT:    vmovx.f16 s4, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
 ; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
 ; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q2, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB6_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    beq .LBB6_8
 ; CHECK-NEXT:  .LBB6_6: @ %for.body.preheader11
 ; CHECK-NEXT:    sub.w lr, r3, r12
 ; CHECK-NEXT:    sub.w r3, r12, #1
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
 ; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
 ; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
 ; CHECK-NEXT:  .LBB6_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s0, [r1, #2]
 ; CHECK-NEXT:    vldr.16 s2, [r0, #2]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    adds r1, #2
 ; CHECK-NEXT:    vadd.f16 s0, s2, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [r2, #4]
 ; CHECK-NEXT:    adds r2, #4
 ; CHECK-NEXT:    le lr, .LBB6_7
 ; CHECK-NEXT:  .LBB6_8: @ %for.cond.cleanup
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %N, 4
   br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
 
for.body:                                         ; preds = %for.body.preheader11, %for.body
   %inc = add nuw i32 %i.09, 1
   %exitcond = icmp eq i32 %inc, %N
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
 define arm_aapcs_vfpcc void @half_half_sub(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
 ; CHECK-LABEL: half_half_sub:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq .LBB7_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB7_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB7_6
 ; CHECK-NEXT:  .LBB7_3: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r5, #1
 ; CHECK-NEXT:    sub.w r6, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
-; CHECK-NEXT:    sub.w r5, r1, #8
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB7_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r8, [r5, #8]!
-; CHECK-NEXT:    ldr r7, [r4, #8]!
-; CHECK-NEXT:    vmov.32 q1[0], r8
-; CHECK-NEXT:    vmov.32 q0[0], r7
-; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r9, [r4]
+; CHECK-NEXT:    ldr r7, [r5]
 ; CHECK-NEXT:    ldr.w r8, [r4, #4]
-; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[0], r9
+; CHECK-NEXT:    ldr.w r10, [r5, #4]
+; CHECK-NEXT:    vmov.32 q1[0], r7
 ; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    adds r4, #8
+; CHECK-NEXT:    vmov.32 q1[1], r10
+; CHECK-NEXT:    adds r5, #8
 ; CHECK-NEXT:    vsub.f16 q0, q0, q1
 ; CHECK-NEXT:    vmovx.f16 s6, s1
 ; CHECK-NEXT:    vmovx.f16 s4, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
 ; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
 ; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q2, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB7_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    beq .LBB7_8
 ; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader11
 ; CHECK-NEXT:    sub.w lr, r3, r12
 ; CHECK-NEXT:    sub.w r3, r12, #1
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
 ; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
 ; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
 ; CHECK-NEXT:  .LBB7_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s0, [r1, #2]
 ; CHECK-NEXT:    vldr.16 s2, [r0, #2]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    adds r1, #2
 ; CHECK-NEXT:    vsub.f16 s0, s2, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [r2, #4]
 ; CHECK-NEXT:    adds r2, #4
 ; CHECK-NEXT:    le lr, .LBB7_7
 ; CHECK-NEXT:  .LBB7_8: @ %for.cond.cleanup
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %N, 4
   br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
 
for.body:                                         ; preds = %for.body.preheader11, %for.body
   %inc = add nuw i32 %i.09, 1
   %exitcond = icmp eq i32 %inc, %N
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 }
 
 define arm_aapcs_vfpcc void @half_short_mul(half* nocapture readonly %a, i16* nocapture readonly %b, float* nocapture %c, i32 %N) {
 ; CHECK-LABEL: half_short_mul:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq .LBB8_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB8_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB8_6
 ; CHECK-NEXT:  .LBB8_3: @ %vector.ph
 ; CHECK-NEXT:    bic r12, r3, #3
 ; CHECK-NEXT:    movs r5, #1
 ; CHECK-NEXT:    sub.w r6, r12, #4
-; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
-; CHECK-NEXT:    sub.w r5, r1, #8
-; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB8_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r8, [r4, #8]!
-; CHECK-NEXT:    vldrh.u32 q0, [r5, #8]!
+; CHECK-NEXT:    ldr.w r9, [r4]
+; CHECK-NEXT:    ldr.w r8, [r4, #4]
+; CHECK-NEXT:    vldrh.u32 q0, [r5], #8
+; CHECK-NEXT:    adds r4, #8
 ; CHECK-NEXT:    vmov r7, s0
 ; CHECK-NEXT:    vmov.16 q1[0], r7
 ; CHECK-NEXT:    vmov r7, s1
 ; CHECK-NEXT:    vmov.16 q1[1], r7
 ; CHECK-NEXT:    vmov r7, s2
 ; CHECK-NEXT:    vmov.16 q1[2], r7
 ; CHECK-NEXT:    vmov r7, s3
 ; CHECK-NEXT:    vmov.16 q1[3], r7
-; CHECK-NEXT:    ldr r7, [r4, #4]
 ; CHECK-NEXT:    vcvt.f16.s16 q0, q1
-; CHECK-NEXT:    vmov.32 q1[0], r8
-; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q1[0], r9
+; CHECK-NEXT:    vmov.32 q1[1], r8
 ; CHECK-NEXT:    vmul.f16 q0, q1, q0
 ; CHECK-NEXT:    vmovx.f16 s6, s1
 ; CHECK-NEXT:    vmovx.f16 s4, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
 ; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
 ; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
-; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    vstrb.8 q2, [r6], #16
 ; CHECK-NEXT:    le lr, .LBB8_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r12, r3
 ; CHECK-NEXT:    beq .LBB8_8
 ; CHECK-NEXT:  .LBB8_6: @ %for.body.preheader13
 ; CHECK-NEXT:    sub.w lr, r3, r12
 ; CHECK-NEXT:    sub.w r3, r12, #1
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
 ; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
 ; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
 ; CHECK-NEXT:  .LBB8_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsh r3, [r1, #2]!
 ; CHECK-NEXT:    vldr.16 s0, [r0, #2]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    vmov s2, r3
 ; CHECK-NEXT:    vcvt.f16.s32 s2, s2
 ; CHECK-NEXT:    vmul.f16 s0, s0, s2
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
 ; CHECK-NEXT:    vstr s0, [r2, #4]
 ; CHECK-NEXT:    adds r2, #4
 ; CHECK-NEXT:    le lr, .LBB8_7
 ; CHECK-NEXT:  .LBB8_8: @ %for.cond.cleanup
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
 entry:
   %cmp10 = icmp eq i32 %N, 0
   br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %N, 4
   br i1 %min.iters.check, label %for.body.preheader13, label %vector.ph
 

llvm/test/CodeGen/Thumb2/mve-vmla.ll

entry:
   %2 = mul nsw <16 x i8> %1, %B
   %3 = add nsw <16 x i8> %2, %A
   ret <16 x i8> %3
 }
 
 define void @vmla32_in_loop(i32* %s1, i32 %x, i32* %d, i32 %n) {
 ; CHECK-LABEL: vmla32_in_loop:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    subs r0, #16
-; CHECK-NEXT:    subs r2, #16
 ; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]!
-; CHECK-NEXT:    vldrw.u32 q1, [r2, #16]!
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
 ; CHECK-NEXT:    subs r3, #4
 ; CHECK-NEXT:    vmla.u32 q1, q0, r1
-; CHECK-NEXT:    vstrw.32 q1, [r2]
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
 entry:
   %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %x, i32 0
   %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
   br label %vector.body
 

 
 for.cond.cleanup:
   ret void
 }
 
 define void @vmla16_in_loop(i16* %s1, i16 %x, i16* %d, i32 %n) {
 ; CHECK-LABEL: vmla16_in_loop:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    subs r0, #16
-; CHECK-NEXT:    subs r2, #16
 ; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]!
-; CHECK-NEXT:    vldrh.u16 q1, [r2, #16]!
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
 ; CHECK-NEXT:    subs r3, #8
 ; CHECK-NEXT:    vmla.u16 q1, q0, r1
-; CHECK-NEXT:    vstrh.16 q1, [r2]
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
 entry:
   %broadcast.splatinsert11 = insertelement <8 x i16> undef, i16 %x, i32 0
   %broadcast.splat12 = shufflevector <8 x i16> %broadcast.splatinsert11, <8 x i16> undef, <8 x i32> zeroinitializer
   br label %vector.body
 

 
 for.cond.cleanup:
   ret void
 }
 
 define void @vmla8_in_loop(i8* %s1, i8 %x, i8* %d, i32 %n) {
 ; CHECK-LABEL: vmla8_in_loop:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    subs r0, #16
-; CHECK-NEXT:    subs r2, #16
 ; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]!
-; CHECK-NEXT:    vldrh.u16 q1, [r2, #16]!
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r2]
 ; CHECK-NEXT:    subs r3, #16
 ; CHECK-NEXT:    vmla.u8 q1, q0, r1
-; CHECK-NEXT:    vstrh.16 q1, [r2]
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
 entry:
   %broadcast.splatinsert11 = insertelement <16 x i8> undef, i8 %x, i32 0
   %broadcast.splat12 = shufflevector <16 x i8> %broadcast.splatinsert11, <16 x i8> undef, <16 x i32> zeroinitializer
   br label %vector.body