[ARM] Extend search for increment in load/store optimizer

[dmgreen]

Summary

Currently the findIncDecAfter will only look at the next instruction for post-inc candidates in the load/store optimizer. This extends that to a search through the current BB, until an instruction that modifies or uses the increment reg is found. This allows more post-inc load/stores and ldm/stm's to be created, especially in cases where a schedule might move instructions further apart.

[dmgreen]

ping

[SjoerdMeijer]

LGTM

[simon_tatham]

What happens about a case like the following:

mov r0, sp
ldrd r1, r2, [sp]
ldr r3, [r0]
add sp, sp, #8

In this case, the LDR of r3 is not directly reading or modifying SP, but it is nonetheless dependent on SP not having been incremented yet, because it's accessing the stack slot that becomes invalid as soon as SP is increased. I don't see any check for that situation here.

[dmgreen]

Ah yeah. Good point, because it may handle an exception on the same stack. I had originally believed that this Post-RA Machine scheduler was running after this, and it could do the same kind of re-arranging. Looking again it seems that it's treating the stack pointer as a scheduling barrier.

The SP was not my motivating case. I'll can easily remove it from this patch.

[dmgreen]

Don't continue iterating with SP.

[simon_tatham]

I was lucky to notice it at all – I only spotted it because one of the test changes in the previous version of the patch did lift an increment of SP past two load instructions. As best I could tell it wasn't harmful in that particular case, but that clued me in to look more closely and see whether it could have been harmful in another case.

LGTM with the SP exemption, thanks.

llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp

1262

(spelling nit, if you can be bothered: it's → its)

[dmgreen]

Thanks.

llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp

findIncDecBefore(MachineBasicBlock::iterator MBBI, Register Reg,
 
   Offset = isIncrementOrDecrement(*PrevMBBI, Reg, Pred, PredReg);
   return Offset == 0 ? EndMBBI : PrevMBBI;
 }
 
 /// Searches for a increment or decrement of \p Reg after \p MBBI.
 static MachineBasicBlock::iterator
 findIncDecAfter(MachineBasicBlock::iterator MBBI, Register Reg,
-                ARMCC::CondCodes Pred, Register PredReg, int &Offset) {
+                ARMCC::CondCodes Pred, Register PredReg, int &Offset,
+                const TargetRegisterInfo *TRI) {
   Offset = 0;
   MachineBasicBlock &MBB = *MBBI->getParent();
   MachineBasicBlock::iterator EndMBBI = MBB.end();
   MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
+  while (NextMBBI != EndMBBI) {
     // Skip debug values.
     while (NextMBBI != EndMBBI && NextMBBI->isDebugInstr())
       ++NextMBBI;
     if (NextMBBI == EndMBBI)
       return EndMBBI;
 
-  Offset = isIncrementOrDecrement(*NextMBBI, Reg, Pred, PredReg);
-  return Offset == 0 ? EndMBBI : NextMBBI;
+    unsigned Off = isIncrementOrDecrement(*NextMBBI, Reg, Pred, PredReg);
+    if (Off) {
+      Offset = Off;
+      return NextMBBI;
+    }
+
+    // SP can only be combined if it is the next instruction after the original
+    // MBBI, otherwise we may be incrementing the stack pointer (invalidating
+    // anything below the new pointer) when it's frame elements are still in
+    // use. Other registers can attempt to look further, until a different use
+    // or def of the register is found.
+    if (Reg == ARM::SP || NextMBBI->readsRegister(Reg, TRI) ||
+        NextMBBI->definesRegister(Reg, TRI))
+      return EndMBBI;
+
+    ++NextMBBI;
+  }
+  return EndMBBI;
 }
 
 /// Fold proceeding/trailing inc/dec of base register into the
 /// LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible:
 ///
 /// stmia rn, <ra, rb, rc>
 /// rn := rn + 4 * 3;
 /// =>
bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineInstr *MI) {
   MachineBasicBlock::iterator MergeInstr
     = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
   ARM_AM::AMSubMode Mode = getLoadStoreMultipleSubMode(Opcode);
   if (Mode == ARM_AM::ia && Offset == -Bytes) {
     Mode = ARM_AM::db;
   } else if (Mode == ARM_AM::ib && Offset == -Bytes) {
     Mode = ARM_AM::da;
   } else {
-    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
+    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset, TRI);
     if (((Mode != ARM_AM::ia && Mode != ARM_AM::ib) || Offset != Bytes) &&
         ((Mode != ARM_AM::da && Mode != ARM_AM::db) || Offset != -Bytes)) {
 
       // We couldn't find an inc/dec to merge. But if the base is dead, we
       // can still change to a writeback form as that will save us 2 bytes
       // of code size. It can create WAW hazards though, so only do it if
       // we're minimizing code size.
       if (!STI->hasMinSize() || !BaseKill)
bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineInstr *MI) {
   MachineBasicBlock::iterator MergeInstr
     = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
   unsigned NewOpc;
   if (!isAM5 && Offset == Bytes) {
     NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
   } else if (Offset == -Bytes) {
     NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
   } else {
-    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
+    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset, TRI);
     if (Offset == Bytes) {
       NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
     } else if (!isAM5 && Offset == -Bytes) {
       NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
     } else
       return false;
   }
   LLVM_DEBUG(dbgs() << "  Erasing old increment: " << *MergeInstr);
bool ARMLoadStoreOpt::MergeBaseUpdateLSDouble(MachineInstr &MI) const {
   MachineBasicBlock &MBB = *MI.getParent();
   int Offset;
   MachineBasicBlock::iterator MergeInstr = findIncDecBefore(MBBI, Base, Pred,
                                                             PredReg, Offset);
   unsigned NewOpc;
   if (Offset == 8 || Offset == -8) {
     NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_PRE : ARM::t2STRD_PRE;
   } else {
-    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
+    MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset, TRI);
     if (Offset == 8 || Offset == -8) {
       NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_POST : ARM::t2STRD_POST;
     } else
       return false;
   }
   LLVM_DEBUG(dbgs() << "  Erasing old increment: " << *MergeInstr);
   MBB.erase(MergeInstr);
 

llvm/test/CodeGen/ARM/indexed-mem.ll

; CHECK-V8A-NEXT:    bx lr
   store i32 %res, i32* %addr.2
   ret i32* %addr
 }
 
 ; TODO: Generate post inc
 define i32* @post_inc_ldrd(i32* %base, i32* %addr.3) {
 ; CHECK-V8M-LABEL: post_inc_ldrd:
 ; CHECK-V8M:       @ %bb.0:
-; CHECK-V8M-NEXT:    ldrd r2, r3, [r0]
-; CHECK-V8M-NEXT:    adds r0, #8
+; CHECK-V8M-NEXT:    ldrd r2, r3, [r0], #8
 ; CHECK-V8M-NEXT:    add r2, r3
 ; CHECK-V8M-NEXT:    str r2, [r1]
 ; CHECK-V8M-NEXT:    bx lr
 ;
 ; CHECK-V8A-LABEL: post_inc_ldrd:
 ; CHECK-V8A:       @ %bb.0:
-; CHECK-V8A-NEXT:    ldm r0, {r2, r3}
-; CHECK-V8A-NEXT:    add r0, r0, #8
+; CHECK-V8A-NEXT:    ldm r0!, {r2, r3}
 ; CHECK-V8A-NEXT:    add r2, r2, r3
 ; CHECK-V8A-NEXT:    str r2, [r1]
 ; CHECK-V8A-NEXT:    bx lr
   %addr = getelementptr i32, i32* %base, i32 0
   %ld = load i32, i32* %addr
   %addr.1 = getelementptr i32, i32* %base, i32 1
   %ld.1 = load i32, i32* %addr.1
   %addr.2 = getelementptr i32, i32* %base, i32 2

llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll

 ; CHECK-NEXT:  .LBB0_8: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, lr, .LBB0_11
 ; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add.w r1, r1, r3, lsl #2
 ; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
 ; CHECK-NEXT:    add.w r0, r0, r3, lsl #2
 ; CHECK-NEXT:  .LBB0_10: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s0, [r1]
-; CHECK-NEXT:    adds r1, #4
-; CHECK-NEXT:    vldr s2, [r2]
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vldmia r1!, {s0}
+; CHECK-NEXT:    vldmia r2!, {s2}
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
-; CHECK-NEXT:    vstr s0, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vstmia r0!, {s0}
 ; CHECK-NEXT:    le lr, .LBB0_10
 ; CHECK-NEXT:  .LBB0_11: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %vector.memcheck
 
 vector.memcheck:                                  ; preds = %entry

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll

 ; CHECK-NEXT:    and lr, r3, #3
 ; CHECK-NEXT:    wls lr, lr, .LBB0_7
 ; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
 ; CHECK-NEXT:    add.w r5, r0, r12, lsl #2
 ; CHECK-NEXT:    add.w r6, r1, r12, lsl #2
 ; CHECK-NEXT:    add.w r7, r2, r12, lsl #2
 ; CHECK-NEXT:  .LBB0_6: @ %for.body.prol
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s0, [r6]
-; CHECK-NEXT:    adds r6, #4
-; CHECK-NEXT:    vldr s2, [r5]
-; CHECK-NEXT:    adds r5, #4
+; CHECK-NEXT:    vldmia r6!, {s0}
 ; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vldmia r5!, {s2}
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
-; CHECK-NEXT:    vstr s0, [r7]
-; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    vstmia r7!, {s0}
 ; CHECK-NEXT:    le lr, .LBB0_6
 ; CHECK-NEXT:  .LBB0_7: @ %for.body.prol.loopexit
 ; CHECK-NEXT:    cmp r4, #3
 ; CHECK-NEXT:    blo .LBB0_10
 ; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w r3, r3, r12
 ; CHECK-NEXT:    lsl.w r12, r12, #2
 ; CHECK-NEXT:  .LBB0_9: @ %for.body

 ; CHECK-NEXT:    and lr, r3, #3
 ; CHECK-NEXT:    wls lr, lr, .LBB1_7
 ; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
 ; CHECK-NEXT:    add.w r5, r0, r12, lsl #2
 ; CHECK-NEXT:    add.w r6, r1, r12, lsl #2
 ; CHECK-NEXT:    add.w r7, r2, r12, lsl #2
 ; CHECK-NEXT:  .LBB1_6: @ %for.body.prol
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s0, [r6]
-; CHECK-NEXT:    adds r6, #4
-; CHECK-NEXT:    vldr s2, [r5]
-; CHECK-NEXT:    adds r5, #4
+; CHECK-NEXT:    vldmia r6!, {s0}
 ; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vldmia r5!, {s2}
 ; CHECK-NEXT:    vadd.f32 s0, s2, s0
-; CHECK-NEXT:    vstr s0, [r7]
-; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    vstmia r7!, {s0}
 ; CHECK-NEXT:    le lr, .LBB1_6
 ; CHECK-NEXT:  .LBB1_7: @ %for.body.prol.loopexit
 ; CHECK-NEXT:    cmp r4, #3
 ; CHECK-NEXT:    blo .LBB1_10
 ; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w r3, r3, r12
 ; CHECK-NEXT:    lsl.w r12, r12, #2
 ; CHECK-NEXT:  .LBB1_9: @ %for.body

 ; CHECK-NEXT:    and lr, r3, #3
 ; CHECK-NEXT:    wls lr, lr, .LBB2_7
 ; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
 ; CHECK-NEXT:    add.w r5, r0, r12, lsl #2
 ; CHECK-NEXT:    add.w r6, r1, r12, lsl #2
 ; CHECK-NEXT:    add.w r7, r2, r12, lsl #2
 ; CHECK-NEXT:  .LBB2_6: @ %for.body.prol
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s0, [r6]
-; CHECK-NEXT:    adds r6, #4
-; CHECK-NEXT:    vldr s2, [r5]
-; CHECK-NEXT:    adds r5, #4
+; CHECK-NEXT:    vldmia r6!, {s0}
 ; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vldmia r5!, {s2}
 ; CHECK-NEXT:    vsub.f32 s0, s2, s0
-; CHECK-NEXT:    vstr s0, [r7]
-; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    vstmia r7!, {s0}
 ; CHECK-NEXT:    le lr, .LBB2_6
 ; CHECK-NEXT:  .LBB2_7: @ %for.body.prol.loopexit
 ; CHECK-NEXT:    cmp r4, #3
 ; CHECK-NEXT:    blo .LBB2_10
 ; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w r3, r3, r12
 ; CHECK-NEXT:    lsl.w r12, r12, #2
 ; CHECK-NEXT:  .LBB2_9: @ %for.body

 ; CHECK-NEXT:  @ %bb.8: @ %for.body.prol.preheader
 ; CHECK-NEXT:    add.w r5, r0, r12, lsl #2
 ; CHECK-NEXT:    add.w r6, r1, r12, lsl #2
 ; CHECK-NEXT:    add.w r7, r2, r12, lsl #2
 ; CHECK-NEXT:  .LBB3_9: @ %for.body.prol
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r4, [r6], #4
 ; CHECK-NEXT:    add.w r12, r12, #1
-; CHECK-NEXT:    vldr s2, [r5]
-; CHECK-NEXT:    adds r5, #4
+; CHECK-NEXT:    vldmia r5!, {s2}
 ; CHECK-NEXT:    vmov s0, r4
 ; CHECK-NEXT:    vcvt.f32.s32 s0, s0
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
-; CHECK-NEXT:    vstr s0, [r7]
-; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    vstmia r7!, {s0}
 ; CHECK-NEXT:    le lr, .LBB3_9
 ; CHECK-NEXT:  .LBB3_10: @ %for.body.prol.loopexit
 ; CHECK-NEXT:    cmp.w r8, #3
 ; CHECK-NEXT:    blo .LBB3_13
 ; CHECK-NEXT:  @ %bb.11: @ %for.body.preheader1
 ; CHECK-NEXT:    add.w r1, r1, r12, lsl #2
 ; CHECK-NEXT:    sub.w r3, r3, r12
 ; CHECK-NEXT:    adds r1, #8

 ; CHECK-NEXT:  .LBB5_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    adds r1, #2
 ; CHECK-NEXT:    vmul.f16 s0, s2, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [r2]
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vstmia r2!, {s0}
 ; CHECK-NEXT:    le lr, .LBB5_7
 ; CHECK-NEXT:  .LBB5_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry

 ; CHECK-NEXT:  .LBB6_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    adds r1, #2
 ; CHECK-NEXT:    vadd.f16 s0, s2, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [r2]
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vstmia r2!, {s0}
 ; CHECK-NEXT:    le lr, .LBB6_7
 ; CHECK-NEXT:  .LBB6_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry

 ; CHECK-NEXT:  .LBB7_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vldr.16 s2, [r0]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    adds r1, #2
 ; CHECK-NEXT:    vsub.f16 s0, s2, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [r2]
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vstmia r2!, {s0}
 ; CHECK-NEXT:    le lr, .LBB7_7
 ; CHECK-NEXT:  .LBB7_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry

 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsh r3, [r1], #2
 ; CHECK-NEXT:    vldr.16 s0, [r0]
 ; CHECK-NEXT:    adds r0, #2
 ; CHECK-NEXT:    vmov s2, r3
 ; CHECK-NEXT:    vcvt.f16.s32 s2, s2
 ; CHECK-NEXT:    vmul.f16 s0, s0, s2
 ; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
-; CHECK-NEXT:    vstr s0, [r2]
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vstmia r2!, {s0}
 ; CHECK-NEXT:    le lr, .LBB8_7
 ; CHECK-NEXT:  .LBB8_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
 entry:
   %cmp10 = icmp eq i32 %N, 0
   br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader
 
 for.body.preheader:                               ; preds = %entry

llvm/test/CodeGen/Thumb2/mve-float32regloops.ll

 ; CHECK-NEXT:  .LBB17_3: @ Parent Loop BB17_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrw.u32 q4, [r1, q0, uxtw #2]
 ; CHECK-NEXT:    vldrw.u32 q5, [r4, q0, uxtw #2]
 ; CHECK-NEXT:    vldrw.u32 q3, [sp, #8]
 ; CHECK-NEXT:    adds r1, #8
 ; CHECK-NEXT:    vfma.f32 q5, q4, r5
 ; CHECK-NEXT:    vfma.f32 q3, q5, q2
-; CHECK-NEXT:    vstmia r7, {s20, s21}
-; CHECK-NEXT:    adds r7, #8
+; CHECK-NEXT:    vstmia r7!, {s20, s21}
 ; CHECK-NEXT:    vfma.f32 q3, q4, q1
 ; CHECK-NEXT:    vstrw.32 q3, [r4]
 ; CHECK-NEXT:    le lr, .LBB17_3
 ; CHECK-NEXT:  @ %bb.4: @ in Loop: Header=BB17_2 Depth=1
 ; CHECK-NEXT:    subs.w r12, r12, #1
 ; CHECK-NEXT:    add.w r0, r0, #20
 ; CHECK-NEXT:    vstrb.8 q3, [r3], #16
 ; CHECK-NEXT:    mov r1, r2

 ; CHECK-NEXT:  @ %bb.4: @ %while.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB20_3 Depth=1
 ; CHECK-NEXT:    vmov q6, q1
 ; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:    mov lr, r3
 ; CHECK-NEXT:  .LBB20_5: @ %while.body
 ; CHECK-NEXT:    @ Parent Loop BB20_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldrd r7, r4, [r1]
-; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    ldrd r7, r4, [r1], #8
 ; CHECK-NEXT:    vfma.f32 q6, q3, r7
 ; CHECK-NEXT:    vmov r7, s24
 ; CHECK-NEXT:    vmov q1, q6
 ; CHECK-NEXT:    vfma.f32 q1, q2, r7
 ; CHECK-NEXT:    vstr s24, [r5]
 ; CHECK-NEXT:    vmov.f32 s7, s0
 ; CHECK-NEXT:    vfma.f32 q1, q4, r4
 ; CHECK-NEXT:    vmov r4, s5

llvm/test/CodeGen/Thumb2/mve-postinc-distribute.ll

 ; CHECK-NEXT:    popeq {r4, r5, r6, pc}
 ; CHECK-NEXT:  .LBB2_6: @ %for.body.preheader12
 ; CHECK-NEXT:    sub.w lr, r3, r12
 ; CHECK-NEXT:    add.w r0, r0, r12, lsl #2
 ; CHECK-NEXT:    add.w r1, r1, r12, lsl #2
 ; CHECK-NEXT:    add.w r2, r2, r12, lsl #2
 ; CHECK-NEXT:  .LBB2_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s0, [r0]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vldr s2, [r1]
-; CHECK-NEXT:    adds r1, #4
+; CHECK-NEXT:    vldmia r0!, {s0}
+; CHECK-NEXT:    vldmia r1!, {s2}
 ; CHECK-NEXT:    vldr s4, [r2]
 ; CHECK-NEXT:    vfma.f32 s4, s2, s0
-; CHECK-NEXT:    vstr s4, [r2]
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vstmia r2!, {s4}
 ; CHECK-NEXT:    le lr, .LBB2_7
 ; CHECK-NEXT:  .LBB2_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp8 = icmp sgt i32 %n, 0
   br i1 %cmp8, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry

llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll

 ; CHECK-NEXT:    popeq {r4, r5, r6, pc}
 ; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader12
 ; CHECK-NEXT:    sub.w lr, r3, r12
 ; CHECK-NEXT:    add.w r0, r0, r12, lsl #2
 ; CHECK-NEXT:    add.w r1, r1, r12, lsl #2
 ; CHECK-NEXT:    add.w r2, r2, r12, lsl #2
 ; CHECK-NEXT:  .LBB0_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s0, [r0]
-; CHECK-NEXT:    adds r0, #4
-; CHECK-NEXT:    vldr s2, [r1]
-; CHECK-NEXT:    adds r1, #4
+; CHECK-NEXT:    vldmia r0!, {s0}
+; CHECK-NEXT:    vldmia r1!, {s2}
 ; CHECK-NEXT:    vldr s4, [r2]
 ; CHECK-NEXT:    vfma.f32 s4, s2, s0
-; CHECK-NEXT:    vstr s4, [r2]
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vstmia r2!, {s4}
 ; CHECK-NEXT:    le lr, .LBB0_7
 ; CHECK-NEXT:  .LBB0_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp8 = icmp sgt i32 %n, 0
   br i1 %cmp8, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry

llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll

 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #1
 ; CHECK-NEXT:    add.w r11, r2, r3, lsl #2
 ; CHECK-NEXT:    add.w r10, r1, r3, lsl #2
 ; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
 ; CHECK-NEXT:    vmvn.i32 q1, #0x80000000
 ; CHECK-NEXT:  .LBB0_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrd r5, r4, [r0]
+; CHECK-NEXT:    ldrd r5, r4, [r0], #8
 ; CHECK-NEXT:    mov.w r3, #-1
-; CHECK-NEXT:    ldrd r8, r7, [r1]
-; CHECK-NEXT:    adds r0, #8
+; CHECK-NEXT:    ldrd r8, r7, [r1], #8
 ; CHECK-NEXT:    smull r4, r7, r7, r4
-; CHECK-NEXT:    adds r1, #8
 ; CHECK-NEXT:    asrl r4, r7, #31
 ; CHECK-NEXT:    smull r6, r5, r8, r5
 ; CHECK-NEXT:    rsbs.w r9, r4, #-2147483648
 ; CHECK-NEXT:    sbcs r3, r7
 ; CHECK-NEXT:    mov.w r3, #0
 ; CHECK-NEXT:    asrl r6, r5, #31
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r3, #1

 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    csetm r4, ne
 ; CHECK-NEXT:    vmov q3[2], q3[0], r3, r4
 ; CHECK-NEXT:    vbic q4, q1, q3
 ; CHECK-NEXT:    vand q2, q2, q3
 ; CHECK-NEXT:    vorr q2, q2, q4
 ; CHECK-NEXT:    vmov r3, s10
 ; CHECK-NEXT:    vmov r4, s8
-; CHECK-NEXT:    strd r4, r3, [r2]
-; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    strd r4, r3, [r2], #8
 ; CHECK-NEXT:    le lr, .LBB0_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    ldrd r7, r3, [sp] @ 8-byte Folded Reload
 ; CHECK-NEXT:    cmp r7, r3
 ; CHECK-NEXT:    beq .LBB0_8
 ; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader
 ; CHECK-NEXT:    sub.w lr, r3, r7
 ; CHECK-NEXT:    mov.w r0, #-1

 ; CHECK-NEXT:    subs r7, r5, #2
 ; CHECK-NEXT:    str r5, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    add.w r8, r2, r5, lsl #2
 ; CHECK-NEXT:    add.w r11, r1, r5, lsl #2
 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #1
 ; CHECK-NEXT:    add.w r12, r0, r5, lsl #2
 ; CHECK-NEXT:  .LBB3_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrd r4, r7, [r0]
-; CHECK-NEXT:    adds r0, #8
-; CHECK-NEXT:    ldrd r5, r10, [r1]
-; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    ldrd r4, r7, [r0], #8
+; CHECK-NEXT:    ldrd r5, r10, [r1], #8
 ; CHECK-NEXT:    umull r4, r5, r5, r4
 ; CHECK-NEXT:    lsrl r4, r5, #31
 ; CHECK-NEXT:    subs.w r6, r4, #-1
 ; CHECK-NEXT:    umull r6, r7, r10, r7
 ; CHECK-NEXT:    sbcs r5, r5, #0
 ; CHECK-NEXT:    mov.w r5, #0
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r5, #1

 ; CHECK-NEXT:    movlo r5, #1
 ; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    csetm r5, ne
 ; CHECK-NEXT:    vmov q0[2], q0[0], r9, r5
 ; CHECK-NEXT:    vand q1, q1, q0
 ; CHECK-NEXT:    vorn q0, q1, q0
 ; CHECK-NEXT:    vmov r4, s2
 ; CHECK-NEXT:    vmov r5, s0
-; CHECK-NEXT:    strd r5, r4, [r2]
-; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    strd r5, r4, [r2], #8
 ; CHECK-NEXT:    le lr, .LBB3_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    ldr r7, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r7, r3
 ; CHECK-NEXT:    beq .LBB3_8
 ; CHECK-NEXT:  .LBB3_6: @ %for.body.preheader
 ; CHECK-NEXT:    sub.w lr, r3, r7
 ; CHECK-NEXT:  .LBB3_7: @ %for.body

llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll

 ; CHECK-NEXT:    vadd.f32 s0, s0, s1
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    beq .LBB5_9
 ; CHECK-NEXT:  .LBB5_7: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w lr, r1, r2
 ; CHECK-NEXT:    add.w r0, r0, r2, lsl #2
 ; CHECK-NEXT:  .LBB5_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s2, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldmia r0!, {s2}
 ; CHECK-NEXT:    vadd.f32 s0, s2, s0
 ; CHECK-NEXT:    le lr, .LBB5_8
 ; CHECK-NEXT:  .LBB5_9: @ %for.cond.cleanup
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:  @ %bb.10:
 ; CHECK-NEXT:  .LCPI5_0:

 ; CHECK-NEXT:    vmul.f32 s0, s0, s1
 ; CHECK-NEXT:    vmul.f32 s0, s0, s4
 ; CHECK-NEXT:    beq .LBB6_9
 ; CHECK-NEXT:  .LBB6_7: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w lr, r1, r2
 ; CHECK-NEXT:    add.w r0, r0, r2, lsl #2
 ; CHECK-NEXT:  .LBB6_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldr s2, [r0]
-; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vldmia r0!, {s2}
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    le lr, .LBB6_8
 ; CHECK-NEXT:  .LBB6_9: @ %for.cond.cleanup
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup

llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll

 ; CHECK-NEXT:    dls lr, r5
 ; CHECK-NEXT:  .LBB1_7: @ %while.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr s0, [r3]
 ; CHECK-NEXT:    vldr s2, [r3, #4]
 ; CHECK-NEXT:    adds r3, #8
 ; CHECK-NEXT:    vmul.f32 s0, s0, s0
 ; CHECK-NEXT:    vfma.f32 s0, s2, s2
-; CHECK-NEXT:    vstr s0, [r12]
-; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vstmia r12!, {s0}
 ; CHECK-NEXT:    le lr, .LBB1_7
 ; CHECK-NEXT:  .LBB1_8: @ %while.end
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:  .LBB1_9:
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:    mov r12, r1
 ; CHECK-NEXT:    mov r5, r2
 ; CHECK-NEXT:    b .LBB1_6