[ARM] Disable WLSTP loops

[dmgreen]

Summary

This checks to see if the loop will likely become a tail predicated loop and disables wls loop generation if so, as the likelihood for reverting is currently too high. These should be fairly rare situations anyway due to the way iterations and element counts are used during lowering. Just not trying can alter how SCEV's are materialized however, leading to different codegen.

It also adds a option to disable all while low overhead loops, for debugging.

[samparker]

Sounds good to me, I would have been in favour of just completely disabling wls for now :)

[dmgreen]

Thanks. WLS can certainly be beneficial in places, so is best to leave on for non-tail predicated loop. We can try to make them less likely to revert and hopefully re-enable them for tail predicated loops in the future, once the performance/correctness is more reliable.

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

 static cl::opt<bool> EnableMaskedLoadStores(
   "enable-arm-maskedldst", cl::Hidden, cl::init(true),
   cl::desc("Enable the generation of masked loads and stores"));
 
 static cl::opt<bool> DisableLowOverheadLoops(
   "disable-arm-loloops", cl::Hidden, cl::init(false),
   cl::desc("Disable the generation of low-overhead loops"));
 
+static cl::opt<bool>
+    AllowWLSLoops("allow-arm-wlsloops", cl::Hidden, cl::init(true),
+                  cl::desc("Enable the generation of WLS loops"));
+
 extern cl::opt<TailPredication::Mode> EnableTailPredication;
 
 extern cl::opt<bool> EnableMaskedGatherScatters;
 
 extern cl::opt<unsigned> MVEMaxSupportedInterleaveFactor;
 
 /// Convert a vector load intrinsic into a simple llvm load instruction.
 /// This is beneficial when the underlying object being addressed comes
if (auto *Call = dyn_cast<IntrinsicInst>(&I)) {
       case Intrinsic::loop_decrement_reg:
         return true;
       }
     }
     return false;
   };
 
   // Scan the instructions to see if there's any that we know will turn into a
-  // call or if this loop is already a low-overhead loop.
+  // call or if this loop is already a low-overhead loop or will become a tail
+  // predicated loop.
+  bool IsTailPredLoop = false;
   auto ScanLoop = [&](Loop *L) {
     for (auto *BB : L->getBlocks()) {
       for (auto &I : *BB) {
         if (maybeLoweredToCall(I) || IsHardwareLoopIntrinsic(I) ||
             isa<InlineAsm>(I)) {
           LLVM_DEBUG(dbgs() << "ARMHWLoops: Bad instruction: " << I << "\n");
           return false;
         }
+        if (auto *II = dyn_cast<IntrinsicInst>(&I))
+          IsTailPredLoop |=
+              II->getIntrinsicID() == Intrinsic::get_active_lane_mask ||
+              II->getIntrinsicID() == Intrinsic::arm_mve_vctp8 ||
+              II->getIntrinsicID() == Intrinsic::arm_mve_vctp16 ||
+              II->getIntrinsicID() == Intrinsic::arm_mve_vctp32 ||
+              II->getIntrinsicID() == Intrinsic::arm_mve_vctp64;
       }
     }
     return true;
   };
 
   // Visit inner loops.
   for (auto Inner : *L)
     if (!ScanLoop(Inner))
       return false;
 
   if (!ScanLoop(L))
     return false;
 
   // TODO: Check whether the trip count calculation is expensive. If L is the
   // inner loop but we know it has a low trip count, calculating that trip
   // count (in the parent loop) may be detrimental.
 
   LLVMContext &C = L->getHeader()->getContext();
   HWLoopInfo.CounterInReg = true;
   HWLoopInfo.IsNestingLegal = false;
-  HWLoopInfo.PerformEntryTest = true;
+  HWLoopInfo.PerformEntryTest = AllowWLSLoops && !IsTailPredLoop;
   HWLoopInfo.CountType = Type::getInt32Ty(C);
   HWLoopInfo.LoopDecrement = ConstantInt::get(HWLoopInfo.CountType, 1);
   return true;
 }
 
 static bool canTailPredicateInstruction(Instruction &I, int &ICmpCount) {
   // We don't allow icmp's, and because we only look at single block loops,
   // we simply count the icmps, i.e. there should only be 1 for the backedge.

llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fullfp16 -tail-predication=enabled %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
 ; CHECK-LABEL: fast_float_mul:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq.w .LBB0_11
 ; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
-; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
-; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
-; CHECK-NEXT:    cmp r4, r2
-; CHECK-NEXT:    mov.w r12, #1
+; CHECK-NEXT:    add.w r5, r0, r3, lsl #2
+; CHECK-NEXT:    add.w r4, r2, r3, lsl #2
+; CHECK-NEXT:    cmp r5, r2
+; CHECK-NEXT:    cset r12, hi
+; CHECK-NEXT:    cmp r4, r0
 ; CHECK-NEXT:    cset lr, hi
-; CHECK-NEXT:    cmp r5, r0
-; CHECK-NEXT:    cset r6, hi
-; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    cmp r5, r1
 ; CHECK-NEXT:    add.w r5, r1, r3, lsl #2
 ; CHECK-NEXT:    cset r4, hi
 ; CHECK-NEXT:    cmp r5, r0
 ; CHECK-NEXT:    cset r5, hi
-; CHECK-NEXT:    ands r5, r4
-; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    ands r4, r5
+; CHECK-NEXT:    lsls r4, r4, #31
 ; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    andeq.w r6, r6, lr
-; CHECK-NEXT:    lslseq.w r6, r6, #31
+; CHECK-NEXT:    andeq.w r5, lr, r12
+; CHECK-NEXT:    lslseq.w r5, r5, #31
 ; CHECK-NEXT:    beq .LBB0_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    subs r6, r3, #1
+; CHECK-NEXT:    subs r5, r3, #1
 ; CHECK-NEXT:    and r7, r3, #3
-; CHECK-NEXT:    cmp r6, #3
+; CHECK-NEXT:    cmp r5, #3
 ; CHECK-NEXT:    bhs .LBB0_6
 ; CHECK-NEXT:  @ %bb.3:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB0_8
 ; CHECK-NEXT:  .LBB0_4: @ %vector.ph
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB0_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r2], #16
 ; CHECK-NEXT:    vmul.f32 q0, q1, q0
 ; CHECK-NEXT:    vstrw.32 q0, [r0], #16
 ; CHECK-NEXT:    letp lr, .LBB0_5
 ; CHECK-NEXT:    b .LBB0_11
 ; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    movs r5, #1
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    add.w lr, r12, r3, lsr #2
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, r5, r3, lsr #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:  .LBB0_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r4, r1, r3
 ; CHECK-NEXT:    adds r5, r2, r3
 ; CHECK-NEXT:    adds r6, r0, r3
 ; CHECK-NEXT:    adds r3, #16
 ; CHECK-NEXT:    vldr s0, [r4]
 ; CHECK-NEXT:    add.w r12, r12, #4

 
 define arm_aapcs_vfpcc float @fast_float_mac(float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
 ; CHECK-LABEL: fast_float_mac:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    cbz r2, .LBB1_4
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    adds r3, r2, #3
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    mov.w r12, #1
 ; CHECK-NEXT:    bic r3, r3, #3
-; CHECK-NEXT:    sub.w r12, r3, #4
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    add.w lr, r12, r3, lsr #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB1_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r2
 ; CHECK-NEXT:    adds r3, #4
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vmov q1, q0

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-operand.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -tail-predication=enabled -o - %s | FileCheck %s
 
 define arm_aapcs_vfpcc void @arm_var_f32_mve(float* %pSrc, i32 %blockSize, float* nocapture %pResult) {
 ; CHECK-LABEL: arm_var_f32_mve:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    cmp r1, #4
-; CHECK-NEXT:    it ge
-; CHECK-NEXT:    movge r4, #4
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    subs r4, r1, r4
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    adds r4, #3
-; CHECK-NEXT:    add.w r12, r3, r4, lsr #2
 ; CHECK-NEXT:    mov r3, r1
 ; CHECK-NEXT:    dlstp.32 lr, r1
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:  .LBB0_1: @ %do.body.i
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r4], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r12], #16
 ; CHECK-NEXT:    vadd.f32 q0, q0, q1
 ; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %arm_mean_f32_mve.exit
 ; CHECK-NEXT:    vmov s4, r1
-; CHECK-NEXT:    dls lr, r12
-; CHECK-NEXT:    vadd.f32 s0, s3, s3
 ; CHECK-NEXT:    mov r3, r1
+; CHECK-NEXT:    vadd.f32 s0, s3, s3
+; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    vcvt.f32.u32 s4, s4
+; CHECK-NEXT:    it ge
+; CHECK-NEXT:    movge r3, #4
+; CHECK-NEXT:    subs r3, r1, r3
+; CHECK-NEXT:    mov.w lr, #1
+; CHECK-NEXT:    adds r3, #3
+; CHECK-NEXT:    add.w lr, lr, r3, lsr #2
+; CHECK-NEXT:    mov r3, r1
+; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    vdiv.f32 s0, s0, s4
-; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov r12, s0
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:  .LBB0_3: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r3
 ; CHECK-NEXT:    subs r3, #4
 ; CHECK-NEXT:    vpsttt
 ; CHECK-NEXT:    vldrwt.u32 q1, [r0], #16
-; CHECK-NEXT:    vsubt.f32 q1, q1, r4
+; CHECK-NEXT:    vsubt.f32 q1, q1, r12
 ; CHECK-NEXT:    vfmat.f32 q0, q1, q1
 ; CHECK-NEXT:    le lr, .LBB0_3
 ; CHECK-NEXT:  @ %bb.4: @ %do.end
 ; CHECK-NEXT:    subs r0, r1, #1
 ; CHECK-NEXT:    vadd.f32 s0, s3, s3
 ; CHECK-NEXT:    vmov s2, r0
 ; CHECK-NEXT:    vcvt.f32.u32 s2, s2
 ; CHECK-NEXT:    vdiv.f32 s0, s0, s2
 ; CHECK-NEXT:    vstr s0, [r2]
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   br label %do.body.i
 
 do.body.i:                                        ; preds = %entry, %do.body.i
   %blkCnt.0.i = phi i32 [ %sub.i, %do.body.i ], [ %blockSize, %entry ]
   %sumVec.0.i = phi <4 x float> [ %3, %do.body.i ], [ zeroinitializer, %entry ]
   %pSrc.addr.0.i = phi float* [ %add.ptr.i, %do.body.i ], [ %pSrc, %entry ]
   %0 = tail call <4 x i1> @llvm.arm.mve.vctp32(i32 %blkCnt.0.i)

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll

 define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_char(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
 ; CHECK-LABEL: test_vec_mul_scalar_add_char:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    ldr.w r12, [sp, #28]
 ; CHECK-NEXT:    cmp.w r12, #0
 ; CHECK-NEXT:    beq.w .LBB5_11
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
-; CHECK-NEXT:    add.w r6, r3, r12, lsl #2
-; CHECK-NEXT:    add.w r4, r1, r12
-; CHECK-NEXT:    cmp r6, r1
-; CHECK-NEXT:    add.w r5, r0, r12
+; CHECK-NEXT:    add.w r4, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r5, r1, r12
+; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    add.w r6, r0, r12
 ; CHECK-NEXT:    cset lr, hi
-; CHECK-NEXT:    cmp r4, r3
-; CHECK-NEXT:    cset r4, hi
-; CHECK-NEXT:    cmp r6, r0
-; CHECK-NEXT:    cset r6, hi
 ; CHECK-NEXT:    cmp r5, r3
 ; CHECK-NEXT:    cset r5, hi
-; CHECK-NEXT:    ands r5, r6
-; CHECK-NEXT:    movs r6, #1
-; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    cmp r4, r0
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    ands r4, r6
+; CHECK-NEXT:    lsls r4, r4, #31
 ; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    andeq.w r5, r4, lr
-; CHECK-NEXT:    lslseq.w r5, r5, #31
+; CHECK-NEXT:    andeq.w r6, r5, lr
+; CHECK-NEXT:    lslseq.w r6, r6, #31
 ; CHECK-NEXT:    beq .LBB5_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    sub.w r5, r12, #1
+; CHECK-NEXT:    sub.w r6, r12, #1
 ; CHECK-NEXT:    and r9, r12, #3
-; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    cmp r6, #3
 ; CHECK-NEXT:    bhs .LBB5_6
 ; CHECK-NEXT:  @ %bb.3:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB5_8
 ; CHECK-NEXT:  .LBB5_4: @ %vector.ph
 ; CHECK-NEXT:    movs r6, #0
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB5_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r6, #4
 ; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
 ; CHECK-NEXT:    vldrb.u32 q1, [r1], #4
 ; CHECK-NEXT:    vmlas.u32 q1, q0, r2
 ; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB5_5
 ; CHECK-NEXT:    b .LBB5_11
 ; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader.new
-; CHECK-NEXT:    bic r5, r12, #3
+; CHECK-NEXT:    bic r6, r12, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    subs r6, #4
 ; CHECK-NEXT:    add.w r4, r3, #8
-; CHECK-NEXT:    subs r5, #4
 ; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    add.w lr, r6, r5, lsr #2
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
 ; CHECK-NEXT:    adds r5, r0, #3
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    adds r6, r1, #1
 ; CHECK-NEXT:  .LBB5_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r8, [r5, #-3]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    ldrb r7, [r6, #-1]

 define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_uchar(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
 ; CHECK-LABEL: test_vec_mul_scalar_add_uchar:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    ldr.w r12, [sp, #28]
 ; CHECK-NEXT:    cmp.w r12, #0
 ; CHECK-NEXT:    beq.w .LBB7_11
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
-; CHECK-NEXT:    add.w r6, r3, r12, lsl #2
-; CHECK-NEXT:    add.w r4, r1, r12
-; CHECK-NEXT:    cmp r6, r1
-; CHECK-NEXT:    add.w r5, r0, r12
+; CHECK-NEXT:    add.w r4, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r5, r1, r12
+; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    add.w r6, r0, r12
 ; CHECK-NEXT:    cset lr, hi
-; CHECK-NEXT:    cmp r4, r3
-; CHECK-NEXT:    cset r4, hi
-; CHECK-NEXT:    cmp r6, r0
-; CHECK-NEXT:    cset r6, hi
 ; CHECK-NEXT:    cmp r5, r3
 ; CHECK-NEXT:    cset r5, hi
-; CHECK-NEXT:    ands r5, r6
-; CHECK-NEXT:    movs r6, #1
-; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    cmp r4, r0
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    ands r4, r6
+; CHECK-NEXT:    lsls r4, r4, #31
 ; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    andeq.w r5, r4, lr
-; CHECK-NEXT:    lslseq.w r5, r5, #31
+; CHECK-NEXT:    andeq.w r6, r5, lr
+; CHECK-NEXT:    lslseq.w r6, r6, #31
 ; CHECK-NEXT:    beq .LBB7_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    sub.w r5, r12, #1
+; CHECK-NEXT:    sub.w r6, r12, #1
 ; CHECK-NEXT:    and r9, r12, #3
-; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    cmp r6, #3
 ; CHECK-NEXT:    bhs .LBB7_6
 ; CHECK-NEXT:  @ %bb.3:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB7_8
 ; CHECK-NEXT:  .LBB7_4: @ %vector.ph
 ; CHECK-NEXT:    movs r6, #0
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB7_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r6, #4
 ; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
 ; CHECK-NEXT:    vldrb.u32 q1, [r1], #4
 ; CHECK-NEXT:    vmlas.u32 q1, q0, r2
 ; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB7_5
 ; CHECK-NEXT:    b .LBB7_11
 ; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader.new
-; CHECK-NEXT:    bic r5, r12, #3
+; CHECK-NEXT:    bic r6, r12, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    subs r6, #4
 ; CHECK-NEXT:    add.w r4, r3, #8
-; CHECK-NEXT:    subs r5, #4
 ; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    add.w lr, r6, r5, lsr #2
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
 ; CHECK-NEXT:    adds r5, r0, #3
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    adds r6, r1, #1
 ; CHECK-NEXT:  .LBB7_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r8, [r5, #-3]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    ldrb r7, [r6, #-1]

 define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_int(i32* nocapture readonly %a, i32* nocapture readonly %b, i32 %c, i32* nocapture %res, i32 %N) {
 ; CHECK-LABEL: test_vec_mul_scalar_add_int:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    ldr.w r12, [sp, #28]
 ; CHECK-NEXT:    cmp.w r12, #0
 ; CHECK-NEXT:    beq.w .LBB9_11
 ; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
-; CHECK-NEXT:    add.w r6, r3, r12, lsl #2
-; CHECK-NEXT:    add.w r4, r1, r12, lsl #2
-; CHECK-NEXT:    cmp r6, r1
-; CHECK-NEXT:    add.w r5, r0, r12, lsl #2
+; CHECK-NEXT:    add.w r4, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r5, r1, r12, lsl #2
+; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    add.w r6, r0, r12, lsl #2
 ; CHECK-NEXT:    cset lr, hi
-; CHECK-NEXT:    cmp r4, r3
-; CHECK-NEXT:    cset r4, hi
-; CHECK-NEXT:    cmp r6, r0
-; CHECK-NEXT:    cset r6, hi
 ; CHECK-NEXT:    cmp r5, r3
 ; CHECK-NEXT:    cset r5, hi
-; CHECK-NEXT:    ands r5, r6
-; CHECK-NEXT:    movs r6, #1
-; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    cmp r4, r0
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    ands r4, r6
+; CHECK-NEXT:    lsls r4, r4, #31
 ; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    andeq.w r5, r4, lr
-; CHECK-NEXT:    lslseq.w r5, r5, #31
+; CHECK-NEXT:    andeq.w r6, r5, lr
+; CHECK-NEXT:    lslseq.w r6, r6, #31
 ; CHECK-NEXT:    beq .LBB9_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    sub.w r5, r12, #1
+; CHECK-NEXT:    sub.w r6, r12, #1
 ; CHECK-NEXT:    and r9, r12, #3
-; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    cmp r6, #3
 ; CHECK-NEXT:    bhs .LBB9_6
 ; CHECK-NEXT:  @ %bb.3:
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB9_8
 ; CHECK-NEXT:  .LBB9_4: @ %vector.ph
 ; CHECK-NEXT:    movs r6, #0
 ; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB9_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r6, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    vmlas.u32 q1, q0, r2
 ; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB9_5
 ; CHECK-NEXT:    b .LBB9_11
 ; CHECK-NEXT:  .LBB9_6: @ %for.body.preheader.new
-; CHECK-NEXT:    bic r5, r12, #3
+; CHECK-NEXT:    bic r6, r12, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    subs r6, #4
 ; CHECK-NEXT:    add.w r4, r3, #8
-; CHECK-NEXT:    subs r5, #4
 ; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    add.w lr, r6, r5, lsr #2
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
 ; CHECK-NEXT:    add.w r5, r0, #8
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    add.w r6, r1, #8
 ; CHECK-NEXT:  .LBB9_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r8, [r5, #-8]
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    ldr r7, [r6, #-8]

llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll

 ; CHECK-NEXT:    vmul.i32 q0, q2, q0
 ; CHECK-NEXT:    vadd.i32 q0, q0, q1
 ; CHECK-NEXT:    le lr, .LBB6_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    vaddv.u32 r12, q0
 ; CHECK-NEXT:    cbz r2, .LBB6_7
 ; CHECK-NEXT:  @ %bb.4: @ %vector.ph47
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    add.w lr, r3, r6, lsr #2
 ; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    vdup.32 q0, r3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r6, lsr #2
 ; CHECK-NEXT:    vmov.32 q0[0], r12
+; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB6_5: @ %vector.body46
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r2
 ; CHECK-NEXT:    vmov q1, q0
 ; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vldrbt.u32 q0, [r0], #4
 ; CHECK-NEXT:    vldrbt.u32 q2, [r1], #4
 ; CHECK-NEXT:    subs r2, #4

 %struct.date = type { i32, i32, i32, i32 }
 @days = internal unnamed_addr constant [2 x [13 x i32]] [[13 x i32] [i32 0, i32 31, i32 28, i32 31, i32 30, i32 31, i32 30, i32 31, i32 31, i32 30, i32 31, i32 30, i32 31], [13 x i32] [i32 0, i32 31, i32 29, i32 31, i32 30, i32 31, i32 30, i32 31, i32 31, i32 30, i32 31, i32 30, i32 31]], align 4
 define i32 @wrongop(%struct.date* nocapture readonly %pd) {
 ; CHECK-LABEL: wrongop:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    mov r1, r0
 ; CHECK-NEXT:    movw r12, #47184
+; CHECK-NEXT:    movw r3, #23593
 ; CHECK-NEXT:    ldrd r2, lr, [r1, #4]
-; CHECK-NEXT:    movw r1, #23593
 ; CHECK-NEXT:    movt r12, #1310
-; CHECK-NEXT:    movt r1, #49807
-; CHECK-NEXT:    mla r1, lr, r1, r12
-; CHECK-NEXT:    movw r3, #55051
+; CHECK-NEXT:    movt r3, #49807
+; CHECK-NEXT:    mla r3, lr, r3, r12
+; CHECK-NEXT:    movw r1, #55051
 ; CHECK-NEXT:    movw r4, #23593
-; CHECK-NEXT:    movt r3, #163
+; CHECK-NEXT:    movt r1, #163
 ; CHECK-NEXT:    ldr r0, [r0]
 ; CHECK-NEXT:    movt r4, #655
-; CHECK-NEXT:    ror.w r12, r1, #4
-; CHECK-NEXT:    cmp r12, r3
-; CHECK-NEXT:    cset r3, lo
-; CHECK-NEXT:    ror.w r1, r1, #2
+; CHECK-NEXT:    ror.w r12, r3, #4
+; CHECK-NEXT:    cmp r12, r1
+; CHECK-NEXT:    cset r1, lo
+; CHECK-NEXT:    ror.w r3, r3, #2
 ; CHECK-NEXT:    mov.w r12, #1
-; CHECK-NEXT:    cmp r1, r4
-; CHECK-NEXT:    csel r1, r3, r12, lo
+; CHECK-NEXT:    cmp r3, r4
+; CHECK-NEXT:    csel r3, r1, r12, lo
 ; CHECK-NEXT:    lsls.w r4, lr, #30
-; CHECK-NEXT:    csel r3, r3, r1, ne
+; CHECK-NEXT:    csel r1, r1, r3, ne
 ; CHECK-NEXT:    cmp r2, #1
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    poplt {r4, pc}
 ; CHECK-NEXT:  .LBB8_1: @ %vector.ph
-; CHECK-NEXT:    adds r1, r2, #3
+; CHECK-NEXT:    movw r3, :lower16:days
 ; CHECK-NEXT:    movs r4, #52
-; CHECK-NEXT:    bic r1, r1, #3
-; CHECK-NEXT:    subs r1, #4
-; CHECK-NEXT:    add.w lr, r12, r1, lsr #2
-; CHECK-NEXT:    movw r1, :lower16:days
-; CHECK-NEXT:    movt r1, :upper16:days
-; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    mla r1, r3, r4, r1
+; CHECK-NEXT:    movt r3, :upper16:days
+; CHECK-NEXT:    mla r1, r1, r4, r3
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    adds r0, r2, #3
+; CHECK-NEXT:    bic r0, r0, #3
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    add.w lr, r12, r0, lsr #2
+; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB8_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r2
 ; CHECK-NEXT:    vmov q1, q0
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrwt.u32 q0, [r1], #16
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vadd.i32 q0, q0, q1

llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve --verify-machineinstrs %s -o - | FileCheck %s
 
 define void @simple(i32* nocapture readonly %x, i32* nocapture readnone %y, i32* nocapture %z, i32 %m, i32 %n) {
 ; CHECK-LABEL: simple:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    ldr r1, [sp, #8]
 ; CHECK-NEXT:    mov r12, r3
-; CHECK-NEXT:    wlstp.32 lr, r1, .LBB0_2
-; CHECK-NEXT:  .LBB0_1: @ %do.body
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    add.w lr, r1, #3
+; CHECK-NEXT:    cmp.w r3, lr, lsr #2
+; CHECK-NEXT:    beq .LBB0_3
+; CHECK-NEXT:  @ %bb.1: @ %do.body.preheader
+; CHECK-NEXT:    dlstp.32 lr, r1
+; CHECK-NEXT:  .LBB0_2: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vaddva.s32 r12, q0
-; CHECK-NEXT:    letp lr, .LBB0_1
-; CHECK-NEXT:  .LBB0_2: @ %if.end
+; CHECK-NEXT:    letp lr, .LBB0_2
+; CHECK-NEXT:  .LBB0_3: @ %if.end
 ; CHECK-NEXT:    str.w r12, [r2]
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %add = add i32 %n, 3
   %div = lshr i32 %add, 2
   %cmp.not = icmp eq i32 %div, 0
   br i1 %cmp.not, label %if.end, label %do.body
 
if.end:                                           ; preds = %do.body, %entry
   %s.1 = phi i32 [ %m, %entry ], [ %add2, %do.body ]
   store i32 %s.1, i32* %z, align 4
   ret void
 }
 
 define void @nested(i32* nocapture readonly %x, i32* nocapture readnone %y, i32* nocapture %z, i32 %m, i32 %n) {
 ; CHECK-LABEL: nested:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    cbz r3, .LBB1_7
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cbz r3, .LBB1_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    ldr.w r12, [sp, #20]
+; CHECK-NEXT:    ldr r5, [sp, #24]
+; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    b .LBB1_3
-; CHECK-NEXT:  .LBB1_2: @ %if.end
-; CHECK-NEXT:    @ in Loop: Header=BB1_3 Depth=1
+; CHECK-NEXT:    b .LBB1_4
+; CHECK-NEXT:  .LBB1_2: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:  .LBB1_3: @ %if.end
+; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    str.w r4, [r2, r1, lsl #2]
 ; CHECK-NEXT:    adds r1, #1
 ; CHECK-NEXT:    cmp r1, r3
-; CHECK-NEXT:    beq .LBB1_7
-; CHECK-NEXT:  .LBB1_3: @ %for.body
+; CHECK-NEXT:    beq .LBB1_8
+; CHECK-NEXT:  .LBB1_4: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB1_5 Depth 2
-; CHECK-NEXT:    add.w r5, r12, #3
-; CHECK-NEXT:    mov r4, r3
-; CHECK-NEXT:    lsr.w lr, r5, #2
-; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    @ Child Loop BB1_6 Depth 2
+; CHECK-NEXT:    adds r7, r5, #3
+; CHECK-NEXT:    cmp.w r12, r7, lsr #2
 ; CHECK-NEXT:    beq .LBB1_2
-; CHECK-NEXT:    b .LBB1_4
-; CHECK-NEXT:  .LBB1_4: @ %do.body.preheader
-; CHECK-NEXT:    @ in Loop: Header=BB1_3 Depth=1
-; CHECK-NEXT:    bic r6, r5, #3
-; CHECK-NEXT:    mov r7, r12
+; CHECK-NEXT:  @ %bb.5: @ %do.body.preheader
+; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT:    bic r6, r7, #3
+; CHECK-NEXT:    dlstp.32 lr, r5
+; CHECK-NEXT:    mov r7, r5
+; CHECK-NEXT:    add.w r8, r0, r6, lsl #2
 ; CHECK-NEXT:    mov r4, r3
-; CHECK-NEXT:    add.w r5, r0, r6, lsl #2
-; CHECK-NEXT:  .LBB1_5: @ %do.body
-; CHECK-NEXT:    @ Parent Loop BB1_3 Depth=1
+; CHECK-NEXT:  .LBB1_6: @ %do.body
+; CHECK-NEXT:    @ Parent Loop BB1_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vctp.32 r7
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
-; CHECK-NEXT:    subs r7, #4
-; CHECK-NEXT:    subs.w lr, lr, #1
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vaddvat.s32 r4, q0
-; CHECK-NEXT:    bne .LBB1_5
-; CHECK-NEXT:    b .LBB1_6
-; CHECK-NEXT:  .LBB1_6: @ %if.end.loopexit
-; CHECK-NEXT:    @ in Loop: Header=BB1_3 Depth=1
-; CHECK-NEXT:    sub.w r12, r12, r6
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    b .LBB1_2
-; CHECK-NEXT:  .LBB1_7: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    vaddva.s32 r4, q0
+; CHECK-NEXT:    letp lr, .LBB1_6
+; CHECK-NEXT:  @ %bb.7: @ %if.end.loopexit
+; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT:    subs r5, r5, r6
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    b .LBB1_3
+; CHECK-NEXT:  .LBB1_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %cmp20.not = icmp eq i32 %m, 0
   br i1 %cmp20.not, label %for.cond.cleanup, label %for.body
 
 for.cond.cleanup:                                 ; preds = %if.end, %entry
   ret void
 
 for.body:                                         ; preds = %entry, %if.end

llvm/test/CodeGen/Thumb2/mve-fma-loops.ll

 ; CHECK-LABEL: fmss1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp r3, #1
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    poplt {r4, pc}
 ; CHECK-NEXT:  .LBB4_1: @ %vector.ph
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    dlstp.32 lr, r3
-; CHECK-NEXT:    eor r12, r4, #-2147483648
+; CHECK-NEXT:    vmov r12, s0
 ; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    dlstp.32 lr, r3
+; CHECK-NEXT:    eor r12, r12, #-2147483648
 ; CHECK-NEXT:  .LBB4_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r4, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    vfmas.f32 q1, q0, r12
 ; CHECK-NEXT:    vstrw.32 q1, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB4_2

 
 for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
 define arm_aapcs_vfpcc void @fmss2(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
 ; CHECK-LABEL: fmss2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    cmp r3, #1
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    poplt {r4, pc}
-; CHECK-NEXT:  .LBB5_1: @ %vector.ph
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    vdup.32 q0, r4
-; CHECK-NEXT:    vneg.f32 q0, q0
+; CHECK-NEXT:    blt .LBB5_3
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    vmov r6, s0
+; CHECK-NEXT:    vdup.32 q0, r6
 ; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    vneg.f32 q0, q0
 ; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB5_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vmov q3, q0
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
 ; CHECK-NEXT:    vfma.f32 q3, q2, q1
 ; CHECK-NEXT:    vstrw.32 q3, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB5_2
-; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:  .LBB5_3: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %cmp8 = icmp sgt i32 %n, 0
   br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4
   %trip.count.minus.1 = add i32 %n, -1

 
 for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
 define arm_aapcs_vfpcc void @fmss3(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
 ; CHECK-LABEL: fmss3:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp r3, #1
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    poplt {r7, pc}
+; CHECK-NEXT:    poplt {r4, pc}
 ; CHECK-NEXT:  .LBB6_1: @ %vector.ph
-; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vdup.32 q0, r4
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB6_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vmov q3, q0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
 ; CHECK-NEXT:    vfms.f32 q3, q2, q1
 ; CHECK-NEXT:    vstrw.32 q3, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB6_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %cmp8 = icmp sgt i32 %n, 0
   br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4
   %trip.count.minus.1 = add i32 %n, -1

 
 for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
 define arm_aapcs_vfpcc void @fmss4(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
 ; CHECK-LABEL: fmss4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp r3, #1
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    poplt {r7, pc}
+; CHECK-NEXT:    poplt {r4, pc}
 ; CHECK-NEXT:  .LBB7_1: @ %vector.ph
-; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vdup.32 q0, r4
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB7_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vmov q3, q0
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
 ; CHECK-NEXT:    vfms.f32 q3, q2, q1
 ; CHECK-NEXT:    vstrw.32 q3, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB7_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %cmp8 = icmp sgt i32 %n, 0
   br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4
   %trip.count.minus.1 = add i32 %n, -1

 ; CHECK-LABEL: fms1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp r3, #1
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    poplt {r4, pc}
 ; CHECK-NEXT:  .LBB8_1: @ %vector.ph
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    dlstp.32 lr, r3
-; CHECK-NEXT:    eor r12, r4, #-2147483648
+; CHECK-NEXT:    vmov r12, s0
 ; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    dlstp.32 lr, r3
+; CHECK-NEXT:    eor r12, r12, #-2147483648
 ; CHECK-NEXT:  .LBB8_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r4, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    vfma.f32 q1, q0, r12
 ; CHECK-NEXT:    vstrw.32 q1, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB8_2

 
 for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
 define arm_aapcs_vfpcc void @fms2(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
 ; CHECK-LABEL: fms2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp r3, #1
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    poplt {r7, pc}
+; CHECK-NEXT:    poplt {r4, pc}
 ; CHECK-NEXT:  .LBB9_1: @ %vector.ph
-; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vdup.32 q0, r4
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB9_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add.w r12, r12, #4
 ; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
 ; CHECK-NEXT:    vfms.f32 q2, q1, q0
 ; CHECK-NEXT:    vstrw.32 q2, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB9_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %cmp8 = icmp sgt i32 %n, 0
   br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4
   %trip.count.minus.1 = add i32 %n, -1

llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll

 
 
 ; Similar but predicated
 define i32 @arm_nn_mat_mul_core_4x_s8(i32 %row_elements, i32 %offset, i8* %row_base, i8* %col_base, i32* nocapture readnone %sum_col, i32* nocapture %output) {
 ; CHECK-LABEL: arm_nn_mat_mul_core_4x_s8:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r10, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r10, lr}
-; CHECK-NEXT:    add.w r7, r0, #15
 ; CHECK-NEXT:    ldr.w r12, [sp, #32]
-; CHECK-NEXT:    asrs r6, r7, #31
-; CHECK-NEXT:    add.w r7, r7, r6, lsr #28
-; CHECK-NEXT:    movs r6, #1
-; CHECK-NEXT:    asrs r5, r7, #4
-; CHECK-NEXT:    cmp r5, #1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    asrgt r6, r7, #4
 ; CHECK-NEXT:    cmp r0, #1
 ; CHECK-NEXT:    blt .LBB4_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    adds r7, r2, r1
 ; CHECK-NEXT:    add.w r5, r2, r1, lsl #1
-; CHECK-NEXT:    add.w r1, r1, r1, lsl #1
-; CHECK-NEXT:    dlstp.8 lr, r0
-; CHECK-NEXT:    add r1, r2
 ; CHECK-NEXT:    mov.w r8, #0
-; CHECK-NEXT:    movs r6, #0
 ; CHECK-NEXT:    movs r4, #0
 ; CHECK-NEXT:    mov.w r10, #0
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    adds r7, r2, r1
+; CHECK-NEXT:    add.w r1, r1, r1, lsl #1
+; CHECK-NEXT:    add r1, r2
+; CHECK-NEXT:    dlstp.8 lr, r0
 ; CHECK-NEXT:  .LBB4_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u8 q0, [r3], #16
 ; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
 ; CHECK-NEXT:    vmlava.s8 r10, q1, q0
 ; CHECK-NEXT:    vldrb.u8 q1, [r5], #16
 ; CHECK-NEXT:    vmlava.s8 r4, q1, q0
 ; CHECK-NEXT:    vldrb.u8 q1, [r7], #16

 ; CHECK-NEXT:    add.w r9, r9, #1
 ; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r9, r0
 ; CHECK-NEXT:    beq .LBB5_8
 ; CHECK-NEXT:  .LBB5_5: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB5_7 Depth 2
 ; CHECK-NEXT:    ldr r0, [sp, #92]
+; CHECK-NEXT:    cmp r2, r2
 ; CHECK-NEXT:    ldr.w r12, [r0, r9, lsl #2]
-; CHECK-NEXT:    subs r0, r2, r2
-; CHECK-NEXT:    ble .LBB5_3
+; CHECK-NEXT:    bge .LBB5_3
 ; CHECK-NEXT:  @ %bb.6: @ %for.body24.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB5_5 Depth=1
 ; CHECK-NEXT:    ldr.w r11, [sp, #88]
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    mov r6, r12
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r12
 ; CHECK-NEXT:    dlstp.16 lr, r11
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mla r3, r9, r11, r1
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mla r3, r9, r11, r0
+; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    ldrd r7, r5, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r8, r12
 ; CHECK-NEXT:    mov r10, r12
 ; CHECK-NEXT:  .LBB5_7: @ %for.body24
 ; CHECK-NEXT:    @ Parent Loop BB5_5 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrb.s16 q0, [r7], #8
 ; CHECK-NEXT:    vadd.i16 q1, q0, r4
 ; CHECK-NEXT:    vldrb.s16 q0, [r3], #8
 ; CHECK-NEXT:    vmlava.s16 r12, q0, q1

 ; CHECK-NEXT:    add r0, r1
 ; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    adds r0, r2, #7
 ; CHECK-NEXT:    lsrs r2, r0, #3
 ; CHECK-NEXT:  .LBB6_3: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB6_5 Depth 2
 ; CHECK-NEXT:    ldr r0, [sp, #92]
+; CHECK-NEXT:    cmp r2, r2
 ; CHECK-NEXT:    ldr.w r12, [r0, r9, lsl #2]
-; CHECK-NEXT:    subs r0, r2, r2
-; CHECK-NEXT:    ble .LBB6_6
+; CHECK-NEXT:    bge .LBB6_6
 ; CHECK-NEXT:  @ %bb.4: @ %for.body24.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB6_3 Depth=1
 ; CHECK-NEXT:    ldr.w r11, [sp, #88]
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    mov r6, r12
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r12
 ; CHECK-NEXT:    dlstp.16 lr, r11
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mla r3, r9, r11, r1
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mla r3, r9, r11, r0
+; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    ldrd r7, r5, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r8, r12
 ; CHECK-NEXT:    mov r10, r12
 ; CHECK-NEXT:  .LBB6_5: @ %for.body24
 ; CHECK-NEXT:    @ Parent Loop BB6_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrb.s16 q0, [r7], #8
 ; CHECK-NEXT:    vadd.i16 q1, q0, r4
 ; CHECK-NEXT:    vldrb.s16 q0, [r3], #8
 ; CHECK-NEXT:    vmlava.s16 r12, q0, q1