[ARM] Add low overhead loops terminators to AnalyzeBranch

[dmgreen]

Summary

This treats low overhead loop branches the same as jump tables and indirect branches in analyzeBranch - they cannot be analyzed but the direct branches on the end of the block may be removed. This helps remove the unnecessary branches earlier, which can help produce better codegen (and change block layout in a number of cases).

[samtebbs]

I remember running into issues where analyzeBranch didn't know about LOL instructions. Thanks for updating it.

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

 }
 
 static inline bool isJumpTableBranchOpcode(int Opc) {
   return Opc == ARM::BR_JTr || Opc == ARM::BR_JTm_i12 ||
          Opc == ARM::BR_JTm_rs || Opc == ARM::BR_JTadd || Opc == ARM::tBR_JTr ||
          Opc == ARM::t2BR_JT;
 }
 
+static inline bool isLowOverheadTerminatorOpcode(int Opc) {
+  return Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
+         Opc == ARM::t2LoopEnd || Opc == ARM::t2LoopEndDec;
+}
+
 static inline
 bool isIndirectBranchOpcode(int Opc) {
   return Opc == ARM::BX || Opc == ARM::MOVPCRX || Opc == ARM::tBRIND;
 }
 
 static inline bool isIndirectCall(const MachineInstr &MI) {
   int Opc = MI.getOpcode();
   switch (Opc) {

llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp

while (isPredicated(*I) || I->isTerminator() || I->isDebugValue()) {
     while (I->isDebugInstr() || !I->isTerminator() ||
            isSpeculationBarrierEndBBOpcode(I->getOpcode()) ){
       if (I == MBB.instr_begin())
         return false;
       --I;
     }
 
     if (isIndirectBranchOpcode(I->getOpcode()) ||
-        isJumpTableBranchOpcode(I->getOpcode())) {
+        isJumpTableBranchOpcode(I->getOpcode()) ||
+        isLowOverheadTerminatorOpcode(I->getOpcode())) {
       // Indirect branches and jump tables can't be analyzed, but we still want
       // to clean up any instructions at the tail of the basic block.
       CantAnalyze = true;
     } else if (isUncondBranchOpcode(I->getOpcode())) {
       TBB = I->getOperand(0).getMBB();
     } else if (isCondBranchOpcode(I->getOpcode())) {
       // Bail out if we encounter multiple conditional branches.
       if (!Cond.empty())

llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll

 ; CHECK-NEXT:    sub.w r12, r2, #1
 ; CHECK-NEXT:    adr r2, .LCPI2_1
 ; CHECK-NEXT:    mov lr, lr
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vdup.32 q1, r12
 ; CHECK-NEXT:    vdup.32 q2, r12
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
-; CHECK-NEXT:    b .LBB2_4
+; CHECK-NEXT:    b .LBB2_5
 ; CHECK-NEXT:  .LBB2_2: @ %cond.load25
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmovx.f16 s0, s28
 ; CHECK-NEXT:    vmov r4, s28
 ; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    vmov.16 q6[0], r4
 ; CHECK-NEXT:    vldr.16 s0, [r1, #6]
 ; CHECK-NEXT:    vmov.16 q6[1], r2
 ; CHECK-NEXT:    vmov r2, s29
 ; CHECK-NEXT:    vmov.16 q6[2], r2
 ; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    vmov.16 q6[3], r2
 ; CHECK-NEXT:  .LBB2_3: @ %else26
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmul.f16 q0, q6, q5
 ; CHECK-NEXT:    adds r0, #8
 ; CHECK-NEXT:    vcvtt.f32.f16 s23, s1
 ; CHECK-NEXT:    adds r1, #8
 ; CHECK-NEXT:    vcvtb.f32.f16 s22, s1
 ; CHECK-NEXT:    adds r3, #4
 ; CHECK-NEXT:    vcvtt.f32.f16 s21, s0
 ; CHECK-NEXT:    vcvtb.f32.f16 s20, s0
 ; CHECK-NEXT:    vadd.f32 q5, q3, q5
 ; CHECK-NEXT:    subs.w lr, lr, #1
-; CHECK-NEXT:    bne .LBB2_4
-; CHECK-NEXT:    b .LBB2_21
-; CHECK-NEXT:  .LBB2_4: @ %vector.body
+; CHECK-NEXT:    bne .LBB2_5
+; CHECK-NEXT:  @ %bb.4: @ %middle.block
+; CHECK-NEXT:    vdup.32 q0, r12
+; CHECK-NEXT:    vcmp.u32 cs, q0, q4
+; CHECK-NEXT:    vpsel q0, q5, q3
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vadd.f32 q0, q0, r0
+; CHECK-NEXT:    b .LBB2_23
+; CHECK-NEXT:  .LBB2_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
 ; CHECK-NEXT:    vmov q3, q5
 ; CHECK-NEXT:    @ implicit-def: $q6
 ; CHECK-NEXT:    vadd.i32 q4, q0, r3
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q4
 ; CHECK-NEXT:    vmrs r4, p0
 ; CHECK-NEXT:    and r2, r4, #1
 ; CHECK-NEXT:    rsbs r5, r2, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    bfi r2, r5, #0, #1
 ; CHECK-NEXT:    ubfx r5, r4, #4, #1
 ; CHECK-NEXT:    rsbs r5, r5, #0
 ; CHECK-NEXT:    bfi r2, r5, #1, #1
 ; CHECK-NEXT:    ubfx r5, r4, #8, #1
 ; CHECK-NEXT:    ubfx r4, r4, #12, #1
 ; CHECK-NEXT:    rsbs r5, r5, #0
 ; CHECK-NEXT:    bfi r2, r5, #2, #1
 ; CHECK-NEXT:    rsbs r4, r4, #0
 ; CHECK-NEXT:    bfi r2, r4, #3, #1
 ; CHECK-NEXT:    lsls r4, r2, #31
-; CHECK-NEXT:    bne .LBB2_9
-; CHECK-NEXT:  @ %bb.5: @ %else
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bne .LBB2_10
+; CHECK-NEXT:  @ %bb.6: @ %else
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    lsls r4, r2, #30
-; CHECK-NEXT:    bpl .LBB2_10
-; CHECK-NEXT:  .LBB2_6: @ %cond.load6
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bpl .LBB2_11
+; CHECK-NEXT:  .LBB2_7: @ %cond.load6
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vldr.16 s20, [r0, #2]
 ; CHECK-NEXT:    vmov r5, s24
 ; CHECK-NEXT:    vmovx.f16 s24, s25
 ; CHECK-NEXT:    vmov r4, s20
 ; CHECK-NEXT:    vmov.16 q5[0], r5
 ; CHECK-NEXT:    vmov.16 q5[1], r4
 ; CHECK-NEXT:    vmov r4, s25
 ; CHECK-NEXT:    vmov.16 q5[2], r4
 ; CHECK-NEXT:    vmov r4, s24
 ; CHECK-NEXT:    vmov.16 q5[3], r4
 ; CHECK-NEXT:    lsls r4, r2, #29
-; CHECK-NEXT:    bmi .LBB2_11
-; CHECK-NEXT:  .LBB2_7: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bmi .LBB2_12
+; CHECK-NEXT:  .LBB2_8: @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmov q6, q5
 ; CHECK-NEXT:    lsls r2, r2, #28
-; CHECK-NEXT:    bmi .LBB2_12
-; CHECK-NEXT:  .LBB2_8: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bmi .LBB2_13
+; CHECK-NEXT:  .LBB2_9: @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmov q5, q6
-; CHECK-NEXT:    b .LBB2_13
-; CHECK-NEXT:  .LBB2_9: @ %cond.load
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    b .LBB2_14
+; CHECK-NEXT:  .LBB2_10: @ %cond.load
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vldr.16 s24, [r0]
 ; CHECK-NEXT:    lsls r4, r2, #30
-; CHECK-NEXT:    bmi .LBB2_6
-; CHECK-NEXT:  .LBB2_10: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bmi .LBB2_7
+; CHECK-NEXT:  .LBB2_11: @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmov q5, q6
 ; CHECK-NEXT:    lsls r4, r2, #29
-; CHECK-NEXT:    bpl .LBB2_7
-; CHECK-NEXT:  .LBB2_11: @ %cond.load9
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bpl .LBB2_8
+; CHECK-NEXT:  .LBB2_12: @ %cond.load9
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmovx.f16 s24, s20
 ; CHECK-NEXT:    vmov r4, s20
 ; CHECK-NEXT:    vldr.16 s28, [r0, #4]
 ; CHECK-NEXT:    vmov r5, s24
 ; CHECK-NEXT:    vmov.16 q6[0], r4
 ; CHECK-NEXT:    vmovx.f16 s20, s21
 ; CHECK-NEXT:    vmov.16 q6[1], r5
 ; CHECK-NEXT:    vmov r4, s28
 ; CHECK-NEXT:    vmov.16 q6[2], r4
 ; CHECK-NEXT:    vmov r4, s20
 ; CHECK-NEXT:    vmov.16 q6[3], r4
 ; CHECK-NEXT:    lsls r2, r2, #28
-; CHECK-NEXT:    bpl .LBB2_8
-; CHECK-NEXT:  .LBB2_12: @ %cond.load12
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bpl .LBB2_9
+; CHECK-NEXT:  .LBB2_13: @ %cond.load12
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmovx.f16 s20, s24
 ; CHECK-NEXT:    vmov r4, s24
 ; CHECK-NEXT:    vmov r2, s20
 ; CHECK-NEXT:    vmov.16 q5[0], r4
 ; CHECK-NEXT:    vmov.16 q5[1], r2
 ; CHECK-NEXT:    vmov r2, s25
 ; CHECK-NEXT:    vldr.16 s24, [r0, #6]
 ; CHECK-NEXT:    vmov.16 q5[2], r2
 ; CHECK-NEXT:    vmov r2, s24
 ; CHECK-NEXT:    vmov.16 q5[3], r2
-; CHECK-NEXT:  .LBB2_13: @ %else13
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:  .LBB2_14: @ %else13
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vcmp.u32 cs, q2, q4
 ; CHECK-NEXT:    @ implicit-def: $q7
 ; CHECK-NEXT:    vmrs r4, p0
 ; CHECK-NEXT:    and r2, r4, #1
 ; CHECK-NEXT:    rsbs r5, r2, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    bfi r2, r5, #0, #1
 ; CHECK-NEXT:    ubfx r5, r4, #4, #1
 ; CHECK-NEXT:    rsbs r5, r5, #0
 ; CHECK-NEXT:    bfi r2, r5, #1, #1
 ; CHECK-NEXT:    ubfx r5, r4, #8, #1
 ; CHECK-NEXT:    ubfx r4, r4, #12, #1
 ; CHECK-NEXT:    rsbs r5, r5, #0
 ; CHECK-NEXT:    bfi r2, r5, #2, #1
 ; CHECK-NEXT:    rsbs r4, r4, #0
 ; CHECK-NEXT:    bfi r2, r4, #3, #1
 ; CHECK-NEXT:    lsls r4, r2, #31
-; CHECK-NEXT:    bne .LBB2_17
-; CHECK-NEXT:  @ %bb.14: @ %else17
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bne .LBB2_18
+; CHECK-NEXT:  @ %bb.15: @ %else17
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    lsls r4, r2, #30
-; CHECK-NEXT:    bpl .LBB2_18
-; CHECK-NEXT:  .LBB2_15: @ %cond.load19
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bpl .LBB2_19
+; CHECK-NEXT:  .LBB2_16: @ %cond.load19
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vldr.16 s24, [r1, #2]
 ; CHECK-NEXT:    vmov r5, s28
 ; CHECK-NEXT:    vmovx.f16 s28, s29
 ; CHECK-NEXT:    vmov r4, s24
 ; CHECK-NEXT:    vmov.16 q6[0], r5
 ; CHECK-NEXT:    vmov.16 q6[1], r4
 ; CHECK-NEXT:    vmov r4, s29
 ; CHECK-NEXT:    vmov.16 q6[2], r4
 ; CHECK-NEXT:    vmov r4, s28
 ; CHECK-NEXT:    vmov.16 q6[3], r4
 ; CHECK-NEXT:    lsls r4, r2, #29
-; CHECK-NEXT:    bmi .LBB2_19
-; CHECK-NEXT:  .LBB2_16: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bmi .LBB2_20
+; CHECK-NEXT:  .LBB2_17: @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmov q7, q6
 ; CHECK-NEXT:    lsls r2, r2, #28
 ; CHECK-NEXT:    bmi.w .LBB2_2
-; CHECK-NEXT:    b .LBB2_20
-; CHECK-NEXT:  .LBB2_17: @ %cond.load16
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    b .LBB2_21
+; CHECK-NEXT:  .LBB2_18: @ %cond.load16
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vldr.16 s28, [r1]
 ; CHECK-NEXT:    lsls r4, r2, #30
-; CHECK-NEXT:    bmi .LBB2_15
-; CHECK-NEXT:  .LBB2_18: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bmi .LBB2_16
+; CHECK-NEXT:  .LBB2_19: @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmov q6, q7
 ; CHECK-NEXT:    lsls r4, r2, #29
-; CHECK-NEXT:    bpl .LBB2_16
-; CHECK-NEXT:  .LBB2_19: @ %cond.load22
-; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    bpl .LBB2_17
+; CHECK-NEXT:  .LBB2_20: @ %cond.load22
+; CHECK-NEXT:    @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmovx.f16 s28, s24
 ; CHECK-NEXT:    vmov r4, s24
 ; CHECK-NEXT:    vldr.16 s0, [r1, #4]
 ; CHECK-NEXT:    vmov r5, s28
 ; CHECK-NEXT:    vmov.16 q7[0], r4
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmov.16 q7[1], r5
 ; CHECK-NEXT:    vmovx.f16 s0, s25
 ; CHECK-NEXT:    vmov.16 q7[2], r4
 ; CHECK-NEXT:    vmov r4, s0
 ; CHECK-NEXT:    vmov.16 q7[3], r4
 ; CHECK-NEXT:    lsls r2, r2, #28
 ; CHECK-NEXT:    bmi.w .LBB2_2
-; CHECK-NEXT:  .LBB2_20: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:  .LBB2_21: @ in Loop: Header=BB2_5 Depth=1
 ; CHECK-NEXT:    vmov q6, q7
 ; CHECK-NEXT:    b .LBB2_3
-; CHECK-NEXT:  .LBB2_21: @ %middle.block
-; CHECK-NEXT:    vdup.32 q0, r12
-; CHECK-NEXT:    vcmp.u32 cs, q0, q4
-; CHECK-NEXT:    vpsel q0, q5, q3
-; CHECK-NEXT:    vmov.f32 s4, s2
-; CHECK-NEXT:    vmov.f32 s5, s3
-; CHECK-NEXT:    vadd.f32 q0, q0, q1
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vadd.f32 q0, q0, r0
-; CHECK-NEXT:    b .LBB2_23
 ; CHECK-NEXT:  .LBB2_22:
 ; CHECK-NEXT:    vldr s0, .LCPI2_0
 ; CHECK-NEXT:  .LBB2_23: @ %for.cond.cleanup
 ; CHECK-NEXT:    @ kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:    .p2align 4

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll

 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB9_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI9_0
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB9_6
 ; CHECK-NEXT:  .LBB9_3:
 ; CHECK-NEXT:    vldr s0, .LCPI9_0
-; CHECK-NEXT:    b .LBB9_9
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:  .LBB9_4: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r2, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vldr s0, .LCPI9_0
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    movs r3, #0

 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB10_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI10_0
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB10_6
 ; CHECK-NEXT:  .LBB10_3:
 ; CHECK-NEXT:    vldr s0, .LCPI10_0
-; CHECK-NEXT:    b .LBB10_9
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:  .LBB10_4: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r2, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vldr s0, .LCPI10_0
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    movs r3, #0

 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB11_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    b .LBB11_6
 ; CHECK-NEXT:  .LBB11_3:
 ; CHECK-NEXT:    vldr s0, .LCPI11_0
-; CHECK-NEXT:    b .LBB11_9
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-NEXT:  .LBB11_4: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r2, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    adds r3, r1, #4

llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main -mattr=+lob --verify-machineinstrs %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc void @test(i16* noalias nocapture readonly %off, i16* noalias nocapture %data, i16* noalias nocapture %dst, i32 %n) {
 ; CHECK-LABEL: test:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    cmp r3, #1
 ; CHECK-NEXT:    blt .LBB0_7
 ; CHECK-NEXT:  @ %bb.1: @ %for.cond1.preheader.us.preheader
 ; CHECK-NEXT:    mov r8, r3
 ; CHECK-NEXT:    lsl.w r12, r3, #1
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:  .LBB0_2: @ %for.cond1.preheader.us
+; CHECK-NEXT:    b .LBB0_4
+; CHECK-NEXT:  .LBB0_2: @ %for.body15.us
+; CHECK-NEXT:    @ Parent Loop BB0_4 Depth=1
+; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    ldrh.w r7, [r0, r6, lsl #1]
+; CHECK-NEXT:    ldrh.w r5, [r1, r6, lsl #1]
+; CHECK-NEXT:    add r5, r7
+; CHECK-NEXT:    strh.w r5, [r2, r6, lsl #1]
+; CHECK-NEXT:    adds r6, #1
+; CHECK-NEXT:    le lr, .LBB0_2
+; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup14.us
+; CHECK-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    adds r3, #1
+; CHECK-NEXT:    add r2, r12
+; CHECK-NEXT:    add r4, r12
+; CHECK-NEXT:    cmp r3, r8
+; CHECK-NEXT:    beq .LBB0_7
+; CHECK-NEXT:  .LBB0_4: @ %for.cond1.preheader.us
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB0_3 Depth 2
 ; CHECK-NEXT:    @ Child Loop BB0_5 Depth 2
+; CHECK-NEXT:    @ Child Loop BB0_2 Depth 2
 ; CHECK-NEXT:    dls lr, r8
 ; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:  .LBB0_3: @ %for.body4.us
-; CHECK-NEXT:    @ Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:  .LBB0_5: @ %for.body4.us
+; CHECK-NEXT:    @ Parent Loop BB0_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldrh.w r5, [r0, r6, lsl #1]
 ; CHECK-NEXT:    ldrh.w r7, [r1, r6, lsl #1]
 ; CHECK-NEXT:    add r5, r7
 ; CHECK-NEXT:    strh.w r5, [r4, r6, lsl #1]
 ; CHECK-NEXT:    adds r6, #1
-; CHECK-NEXT:    le lr, .LBB0_3
-; CHECK-NEXT:  @ %bb.4: @ %for.body15.us.preheader
-; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB0_5
+; CHECK-NEXT:  @ %bb.6: @ %for.body15.us.preheader
+; CHECK-NEXT:    @ in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    dls lr, r8
 ; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:  .LBB0_5: @ %for.body15.us
-; CHECK-NEXT:    @ Parent Loop BB0_2 Depth=1
-; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldrh.w r7, [r0, r6, lsl #1]
-; CHECK-NEXT:    ldrh.w r5, [r1, r6, lsl #1]
-; CHECK-NEXT:    add r5, r7
-; CHECK-NEXT:    strh.w r5, [r2, r6, lsl #1]
-; CHECK-NEXT:    adds r6, #1
-; CHECK-NEXT:    le lr, .LBB0_5
-; CHECK-NEXT:  @ %bb.6: @ %for.cond.cleanup14.us
-; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT:    adds r3, #1
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    add r4, r12
-; CHECK-NEXT:    cmp r3, r8
-; CHECK-NEXT:    bne .LBB0_2
+; CHECK-NEXT:    b .LBB0_2
 ; CHECK-NEXT:  .LBB0_7: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %cmp252 = icmp sgt i32 %n, 0
   br i1 %cmp252, label %for.cond1.preheader.us, label %for.cond.cleanup
 
 for.cond1.preheader.us: ; preds = %entry, %for.cond.cleanup14.us
   %i.057.us = phi i32 [ %inc29.us, %for.cond.cleanup14.us ], [ 0, %entry ]

llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll

 ; ENABLED-NEXT:  @ %bb.1: @ %for.body.lr.ph
 ; ENABLED-NEXT:    ldr r0, [sp, #36]
 ; ENABLED-NEXT:    add.w r12, r2, #3
 ; ENABLED-NEXT:    ldr.w r10, [sp] @ 4-byte Reload
 ; ENABLED-NEXT:    mov.w r8, #0
 ; ENABLED-NEXT:    mov r9, r12
 ; ENABLED-NEXT:    uxth r0, r0
 ; ENABLED-NEXT:    rsbs r5, r0, #0
-; ENABLED-NEXT:    b .LBB0_4
-; ENABLED-NEXT:  .LBB0_2: @ in Loop: Header=BB0_4 Depth=1
-; ENABLED-NEXT:    movs r0, #0
-; ENABLED-NEXT:  .LBB0_3: @ %for.end
-; ENABLED-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; ENABLED-NEXT:    b .LBB0_5
+; ENABLED-NEXT:  .LBB0_2: @ %vector.body
+; ENABLED-NEXT:    @ Parent Loop BB0_5 Depth=1
+; ENABLED-NEXT:    @ => This Inner Loop Header: Depth=2
+; ENABLED-NEXT:    vctp.32 r4
+; ENABLED-NEXT:    vmov q0, q1
+; ENABLED-NEXT:    vpstt
+; ENABLED-NEXT:    vldrht.s32 q1, [r0], #8
+; ENABLED-NEXT:    vldrht.s32 q2, [r7], #8
+; ENABLED-NEXT:    mov lr, r6
+; ENABLED-NEXT:    vmul.i32 q1, q2, q1
+; ENABLED-NEXT:    subs r6, #1
+; ENABLED-NEXT:    vshl.s32 q1, r5
+; ENABLED-NEXT:    subs r4, #4
+; ENABLED-NEXT:    vadd.i32 q1, q1, q0
+; ENABLED-NEXT:    le lr, .LBB0_2
+; ENABLED-NEXT:  @ %bb.3: @ %middle.block
+; ENABLED-NEXT:    @ in Loop: Header=BB0_5 Depth=1
+; ENABLED-NEXT:    vpsel q0, q1, q0
+; ENABLED-NEXT:    vaddv.u32 r0, q0
+; ENABLED-NEXT:  .LBB0_4: @ %for.end
+; ENABLED-NEXT:    @ in Loop: Header=BB0_5 Depth=1
 ; ENABLED-NEXT:    lsrs r0, r0, #16
 ; ENABLED-NEXT:    sub.w r9, r9, #1
 ; ENABLED-NEXT:    strh.w r0, [r1, r8, lsl #1]
 ; ENABLED-NEXT:    add.w r8, r8, #1
 ; ENABLED-NEXT:    add.w r10, r10, #2
 ; ENABLED-NEXT:    cmp r8, r3
 ; ENABLED-NEXT:    beq .LBB0_8
-; ENABLED-NEXT:  .LBB0_4: @ %for.body
+; ENABLED-NEXT:  .LBB0_5: @ %for.body
 ; ENABLED-NEXT:    @ =>This Loop Header: Depth=1
-; ENABLED-NEXT:    @ Child Loop BB0_6 Depth 2
+; ENABLED-NEXT:    @ Child Loop BB0_2 Depth 2
 ; ENABLED-NEXT:    cmp r2, r8
-; ENABLED-NEXT:    ble .LBB0_2
-; ENABLED-NEXT:  @ %bb.5: @ %vector.ph
-; ENABLED-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; ENABLED-NEXT:    ble .LBB0_7
+; ENABLED-NEXT:  @ %bb.6: @ %vector.ph
+; ENABLED-NEXT:    @ in Loop: Header=BB0_5 Depth=1
 ; ENABLED-NEXT:    bic r0, r9, #3
 ; ENABLED-NEXT:    movs r7, #1
 ; ENABLED-NEXT:    subs r0, #4
 ; ENABLED-NEXT:    sub.w r4, r2, r8
 ; ENABLED-NEXT:    vmov.i32 q1, #0x0
 ; ENABLED-NEXT:    add.w r6, r7, r0, lsr #2
 ; ENABLED-NEXT:    sub.w r0, r12, r8
 ; ENABLED-NEXT:    bic r0, r0, #3
 ; ENABLED-NEXT:    subs r0, #4
 ; ENABLED-NEXT:    add.w r0, r7, r0, lsr #2
 ; ENABLED-NEXT:    mov r7, r10
 ; ENABLED-NEXT:    dls lr, r0
 ; ENABLED-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; ENABLED-NEXT:  .LBB0_6: @ %vector.body
-; ENABLED-NEXT:    @ Parent Loop BB0_4 Depth=1
-; ENABLED-NEXT:    @ => This Inner Loop Header: Depth=2
-; ENABLED-NEXT:    vctp.32 r4
-; ENABLED-NEXT:    vmov q0, q1
-; ENABLED-NEXT:    vpstt
-; ENABLED-NEXT:    vldrht.s32 q1, [r0], #8
-; ENABLED-NEXT:    vldrht.s32 q2, [r7], #8
-; ENABLED-NEXT:    mov lr, r6
-; ENABLED-NEXT:    vmul.i32 q1, q2, q1
-; ENABLED-NEXT:    subs r6, #1
-; ENABLED-NEXT:    vshl.s32 q1, r5
-; ENABLED-NEXT:    subs r4, #4
-; ENABLED-NEXT:    vadd.i32 q1, q1, q0
-; ENABLED-NEXT:    le lr, .LBB0_6
-; ENABLED-NEXT:  @ %bb.7: @ %middle.block
-; ENABLED-NEXT:    @ in Loop: Header=BB0_4 Depth=1
-; ENABLED-NEXT:    vpsel q0, q1, q0
-; ENABLED-NEXT:    vaddv.u32 r0, q0
-; ENABLED-NEXT:    b .LBB0_3
+; ENABLED-NEXT:    b .LBB0_2
+; ENABLED-NEXT:  .LBB0_7: @ in Loop: Header=BB0_5 Depth=1
+; ENABLED-NEXT:    movs r0, #0
+; ENABLED-NEXT:    b .LBB0_4
 ; ENABLED-NEXT:  .LBB0_8: @ %for.end17
 ; ENABLED-NEXT:    add sp, #4
 ; ENABLED-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 ;
 ; NOREDUCTIONS-LABEL: varying_outer_2d_reduction:
 ; NOREDUCTIONS:       @ %bb.0: @ %entry
 ; NOREDUCTIONS-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
 ; NOREDUCTIONS-NEXT:    sub sp, #4
 ; NOREDUCTIONS-NEXT:    cmp r3, #1
 ; NOREDUCTIONS-NEXT:    str r0, [sp] @ 4-byte Spill
 ; NOREDUCTIONS-NEXT:    blt .LBB0_8
 ; NOREDUCTIONS-NEXT:  @ %bb.1: @ %for.body.lr.ph
 ; NOREDUCTIONS-NEXT:    ldr r0, [sp, #36]
 ; NOREDUCTIONS-NEXT:    add.w r12, r2, #3
 ; NOREDUCTIONS-NEXT:    ldr.w r10, [sp] @ 4-byte Reload
 ; NOREDUCTIONS-NEXT:    mov.w r8, #0
 ; NOREDUCTIONS-NEXT:    mov r9, r12
 ; NOREDUCTIONS-NEXT:    uxth r0, r0
 ; NOREDUCTIONS-NEXT:    rsbs r5, r0, #0
-; NOREDUCTIONS-NEXT:    b .LBB0_4
-; NOREDUCTIONS-NEXT:  .LBB0_2: @ in Loop: Header=BB0_4 Depth=1
-; NOREDUCTIONS-NEXT:    movs r0, #0
-; NOREDUCTIONS-NEXT:  .LBB0_3: @ %for.end
-; NOREDUCTIONS-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; NOREDUCTIONS-NEXT:    b .LBB0_5
+; NOREDUCTIONS-NEXT:  .LBB0_2: @ %vector.body
+; NOREDUCTIONS-NEXT:    @ Parent Loop BB0_5 Depth=1
+; NOREDUCTIONS-NEXT:    @ => This Inner Loop Header: Depth=2
+; NOREDUCTIONS-NEXT:    vctp.32 r4
+; NOREDUCTIONS-NEXT:    vmov q0, q1
+; NOREDUCTIONS-NEXT:    vpstt
+; NOREDUCTIONS-NEXT:    vldrht.s32 q1, [r0], #8
+; NOREDUCTIONS-NEXT:    vldrht.s32 q2, [r7], #8
+; NOREDUCTIONS-NEXT:    mov lr, r6
+; NOREDUCTIONS-NEXT:    vmul.i32 q1, q2, q1
+; NOREDUCTIONS-NEXT:    subs r6, #1
+; NOREDUCTIONS-NEXT:    vshl.s32 q1, r5
+; NOREDUCTIONS-NEXT:    subs r4, #4
+; NOREDUCTIONS-NEXT:    vadd.i32 q1, q1, q0
+; NOREDUCTIONS-NEXT:    le lr, .LBB0_2
+; NOREDUCTIONS-NEXT:  @ %bb.3: @ %middle.block
+; NOREDUCTIONS-NEXT:    @ in Loop: Header=BB0_5 Depth=1
+; NOREDUCTIONS-NEXT:    vpsel q0, q1, q0
+; NOREDUCTIONS-NEXT:    vaddv.u32 r0, q0
+; NOREDUCTIONS-NEXT:  .LBB0_4: @ %for.end
+; NOREDUCTIONS-NEXT:    @ in Loop: Header=BB0_5 Depth=1
 ; NOREDUCTIONS-NEXT:    lsrs r0, r0, #16
 ; NOREDUCTIONS-NEXT:    sub.w r9, r9, #1
 ; NOREDUCTIONS-NEXT:    strh.w r0, [r1, r8, lsl #1]
 ; NOREDUCTIONS-NEXT:    add.w r8, r8, #1
 ; NOREDUCTIONS-NEXT:    add.w r10, r10, #2
 ; NOREDUCTIONS-NEXT:    cmp r8, r3
 ; NOREDUCTIONS-NEXT:    beq .LBB0_8
-; NOREDUCTIONS-NEXT:  .LBB0_4: @ %for.body
+; NOREDUCTIONS-NEXT:  .LBB0_5: @ %for.body
 ; NOREDUCTIONS-NEXT:    @ =>This Loop Header: Depth=1
-; NOREDUCTIONS-NEXT:    @ Child Loop BB0_6 Depth 2
+; NOREDUCTIONS-NEXT:    @ Child Loop BB0_2 Depth 2
 ; NOREDUCTIONS-NEXT:    cmp r2, r8
-; NOREDUCTIONS-NEXT:    ble .LBB0_2
-; NOREDUCTIONS-NEXT:  @ %bb.5: @ %vector.ph
-; NOREDUCTIONS-NEXT:    @ in Loop: Header=BB0_4 Depth=1
+; NOREDUCTIONS-NEXT:    ble .LBB0_7
+; NOREDUCTIONS-NEXT:  @ %bb.6: @ %vector.ph
+; NOREDUCTIONS-NEXT:    @ in Loop: Header=BB0_5 Depth=1
 ; NOREDUCTIONS-NEXT:    bic r0, r9, #3
 ; NOREDUCTIONS-NEXT:    movs r7, #1
 ; NOREDUCTIONS-NEXT:    subs r0, #4
 ; NOREDUCTIONS-NEXT:    sub.w r4, r2, r8
 ; NOREDUCTIONS-NEXT:    vmov.i32 q1, #0x0
 ; NOREDUCTIONS-NEXT:    add.w r6, r7, r0, lsr #2
 ; NOREDUCTIONS-NEXT:    sub.w r0, r12, r8
 ; NOREDUCTIONS-NEXT:    bic r0, r0, #3
 ; NOREDUCTIONS-NEXT:    subs r0, #4
 ; NOREDUCTIONS-NEXT:    add.w r0, r7, r0, lsr #2
 ; NOREDUCTIONS-NEXT:    mov r7, r10
 ; NOREDUCTIONS-NEXT:    dls lr, r0
 ; NOREDUCTIONS-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; NOREDUCTIONS-NEXT:  .LBB0_6: @ %vector.body
-; NOREDUCTIONS-NEXT:    @ Parent Loop BB0_4 Depth=1
-; NOREDUCTIONS-NEXT:    @ => This Inner Loop Header: Depth=2
-; NOREDUCTIONS-NEXT:    vctp.32 r4
-; NOREDUCTIONS-NEXT:    vmov q0, q1
-; NOREDUCTIONS-NEXT:    vpstt
-; NOREDUCTIONS-NEXT:    vldrht.s32 q1, [r0], #8
-; NOREDUCTIONS-NEXT:    vldrht.s32 q2, [r7], #8
-; NOREDUCTIONS-NEXT:    mov lr, r6
-; NOREDUCTIONS-NEXT:    vmul.i32 q1, q2, q1
-; NOREDUCTIONS-NEXT:    subs r6, #1
-; NOREDUCTIONS-NEXT:    vshl.s32 q1, r5
-; NOREDUCTIONS-NEXT:    subs r4, #4
-; NOREDUCTIONS-NEXT:    vadd.i32 q1, q1, q0
-; NOREDUCTIONS-NEXT:    le lr, .LBB0_6
-; NOREDUCTIONS-NEXT:  @ %bb.7: @ %middle.block
-; NOREDUCTIONS-NEXT:    @ in Loop: Header=BB0_4 Depth=1
-; NOREDUCTIONS-NEXT:    vpsel q0, q1, q0
-; NOREDUCTIONS-NEXT:    vaddv.u32 r0, q0
-; NOREDUCTIONS-NEXT:    b .LBB0_3
+; NOREDUCTIONS-NEXT:    b .LBB0_2
+; NOREDUCTIONS-NEXT:  .LBB0_7: @ in Loop: Header=BB0_5 Depth=1
+; NOREDUCTIONS-NEXT:    movs r0, #0
+; NOREDUCTIONS-NEXT:    b .LBB0_4
 ; NOREDUCTIONS-NEXT:  .LBB0_8: @ %for.end17
 ; NOREDUCTIONS-NEXT:    add sp, #4
 ; NOREDUCTIONS-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
 entry:
   %conv = sext i16 %N to i32
   %cmp36 = icmp sgt i16 %N, 0
   br i1 %cmp36, label %for.body.lr.ph, label %for.end17
 

llvm/test/CodeGen/Thumb2/LowOverheadLoops/vcmp-vpst-combination.ll

 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vmovt q2, q1
 ; CHECK-NEXT:    vaddva.u32 r2, q2
 ; CHECK-NEXT:    letp lr, .LBB1_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB1_4:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp7 = icmp sgt i32 %len, 0
   br i1 %cmp7, label %vector.ph, label %for.cond.cleanup
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %len, 3
   %n.vec = and i32 %n.rnd.up, -4

llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll

 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    cbz r3, .LBB1_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    ldr r5, [sp, #28]
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    b .LBB1_4
-; CHECK-NEXT:  .LBB1_2: @ in Loop: Header=BB1_4 Depth=1
-; CHECK-NEXT:    mov r4, r3
-; CHECK-NEXT:  .LBB1_3: @ %if.end
-; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
-; CHECK-NEXT:    str.w r4, [r2, r1, lsl #2]
-; CHECK-NEXT:    adds r1, #1
-; CHECK-NEXT:    cmp r1, r3
-; CHECK-NEXT:    beq .LBB1_8
-; CHECK-NEXT:  .LBB1_4: @ %for.body
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB1_6 Depth 2
-; CHECK-NEXT:    adds r7, r5, #3
-; CHECK-NEXT:    cmp.w r12, r7, lsr #2
-; CHECK-NEXT:    beq .LBB1_2
-; CHECK-NEXT:  @ %bb.5: @ %do.body.preheader
-; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT:    b .LBB1_6
+; CHECK-NEXT:  .LBB1_2: @ %do.body.preheader
+; CHECK-NEXT:    @ in Loop: Header=BB1_6 Depth=1
 ; CHECK-NEXT:    bic r9, r7, #3
 ; CHECK-NEXT:    mov r7, r5
 ; CHECK-NEXT:    mov r4, r3
 ; CHECK-NEXT:    add.w r8, r0, r9, lsl #2
 ; CHECK-NEXT:    dlstp.32 lr, r5
-; CHECK-NEXT:  .LBB1_6: @ %do.body
-; CHECK-NEXT:    @ Parent Loop BB1_4 Depth=1
+; CHECK-NEXT:  .LBB1_3: @ %do.body
+; CHECK-NEXT:    @ Parent Loop BB1_6 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vaddva.s32 r4, q0
-; CHECK-NEXT:    letp lr, .LBB1_6
-; CHECK-NEXT:  @ %bb.7: @ %if.end.loopexit
-; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT:    letp lr, .LBB1_3
+; CHECK-NEXT:  @ %bb.4: @ %if.end.loopexit
+; CHECK-NEXT:    @ in Loop: Header=BB1_6 Depth=1
 ; CHECK-NEXT:    sub.w r5, r5, r9
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    b .LBB1_3
+; CHECK-NEXT:  .LBB1_5: @ %if.end
+; CHECK-NEXT:    @ in Loop: Header=BB1_6 Depth=1
+; CHECK-NEXT:    str.w r4, [r2, r1, lsl #2]
+; CHECK-NEXT:    adds r1, #1
+; CHECK-NEXT:    cmp r1, r3
+; CHECK-NEXT:    beq .LBB1_8
+; CHECK-NEXT:  .LBB1_6: @ %for.body
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB1_3 Depth 2
+; CHECK-NEXT:    adds r7, r5, #3
+; CHECK-NEXT:    cmp.w r12, r7, lsr #2
+; CHECK-NEXT:    bne .LBB1_2
+; CHECK-NEXT:  @ %bb.7: @ in Loop: Header=BB1_6 Depth=1
+; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:    b .LBB1_5
 ; CHECK-NEXT:  .LBB1_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
 entry:
   %cmp20.not = icmp eq i32 %m, 0
   br i1 %cmp20.not, label %for.cond.cleanup, label %for.body
 
 for.cond.cleanup:                                 ; preds = %if.end, %entry
   ret void

llvm/test/CodeGen/Thumb2/mve-float16regloops.ll

 ; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    subs r3, r7, #2
 ; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    rsbs r3, r4, #0
 ; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    add.w r3, r12, #16
 ; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    b .LBB16_4
-; CHECK-NEXT:  .LBB16_3: @ %while.end
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    b .LBB16_6
+; CHECK-NEXT:  .LBB16_3: @ %while.body76
+; CHECK-NEXT:    @ Parent Loop BB16_6 Depth=1
+; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    ldrh r1, [r6], #2
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #2
+; CHECK-NEXT:    subs.w lr, lr, #1
+; CHECK-NEXT:    vfma.f16 q0, q1, r1
+; CHECK-NEXT:    bne .LBB16_3
+; CHECK-NEXT:  @ %bb.4: @ %while.end.loopexit
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT:    add.w r5, r5, r8, lsl #1
+; CHECK-NEXT:  .LBB16_5: @ %while.end
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    subs.w r9, r9, #1
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #8
 ; CHECK-NEXT:    add.w r0, r5, r0, lsl #1
 ; CHECK-NEXT:    add.w r5, r0, #8
 ; CHECK-NEXT:    beq.w .LBB16_12
-; CHECK-NEXT:  .LBB16_4: @ %while.body
+; CHECK-NEXT:  .LBB16_6: @ %while.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB16_6 Depth 2
-; CHECK-NEXT:    @ Child Loop BB16_10 Depth 2
+; CHECK-NEXT:    @ Child Loop BB16_8 Depth 2
+; CHECK-NEXT:    @ Child Loop BB16_3 Depth 2
 ; CHECK-NEXT:    vldrw.u32 q0, [r1], #8
 ; CHECK-NEXT:    ldrh.w lr, [r12, #14]
 ; CHECK-NEXT:    ldrh.w r0, [r12, #12]
 ; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    ldrh.w r4, [r12, #10]
 ; CHECK-NEXT:    ldrh.w r7, [r12, #8]
 ; CHECK-NEXT:    ldrh.w r3, [r12, #6]

 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #12]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    add.w r0, r5, #14
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    adds r5, #16
 ; CHECK-NEXT:    vfma.f16 q0, q1, lr
 ; CHECK-NEXT:    cmp r0, #16
-; CHECK-NEXT:    blo .LBB16_7
-; CHECK-NEXT:  @ %bb.5: @ %for.body.preheader
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    blo .LBB16_11
+; CHECK-NEXT:  @ %bb.7: @ %for.body.preheader
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    dls lr, r0
 ; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:  .LBB16_6: @ %for.body
-; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
+; CHECK-NEXT:  .LBB16_8: @ %for.body
+; CHECK-NEXT:    @ Parent Loop BB16_6 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldrh r0, [r6], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r5]
 ; CHECK-NEXT:    adds r1, r5, #2
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    ldrh r0, [r6, #-14]
 ; CHECK-NEXT:    adds r1, r5, #6

 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    ldrh r0, [r6, #-4]
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #12]
 ; CHECK-NEXT:    vfma.f16 q0, q1, r0
 ; CHECK-NEXT:    add.w r0, r5, #14
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    adds r5, #16
 ; CHECK-NEXT:    vfma.f16 q0, q1, r1
-; CHECK-NEXT:    le lr, .LBB16_6
-; CHECK-NEXT:    b .LBB16_8
-; CHECK-NEXT:  .LBB16_7: @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:  .LBB16_8: @ %for.end
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    le lr, .LBB16_8
+; CHECK-NEXT:  .LBB16_9: @ %for.end
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    beq.w .LBB16_3
-; CHECK-NEXT:    b .LBB16_9
-; CHECK-NEXT:  .LBB16_9: @ %while.body76.preheader
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    beq.w .LBB16_5
+; CHECK-NEXT:  @ %bb.10: @ %while.body76.preheader
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov lr, r8
-; CHECK-NEXT:  .LBB16_10: @ %while.body76
-; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
-; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldrh r1, [r6], #2
-; CHECK-NEXT:    vldrh.u16 q1, [r0], #2
-; CHECK-NEXT:    subs.w lr, lr, #1
-; CHECK-NEXT:    vfma.f16 q0, q1, r1
-; CHECK-NEXT:    bne .LBB16_10
-; CHECK-NEXT:    b .LBB16_11
-; CHECK-NEXT:  .LBB16_11: @ %while.end.loopexit
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    add.w r5, r5, r8, lsl #1
 ; CHECK-NEXT:    b .LBB16_3
+; CHECK-NEXT:  .LBB16_11: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    b .LBB16_9
 ; CHECK-NEXT:  .LBB16_12: @ %if.end
 ; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %pState1 = getelementptr inbounds %struct.arm_fir_instance_f32, %struct.arm_fir_instance_f32* %S, i32 0, i32 1
   %0 = load half*, half** %pState1, align 4
   %pCoeffs2 = getelementptr inbounds %struct.arm_fir_instance_f32, %struct.arm_fir_instance_f32* %S, i32 0, i32 2
   %1 = load half*, half** %pCoeffs2, align 4

llvm/test/CodeGen/Thumb2/mve-float32regloops.ll

 ; CHECK-NEXT:    rsbs r0, r4, #0
 ; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    and r3, r7, #7
 ; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    add.w r0, r12, #32
 ; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    b .LBB16_4
-; CHECK-NEXT:  .LBB16_3: @ %while.end
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    b .LBB16_6
+; CHECK-NEXT:  .LBB16_3: @ %while.body76
+; CHECK-NEXT:    @ Parent Loop BB16_6 Depth=1
+; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    ldr r0, [r7], #4
+; CHECK-NEXT:    vldrw.u32 q1, [r6], #4
+; CHECK-NEXT:    subs.w lr, lr, #1
+; CHECK-NEXT:    vfma.f32 q0, q1, r0
+; CHECK-NEXT:    bne .LBB16_3
+; CHECK-NEXT:  @ %bb.4: @ %while.end.loopexit
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT:    add.w r5, r5, r3, lsl #2
+; CHECK-NEXT:  .LBB16_5: @ %while.end
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    subs.w r10, r10, #1
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    add.w r0, r5, r0, lsl #2
 ; CHECK-NEXT:    add.w r5, r0, #16
 ; CHECK-NEXT:    beq .LBB16_12
-; CHECK-NEXT:  .LBB16_4: @ %while.body
+; CHECK-NEXT:  .LBB16_6: @ %while.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB16_6 Depth 2
-; CHECK-NEXT:    @ Child Loop BB16_10 Depth 2
+; CHECK-NEXT:    @ Child Loop BB16_8 Depth 2
+; CHECK-NEXT:    @ Child Loop BB16_3 Depth 2
 ; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
 ; CHECK-NEXT:    ldrd r7, r6, [r12]
 ; CHECK-NEXT:    ldrd r0, r4, [r12, #8]
 ; CHECK-NEXT:    ldrd r3, lr, [r12, #16]
 ; CHECK-NEXT:    ldrd r11, r8, [r12, #24]
 ; CHECK-NEXT:    vstrb.8 q0, [r9], #16
 ; CHECK-NEXT:    vldrw.u32 q0, [r5], #32
 ; CHECK-NEXT:    strd r9, r1, [sp, #24] @ 8-byte Folded Spill

 ; CHECK-NEXT:    vldrw.u32 q3, [r5, #-8]
 ; CHECK-NEXT:    vfma.f32 q0, q5, r3
 ; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    vfma.f32 q0, q2, lr
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #-4]
 ; CHECK-NEXT:    vfma.f32 q0, q3, r11
 ; CHECK-NEXT:    cmp r0, #16
 ; CHECK-NEXT:    vfma.f32 q0, q1, r8
-; CHECK-NEXT:    blo .LBB16_7
-; CHECK-NEXT:  @ %bb.5: @ %for.body.preheader
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    blo .LBB16_11
+; CHECK-NEXT:  @ %bb.7: @ %for.body.preheader
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    dls lr, r0
 ; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:  .LBB16_6: @ %for.body
-; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
+; CHECK-NEXT:  .LBB16_8: @ %for.body
+; CHECK-NEXT:    @ Parent Loop BB16_6 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldm.w r7, {r0, r3, r4, r6}
 ; CHECK-NEXT:    vldrw.u32 q1, [r5], #32
 ; CHECK-NEXT:    add.w r11, r7, #16
 ; CHECK-NEXT:    vldrw.u32 q6, [r5, #-24]
 ; CHECK-NEXT:    vldrw.u32 q4, [r5, #-20]
 ; CHECK-NEXT:    vfma.f32 q0, q1, r0
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #-28]
 ; CHECK-NEXT:    ldm.w r11, {r1, r8, r11}
 ; CHECK-NEXT:    vldrw.u32 q5, [r5, #-16]
 ; CHECK-NEXT:    vfma.f32 q0, q1, r3
 ; CHECK-NEXT:    vldrw.u32 q2, [r5, #-12]
 ; CHECK-NEXT:    vfma.f32 q0, q6, r4
 ; CHECK-NEXT:    vldrw.u32 q3, [r5, #-8]
 ; CHECK-NEXT:    vfma.f32 q0, q4, r6
 ; CHECK-NEXT:    ldr.w r9, [r7, #28]
 ; CHECK-NEXT:    vfma.f32 q0, q5, r1
 ; CHECK-NEXT:    vldrw.u32 q1, [r5, #-4]
 ; CHECK-NEXT:    vfma.f32 q0, q2, r8
 ; CHECK-NEXT:    adds r7, #32
 ; CHECK-NEXT:    vfma.f32 q0, q3, r11
 ; CHECK-NEXT:    vfma.f32 q0, q1, r9
-; CHECK-NEXT:    le lr, .LBB16_6
-; CHECK-NEXT:    b .LBB16_8
-; CHECK-NEXT:  .LBB16_7: @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:  .LBB16_8: @ %for.end
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    le lr, .LBB16_8
+; CHECK-NEXT:  .LBB16_9: @ %for.end
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    ldrd r9, r1, [sp, #24] @ 8-byte Folded Reload
 ; CHECK-NEXT:    ldr r3, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    cmp.w r3, #0
-; CHECK-NEXT:    beq .LBB16_3
-; CHECK-NEXT:    b .LBB16_9
-; CHECK-NEXT:  .LBB16_9: @ %while.body76.preheader
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT:    beq .LBB16_5
+; CHECK-NEXT:  @ %bb.10: @ %while.body76.preheader
+; CHECK-NEXT:    @ in Loop: Header=BB16_6 Depth=1
 ; CHECK-NEXT:    mov r6, r5
 ; CHECK-NEXT:    mov lr, r3
-; CHECK-NEXT:  .LBB16_10: @ %while.body76
-; CHECK-NEXT:    @ Parent Loop BB16_4 Depth=1
-; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    ldr r0, [r7], #4
-; CHECK-NEXT:    vldrw.u32 q1, [r6], #4
-; CHECK-NEXT:    subs.w lr, lr, #1
-; CHECK-NEXT:    vfma.f32 q0, q1, r0
-; CHECK-NEXT:    bne .LBB16_10
-; CHECK-NEXT:    b .LBB16_11
-; CHECK-NEXT:  .LBB16_11: @ %while.end.loopexit
-; CHECK-NEXT:    @ in Loop: Header=BB16_4 Depth=1
-; CHECK-NEXT:    add.w r5, r5, r3, lsl #2
 ; CHECK-NEXT:    b .LBB16_3
+; CHECK-NEXT:  .LBB16_11: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    b .LBB16_9
 ; CHECK-NEXT:  .LBB16_12: @ %if.end
 ; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %pState1 = getelementptr inbounds %struct.arm_fir_instance_f32, %struct.arm_fir_instance_f32* %S, i32 0, i32 1
   %0 = load float*, float** %pState1, align 4

 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    ldr.w lr, [sp, #16]
 ; CHECK-NEXT:    movs r4, #0
 ; CHECK-NEXT:    cmp.w r4, lr, lsr #2
 ; CHECK-NEXT:    beq .LBB18_5
 ; CHECK-NEXT:  @ %bb.1: @ %do.body.preheader
 ; CHECK-NEXT:    ldr.w r12, [sp, #20]
 ; CHECK-NEXT:    lsr.w r5, lr, #2
-; CHECK-NEXT:  .LBB18_2: @ %do.body
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB18_3 Depth 2
-; CHECK-NEXT:    ldr r4, [r2]
-; CHECK-NEXT:    dls lr, r5
-; CHECK-NEXT:    vdup.32 q0, r4
-; CHECK-NEXT:  .LBB18_3: @ %while.body
-; CHECK-NEXT:    @ Parent Loop BB18_2 Depth=1
+; CHECK-NEXT:    b .LBB18_4
+; CHECK-NEXT:  .LBB18_2: @ %while.body
+; CHECK-NEXT:    @ Parent Loop BB18_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
 ; CHECK-NEXT:    vfms.f32 q2, q1, q0
 ; CHECK-NEXT:    vstrb.8 q2, [r3], #16
-; CHECK-NEXT:    le lr, .LBB18_3
-; CHECK-NEXT:  @ %bb.4: @ %while.end
-; CHECK-NEXT:    @ in Loop: Header=BB18_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB18_2
+; CHECK-NEXT:  @ %bb.3: @ %while.end
+; CHECK-NEXT:    @ in Loop: Header=BB18_4 Depth=1
 ; CHECK-NEXT:    subs.w r12, r12, #1
 ; CHECK-NEXT:    add.w r2, r2, #4
-; CHECK-NEXT:    bne .LBB18_2
+; CHECK-NEXT:    beq .LBB18_5
+; CHECK-NEXT:  .LBB18_4: @ %do.body
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB18_2 Depth 2
+; CHECK-NEXT:    ldr r4, [r2]
+; CHECK-NEXT:    dls lr, r5
+; CHECK-NEXT:    vdup.32 q0, r4
+; CHECK-NEXT:    b .LBB18_2
 ; CHECK-NEXT:  .LBB18_5: @ %do.end
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %shr = lshr i32 %N, 2
   %cmp15 = icmp eq i32 %shr, 0
   br i1 %cmp15, label %do.end, label %do.body
 
 do.body:                                          ; preds = %entry, %while.end

llvm/test/CodeGen/Thumb2/mve-gather-increment.ll

 ; CHECK-NEXT:  .LBB8_1: @ %vector.ph.preheader
 ; CHECK-NEXT:    bic r12, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w lr, r12, #4
 ; CHECK-NEXT:    add.w r4, r3, lr, lsr #2
 ; CHECK-NEXT:    adr r3, .LCPI8_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r3]
 ; CHECK-NEXT:    vadd.i32 q0, q0, r0
-; CHECK-NEXT:  .LBB8_2: @ %vector.ph
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB8_3 Depth 2
-; CHECK-NEXT:    dls lr, r4
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:  .LBB8_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB8_2 Depth=1
+; CHECK-NEXT:    b .LBB8_4
+; CHECK-NEXT:  .LBB8_2: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB8_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrw.u32 q2, [q1, #16]!
 ; CHECK-NEXT:    vstrb.8 q2, [r0], #16
-; CHECK-NEXT:    le lr, .LBB8_3
-; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB8_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB8_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB8_4 Depth=1
 ; CHECK-NEXT:    cmp r12, r2
-; CHECK-NEXT:    bne .LBB8_2
-; CHECK-NEXT:  @ %bb.5: @ %for.cond.cleanup
+; CHECK-NEXT:    beq .LBB8_5
+; CHECK-NEXT:  .LBB8_4: @ %vector.ph
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB8_2 Depth 2
+; CHECK-NEXT:    dls lr, r4
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    b .LBB8_2
+; CHECK-NEXT:  .LBB8_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.6:
 ; CHECK-NEXT:  .LCPI8_0:
 ; CHECK-NEXT:    .long 4294967280 @ 0xfffffff0
 ; CHECK-NEXT:    .long 4294967284 @ 0xfffffff4
 ; CHECK-NEXT:    .long 4294967288 @ 0xfffffff8
 ; CHECK-NEXT:    .long 4294967292 @ 0xfffffffc

 ; CHECK-NEXT:    vldrw.u32 q1, [r4]
 ; CHECK-NEXT:    add.w r3, r3, lr, lsr #2
 ; CHECK-NEXT:    adr.w lr, .LCPI9_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r5]
 ; CHECK-NEXT:    vldrw.u32 q2, [lr]
 ; CHECK-NEXT:    vadd.i32 q1, q1, r0
 ; CHECK-NEXT:    vadd.i32 q0, q0, r0
 ; CHECK-NEXT:    vadd.i32 q2, q2, r0
-; CHECK-NEXT:  .LBB9_2: @ %vector.ph
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB9_3 Depth 2
-; CHECK-NEXT:    dls lr, r3
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    vmov q3, q1
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov q5, q2
-; CHECK-NEXT:  .LBB9_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB9_2 Depth=1
+; CHECK-NEXT:    b .LBB9_4
+; CHECK-NEXT:  .LBB9_2: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB9_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrw.u32 q6, [q5, #48]!
 ; CHECK-NEXT:    vldrw.u32 q7, [q3, #48]!
 ; CHECK-NEXT:    vadd.i32 q6, q7, q6
 ; CHECK-NEXT:    vldrw.u32 q7, [q4, #48]!
 ; CHECK-NEXT:    vadd.i32 q6, q6, q7
 ; CHECK-NEXT:    vstrb.8 q6, [r0], #16
-; CHECK-NEXT:    le lr, .LBB9_3
-; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB9_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB9_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB9_4 Depth=1
 ; CHECK-NEXT:    cmp r12, r2
-; CHECK-NEXT:    bne .LBB9_2
+; CHECK-NEXT:    beq .LBB9_5
+; CHECK-NEXT:  .LBB9_4: @ %vector.ph
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB9_2 Depth 2
+; CHECK-NEXT:    dls lr, r3
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    vmov q3, q1
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov q5, q2
+; CHECK-NEXT:    b .LBB9_2
 ; CHECK-NEXT:  .LBB9_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.6:
 ; CHECK-NEXT:  .LCPI9_0:
 ; CHECK-NEXT:    .long 4294967248 @ 0xffffffd0
 ; CHECK-NEXT:    .long 4294967260 @ 0xffffffdc

 ; CHECK-NEXT:  .LBB10_1: @ %vector.ph.preheader
 ; CHECK-NEXT:    bic r12, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w lr, r12, #4
 ; CHECK-NEXT:    add.w r4, r3, lr, lsr #2
 ; CHECK-NEXT:    adr r3, .LCPI10_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r3]
 ; CHECK-NEXT:    vadd.i32 q0, q0, r0
-; CHECK-NEXT:  .LBB10_2: @ %vector.ph
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB10_3 Depth 2
-; CHECK-NEXT:    dls lr, r4
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:  .LBB10_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB10_2 Depth=1
+; CHECK-NEXT:    b .LBB10_4
+; CHECK-NEXT:  .LBB10_2: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB10_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrw.u32 q2, [q1, #508]!
 ; CHECK-NEXT:    vstrb.8 q2, [r0], #16
-; CHECK-NEXT:    le lr, .LBB10_3
-; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB10_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB10_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB10_4 Depth=1
 ; CHECK-NEXT:    cmp r12, r2
-; CHECK-NEXT:    bne .LBB10_2
-; CHECK-NEXT:  @ %bb.5: @ %for.cond.cleanup
+; CHECK-NEXT:    beq .LBB10_5
+; CHECK-NEXT:  .LBB10_4: @ %vector.ph
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB10_2 Depth 2
+; CHECK-NEXT:    dls lr, r4
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    b .LBB10_2
+; CHECK-NEXT:  .LBB10_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.6:
 ; CHECK-NEXT:  .LCPI10_0:
 ; CHECK-NEXT:    .long 4294966788 @ 0xfffffe04
 ; CHECK-NEXT:    .long 4294966792 @ 0xfffffe08
 ; CHECK-NEXT:    .long 4294966796 @ 0xfffffe0c
 ; CHECK-NEXT:    .long 4294966800 @ 0xfffffe10

 ; CHECK-NEXT:    bic r8, r2, #7
 ; CHECK-NEXT:    movs r5, #1
 ; CHECK-NEXT:    sub.w r6, r8, #8
 ; CHECK-NEXT:    vmov.i16 q1, #0x8
 ; CHECK-NEXT:    add.w r1, r5, r6, lsr #3
 ; CHECK-NEXT:    adr r6, .LCPI11_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r6]
 ; CHECK-NEXT:    str r1, [sp] @ 4-byte Spill
-; CHECK-NEXT:  .LBB11_2: @ %vector.ph
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB11_3 Depth 2
-; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; CHECK-NEXT:    vmov q2, q0
-; CHECK-NEXT:    dls lr, r1
-; CHECK-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:  .LBB11_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=1
+; CHECK-NEXT:    b .LBB11_4
+; CHECK-NEXT:  .LBB11_2: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB11_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vmov.u16 r7, q2[6]
 ; CHECK-NEXT:    vmov.u16 r3, q2[4]
 ; CHECK-NEXT:    vmov q4[2], q4[0], r3, r7
 ; CHECK-NEXT:    vmov.u16 r3, q2[7]
 ; CHECK-NEXT:    vmov.u16 r7, q2[5]
 ; CHECK-NEXT:    vmov.u16 r5, q2[2]
 ; CHECK-NEXT:    vmov q4[3], q4[1], r7, r3

 ; CHECK-NEXT:    ldrh r5, [r5]
 ; CHECK-NEXT:    vmov.16 q3[3], r9
 ; CHECK-NEXT:    ldrh r6, [r6]
 ; CHECK-NEXT:    vmov.16 q3[4], r11
 ; CHECK-NEXT:    vmov.16 q3[5], r7
 ; CHECK-NEXT:    vmov.16 q3[6], r5
 ; CHECK-NEXT:    vmov.16 q3[7], r6
 ; CHECK-NEXT:    vstrb.8 q3, [r4], #16
-; CHECK-NEXT:    le lr, .LBB11_3
-; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB11_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB11_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB11_4 Depth=1
 ; CHECK-NEXT:    cmp r8, r2
-; CHECK-NEXT:    bne .LBB11_2
+; CHECK-NEXT:    beq .LBB11_5
+; CHECK-NEXT:  .LBB11_4: @ %vector.ph
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB11_2 Depth 2
+; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    dls lr, r1
+; CHECK-NEXT:    ldr r4, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    b .LBB11_2
 ; CHECK-NEXT:  .LBB11_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.6:
 ; CHECK-NEXT:  .LCPI11_0:

 ; CHECK-NEXT:    vldrw.u32 q0, [r3]
 ; CHECK-NEXT:    adr r7, .LCPI12_1
 ; CHECK-NEXT:    vmov.i16 q3, #0x18
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r7]
 ; CHECK-NEXT:    str r1, [sp, #52] @ 4-byte Spill
 ; CHECK-NEXT:    vstrw.32 q3, [sp, #64] @ 16-byte Spill
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
-; CHECK-NEXT:  .LBB12_2: @ %vector.ph
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB12_3 Depth 2
-; CHECK-NEXT:    ldr r1, [sp, #52] @ 4-byte Reload
-; CHECK-NEXT:    dls lr, r1
-; CHECK-NEXT:    ldr r4, [sp, #60] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q7, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT:    vldrw.u32 q5, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vldrw.u32 q6, [sp] @ 16-byte Reload
-; CHECK-NEXT:  .LBB12_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB12_2 Depth=1
+; CHECK-NEXT:    b .LBB12_4
+; CHECK-NEXT:  .LBB12_2: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB12_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vmov.u16 r3, q5[2]
 ; CHECK-NEXT:    vmov.u16 r5, q5[0]
 ; CHECK-NEXT:    vmov q0[2], q0[0], r5, r3
 ; CHECK-NEXT:    vmov.u16 r3, q5[3]
 ; CHECK-NEXT:    vmov.u16 r5, q5[1]
 ; CHECK-NEXT:    vmov.u16 r7, q7[6]
 ; CHECK-NEXT:    vmov q0[3], q0[1], r5, r3

 ; CHECK-NEXT:    vmov.16 q0[5], r1
 ; CHECK-NEXT:    vmov r1, s19
 ; CHECK-NEXT:    vmov.16 q0[6], r7
 ; CHECK-NEXT:    ldrh r1, [r1]
 ; CHECK-NEXT:    vmov.16 q0[7], r1
 ; CHECK-NEXT:    vadd.i16 q0, q0, q2
 ; CHECK-NEXT:    vadd.i16 q0, q0, q1
 ; CHECK-NEXT:    vstrb.8 q0, [r4], #16
-; CHECK-NEXT:    le lr, .LBB12_3
-; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB12_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB12_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB12_4 Depth=1
 ; CHECK-NEXT:    ldr r1, [sp, #56] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r1, r2
-; CHECK-NEXT:    bne.w .LBB12_2
+; CHECK-NEXT:    beq .LBB12_5
+; CHECK-NEXT:  .LBB12_4: @ %vector.ph
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB12_2 Depth 2
+; CHECK-NEXT:    ldr r1, [sp, #52] @ 4-byte Reload
+; CHECK-NEXT:    dls lr, r1
+; CHECK-NEXT:    ldr r4, [sp, #60] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q7, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q6, [sp] @ 16-byte Reload
+; CHECK-NEXT:    b .LBB12_2
 ; CHECK-NEXT:  .LBB12_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    add sp, #104
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.6:
 ; CHECK-NEXT:  .LCPI12_0:

llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll

 ; CHECK-NEXT:    vshl.i32 q3, q1, #3
 ; CHECK-NEXT:    subs r7, #4
 ; CHECK-NEXT:    add.w r10, r6, r7, lsr #2
 ; CHECK-NEXT:    adr r7, .LCPI9_0
 ; CHECK-NEXT:    adr r6, .LCPI9_1
 ; CHECK-NEXT:    vldrw.u32 q2, [r7]
 ; CHECK-NEXT:    vldrw.u32 q0, [r6]
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
-; CHECK-NEXT:  .LBB9_1: @ %for.cond8.preheader.us.us.preheader
+; CHECK-NEXT:    b .LBB9_2
+; CHECK-NEXT:  .LBB9_1: @ %for.cond4.for.cond.cleanup6_crit_edge.us
+; CHECK-NEXT:    @ in Loop: Header=BB9_2 Depth=1
+; CHECK-NEXT:    add.w r8, r8, #1
+; CHECK-NEXT:    cmp r8, r3
+; CHECK-NEXT:    beq .LBB9_6
+; CHECK-NEXT:  .LBB9_2: @ %for.cond8.preheader.us.us.preheader
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB9_2 Depth 2
+; CHECK-NEXT:    @ Child Loop BB9_5 Depth 2
 ; CHECK-NEXT:    @ Child Loop BB9_3 Depth 3
 ; CHECK-NEXT:    mul r11, r8, r9
 ; CHECK-NEXT:    movs r5, #0
 ; CHECK-NEXT:    mul r7, r8, r12
-; CHECK-NEXT:  .LBB9_2: @ %vector.ph
-; CHECK-NEXT:    @ Parent Loop BB9_1 Depth=1
-; CHECK-NEXT:    @ => This Loop Header: Depth=2
-; CHECK-NEXT:    @ Child Loop BB9_3 Depth 3
-; CHECK-NEXT:    vdup.32 q5, r7
-; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vshl.i32 q5, q5, #2
-; CHECK-NEXT:    vmov q6, q1
-; CHECK-NEXT:    vadd.i32 q5, q5, r0
-; CHECK-NEXT:    dls lr, r10
-; CHECK-NEXT:    vmov.i32 q4, #0x0
-; CHECK-NEXT:    vadd.i32 q5, q5, q0
-; CHECK-NEXT:    vmlas.u32 q6, q2, r5
+; CHECK-NEXT:    b .LBB9_5
 ; CHECK-NEXT:  .LBB9_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB9_1 Depth=1
-; CHECK-NEXT:    @ Parent Loop BB9_2 Depth=2
+; CHECK-NEXT:    @ Parent Loop BB9_2 Depth=1
+; CHECK-NEXT:    @ Parent Loop BB9_5 Depth=2
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=3
 ; CHECK-NEXT:    vadd.i32 q7, q6, q3
 ; CHECK-NEXT:    vldrw.u32 q0, [r1, q6, uxtw #2]
 ; CHECK-NEXT:    vldrw.u32 q6, [q5, #32]!
 ; CHECK-NEXT:    vmul.i32 q0, q0, q6
 ; CHECK-NEXT:    vmov q6, q7
 ; CHECK-NEXT:    vadd.i32 q4, q0, q4
 ; CHECK-NEXT:    le lr, .LBB9_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB9_2 Depth=2
+; CHECK-NEXT:    @ in Loop: Header=BB9_5 Depth=2
 ; CHECK-NEXT:    add.w r4, r5, r11
 ; CHECK-NEXT:    adds r5, #1
 ; CHECK-NEXT:    vaddv.u32 r6, q4
 ; CHECK-NEXT:    cmp r5, r9
 ; CHECK-NEXT:    str.w r6, [r2, r4, lsl #2]
-; CHECK-NEXT:    bne .LBB9_2
-; CHECK-NEXT:  @ %bb.5: @ %for.cond4.for.cond.cleanup6_crit_edge.us
-; CHECK-NEXT:    @ in Loop: Header=BB9_1 Depth=1
-; CHECK-NEXT:    add.w r8, r8, #1
-; CHECK-NEXT:    cmp r8, r3
-; CHECK-NEXT:    bne .LBB9_1
-; CHECK-NEXT:  @ %bb.6: @ %for.end25
+; CHECK-NEXT:    beq .LBB9_1
+; CHECK-NEXT:  .LBB9_5: @ %vector.ph
+; CHECK-NEXT:    @ Parent Loop BB9_2 Depth=1
+; CHECK-NEXT:    @ => This Loop Header: Depth=2
+; CHECK-NEXT:    @ Child Loop BB9_3 Depth 3
+; CHECK-NEXT:    vdup.32 q5, r7
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vshl.i32 q5, q5, #2
+; CHECK-NEXT:    vmov q6, q1
+; CHECK-NEXT:    vadd.i32 q5, q5, r0
+; CHECK-NEXT:    dls lr, r10
+; CHECK-NEXT:    vmov.i32 q4, #0x0
+; CHECK-NEXT:    vadd.i32 q5, q5, q0
+; CHECK-NEXT:    vmlas.u32 q6, q2, r5
+; CHECK-NEXT:    b .LBB9_3
+; CHECK-NEXT:  .LBB9_6: @ %for.end25
 ; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.7:
 ; CHECK-NEXT:  .LCPI9_0:
 ; CHECK-NEXT:    .long 0 @ 0x0

 ; CHECK-NEXT:    ldr r1, [sp, #96]
 ; CHECK-NEXT:    vdup.32 q1, r2
 ; CHECK-NEXT:    vldrw.u32 q0, [r7]
 ; CHECK-NEXT:    mov.w r10, #0
 ; CHECK-NEXT:    mov.w r9, #6
 ; CHECK-NEXT:    movs r6, #11
 ; CHECK-NEXT:    vshl.i32 q1, q1, #2
 ; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:  .LBB11_1: @ %for.body10.i
+; CHECK-NEXT:    b .LBB11_2
+; CHECK-NEXT:  .LBB11_1: @ %for.cond.cleanup20.i
+; CHECK-NEXT:    @ in Loop: Header=BB11_2 Depth=1
+; CHECK-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    ldr r7, [sp, #148]
+; CHECK-NEXT:    adds r5, #1
+; CHECK-NEXT:    cmp r5, r7
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq r5, #0
+; CHECK-NEXT:  .LBB11_2: @ %for.body10.i
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB11_2 Depth 2
-; CHECK-NEXT:    @ Child Loop BB11_3 Depth 3
-; CHECK-NEXT:    @ Child Loop BB11_4 Depth 4
-; CHECK-NEXT:    @ Child Loop BB11_5 Depth 5
+; CHECK-NEXT:    @ Child Loop BB11_4 Depth 2
+; CHECK-NEXT:    @ Child Loop BB11_9 Depth 3
+; CHECK-NEXT:    @ Child Loop BB11_5 Depth 4
+; CHECK-NEXT:    @ Child Loop BB11_6 Depth 5
 ; CHECK-NEXT:    movs r7, #0
 ; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:  .LBB11_2: @ %for.cond22.preheader.i
-; CHECK-NEXT:    @ Parent Loop BB11_1 Depth=1
+; CHECK-NEXT:    b .LBB11_4
+; CHECK-NEXT:  .LBB11_3: @ %for.cond.cleanup26.i
+; CHECK-NEXT:    @ in Loop: Header=BB11_4 Depth=2
+; CHECK-NEXT:    adds r7, #1
+; CHECK-NEXT:    cmp r7, r3
+; CHECK-NEXT:    beq .LBB11_1
+; CHECK-NEXT:  .LBB11_4: @ %for.cond22.preheader.i
+; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=1
 ; CHECK-NEXT:    @ => This Loop Header: Depth=2
-; CHECK-NEXT:    @ Child Loop BB11_3 Depth 3
-; CHECK-NEXT:    @ Child Loop BB11_4 Depth 4
-; CHECK-NEXT:    @ Child Loop BB11_5 Depth 5
+; CHECK-NEXT:    @ Child Loop BB11_9 Depth 3
+; CHECK-NEXT:    @ Child Loop BB11_5 Depth 4
+; CHECK-NEXT:    @ Child Loop BB11_6 Depth 5
 ; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:  .LBB11_3: @ %for.body27.i
-; CHECK-NEXT:    @ Parent Loop BB11_1 Depth=1
-; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=2
-; CHECK-NEXT:    @ => This Loop Header: Depth=3
-; CHECK-NEXT:    @ Child Loop BB11_4 Depth 4
-; CHECK-NEXT:    @ Child Loop BB11_5 Depth 5
-; CHECK-NEXT:    dls lr, r9
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    mov.w r11, #4
-; CHECK-NEXT:  .LBB11_4: @ %for.body78.us.i
-; CHECK-NEXT:    @ Parent Loop BB11_1 Depth=1
-; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=2
-; CHECK-NEXT:    @ Parent Loop BB11_3 Depth=3
+; CHECK-NEXT:    b .LBB11_9
+; CHECK-NEXT:  .LBB11_5: @ %for.body78.us.i
+; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=1
+; CHECK-NEXT:    @ Parent Loop BB11_4 Depth=2
+; CHECK-NEXT:    @ Parent Loop BB11_9 Depth=3
 ; CHECK-NEXT:    @ => This Loop Header: Depth=4
-; CHECK-NEXT:    @ Child Loop BB11_5 Depth 5
+; CHECK-NEXT:    @ Child Loop BB11_6 Depth 5
 ; CHECK-NEXT:    mul r4, r11, r6
 ; CHECK-NEXT:    vdup.32 q3, r5
 ; CHECK-NEXT:    vdup.32 q2, r7
 ; CHECK-NEXT:    vadd.i32 q4, q0, r4
 ; CHECK-NEXT:    vmla.u32 q3, q4, r2
 ; CHECK-NEXT:    adds r4, #113
 ; CHECK-NEXT:    vadd.i32 q4, q0, r4
 ; CHECK-NEXT:    mov r4, r8
 ; CHECK-NEXT:    vmla.u32 q2, q4, r2
-; CHECK-NEXT:  .LBB11_5: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB11_1 Depth=1
-; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=2
-; CHECK-NEXT:    @ Parent Loop BB11_3 Depth=3
-; CHECK-NEXT:    @ Parent Loop BB11_4 Depth=4
+; CHECK-NEXT:  .LBB11_6: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=1
+; CHECK-NEXT:    @ Parent Loop BB11_4 Depth=2
+; CHECK-NEXT:    @ Parent Loop BB11_9 Depth=3
+; CHECK-NEXT:    @ Parent Loop BB11_5 Depth=4
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=5
 ; CHECK-NEXT:    vldrb.s32 q6, [r0, q2]
 ; CHECK-NEXT:    vadd.i32 q5, q2, q1
 ; CHECK-NEXT:    vadd.i32 q4, q3, q1
 ; CHECK-NEXT:    subs r4, #4
 ; CHECK-NEXT:    vadd.i32 q2, q6, r2
 ; CHECK-NEXT:    vldrb.s32 q6, [r1, q3]
 ; CHECK-NEXT:    vmov q3, q4
 ; CHECK-NEXT:    vmlava.u32 r12, q2, q6
 ; CHECK-NEXT:    vmov q2, q5
-; CHECK-NEXT:    bne .LBB11_5
-; CHECK-NEXT:  @ %bb.6: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB11_4 Depth=4
+; CHECK-NEXT:    bne .LBB11_6
+; CHECK-NEXT:  @ %bb.7: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB11_5 Depth=4
 ; CHECK-NEXT:    add.w r11, r11, #1
-; CHECK-NEXT:    le lr, .LBB11_4
-; CHECK-NEXT:  @ %bb.7: @ %for.cond.cleanup77.i
-; CHECK-NEXT:    @ in Loop: Header=BB11_3 Depth=3
+; CHECK-NEXT:    le lr, .LBB11_5
+; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup77.i
+; CHECK-NEXT:    @ in Loop: Header=BB11_9 Depth=3
 ; CHECK-NEXT:    adds r5, #1
 ; CHECK-NEXT:    add.w r10, r10, #1
 ; CHECK-NEXT:    cmp r5, r2
-; CHECK-NEXT:    bne .LBB11_3
-; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup26.i
-; CHECK-NEXT:    @ in Loop: Header=BB11_2 Depth=2
-; CHECK-NEXT:    adds r7, #1
-; CHECK-NEXT:    cmp r7, r3
-; CHECK-NEXT:    bne .LBB11_2
-; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup20.i
-; CHECK-NEXT:    @ in Loop: Header=BB11_1 Depth=1
-; CHECK-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    ldr r7, [sp, #148]
-; CHECK-NEXT:    adds r5, #1
-; CHECK-NEXT:    cmp r5, r7
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq r5, #0
-; CHECK-NEXT:    b .LBB11_1
+; CHECK-NEXT:    beq .LBB11_3
+; CHECK-NEXT:  .LBB11_9: @ %for.body27.i
+; CHECK-NEXT:    @ Parent Loop BB11_2 Depth=1
+; CHECK-NEXT:    @ Parent Loop BB11_4 Depth=2
+; CHECK-NEXT:    @ => This Loop Header: Depth=3
+; CHECK-NEXT:    @ Child Loop BB11_5 Depth 4
+; CHECK-NEXT:    @ Child Loop BB11_6 Depth 5
+; CHECK-NEXT:    dls lr, r9
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    mov.w r11, #4
+; CHECK-NEXT:    b .LBB11_5
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.10:
 ; CHECK-NEXT:  .LCPI11_0:
 ; CHECK-NEXT:    .long 0 @ 0x0
 ; CHECK-NEXT:    .long 1 @ 0x1
 ; CHECK-NEXT:    .long 2 @ 0x2
 ; CHECK-NEXT:    .long 3 @ 0x3
 entry:

llvm/test/CodeGen/Thumb2/mve-gather-tailpred.ll

 ; CHECK-NEXT:  .LBB0_1: @ %vector.ph.preheader
 ; CHECK-NEXT:    bic r12, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w lr, r12, #4
 ; CHECK-NEXT:    add.w r4, r3, lr, lsr #2
 ; CHECK-NEXT:    adr r3, .LCPI0_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r3]
 ; CHECK-NEXT:    vadd.i32 q0, q0, r0
-; CHECK-NEXT:  .LBB0_2: @ %vector.ph
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB0_3 Depth 2
-; CHECK-NEXT:    dls lr, r4
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:  .LBB0_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    b .LBB0_4
+; CHECK-NEXT:  .LBB0_2: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB0_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vldrw.u32 q2, [q1, #16]!
 ; CHECK-NEXT:    vstrb.8 q2, [r0], #16
-; CHECK-NEXT:    le lr, .LBB0_3
-; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB0_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    cmp r12, r2
-; CHECK-NEXT:    bne .LBB0_2
-; CHECK-NEXT:  @ %bb.5: @ %for.cond.cleanup
+; CHECK-NEXT:    beq .LBB0_5
+; CHECK-NEXT:  .LBB0_4: @ %vector.ph
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB0_2 Depth 2
+; CHECK-NEXT:    dls lr, r4
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    b .LBB0_2
+; CHECK-NEXT:  .LBB0_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.6:
 ; CHECK-NEXT:  .LCPI0_0:
 ; CHECK-NEXT:    .long 4294967280 @ 0xfffffff0
 ; CHECK-NEXT:    .long 4294967284 @ 0xfffffff4
 ; CHECK-NEXT:    .long 4294967288 @ 0xfffffff8
 ; CHECK-NEXT:    .long 4294967292 @ 0xfffffffc

llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll

for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
 define arm_aapcs_vfpcc void @usatmul_4_q15(i16* nocapture readonly %pSrcA, i16* nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: usatmul_4_q15:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB11_8
+; CHECK-NEXT:    cbz r3, .LBB11_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB11_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r5, #0
 ; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    mov r4, r2

 
 ; i8
 
 define arm_aapcs_vfpcc void @ssatmul_4_q7(i8* nocapture readonly %pSrcA, i8* nocapture readonly %pSrcB, i8* noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: ssatmul_4_q7:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB13_8
+; CHECK-NEXT:    cbz r3, .LBB13_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB13_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r5, #0
 ; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    mov r4, r2

llvm/test/CodeGen/Thumb2/mve-scatter-increment.ll

 ; CHECK-NEXT:    vadd.i32 q3, q3, r0
 ; CHECK-NEXT:    add.w r3, r3, r12, lsr #2
 ; CHECK-NEXT:    vstrw.32 q3, [sp] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q3, [lr]
 ; CHECK-NEXT:    adr.w r12, .LCPI3_0
 ; CHECK-NEXT:    vadd.i32 q4, q3, r0
 ; CHECK-NEXT:    vldrw.u32 q3, [r12]
 ; CHECK-NEXT:    vadd.i32 q3, q3, r0
-; CHECK-NEXT:  .LBB3_2: @ %vector.ph
-; CHECK-NEXT:    @ =>This Loop Header: Depth=1
-; CHECK-NEXT:    @ Child Loop BB3_3 Depth 2
-; CHECK-NEXT:    dls lr, r3
-; CHECK-NEXT:    vmov q6, q4
-; CHECK-NEXT:    vldrw.u32 q7, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vmov q5, q3
-; CHECK-NEXT:  .LBB3_3: @ %vector.body
-; CHECK-NEXT:    @ Parent Loop BB3_2 Depth=1
+; CHECK-NEXT:    b .LBB3_4
+; CHECK-NEXT:  .LBB3_2: @ %vector.body
+; CHECK-NEXT:    @ Parent Loop BB3_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vstrw.32 q0, [q5, #48]!
 ; CHECK-NEXT:    vstrw.32 q1, [q6, #48]!
 ; CHECK-NEXT:    vstrw.32 q2, [q7, #48]!
-; CHECK-NEXT:    le lr, .LBB3_3
-; CHECK-NEXT:  @ %bb.4: @ %middle.block
-; CHECK-NEXT:    @ in Loop: Header=BB3_2 Depth=1
+; CHECK-NEXT:    le lr, .LBB3_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    @ in Loop: Header=BB3_4 Depth=1
 ; CHECK-NEXT:    cmp r2, r1
-; CHECK-NEXT:    bne .LBB3_2
+; CHECK-NEXT:    beq .LBB3_5
+; CHECK-NEXT:  .LBB3_4: @ %vector.ph
+; CHECK-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-NEXT:    @ Child Loop BB3_2 Depth 2
+; CHECK-NEXT:    dls lr, r3
+; CHECK-NEXT:    vmov q6, q4
+; CHECK-NEXT:    vldrw.u32 q7, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov q5, q3
+; CHECK-NEXT:    b .LBB3_2
 ; CHECK-NEXT:  .LBB3_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.6:
 ; CHECK-NEXT:  .LCPI3_0:
 ; CHECK-NEXT:    .long 4294967248 @ 0xffffffd0

llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB0_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    b .LBB0_7
 ; CHECK-NEXT:  .LBB0_3:
 ; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    b .LBB0_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB0_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    mov r2, r12

 ; CHECK-NEXT:    sub.w lr, r1, r3
 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB0_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    add r0, r1
 ; CHECK-NEXT:    le lr, .LBB0_8
-; CHECK-NEXT:  .LBB0_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB2_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r2, #-1
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB2_7
 ; CHECK-NEXT:  .LBB2_3:
-; CHECK-NEXT:    mov.w r2, #-1
-; CHECK-NEXT:    b .LBB2_9
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB2_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i8 q0, #0xff
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB3_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB3_7
 ; CHECK-NEXT:  .LBB3_3:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    b .LBB3_9
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB3_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB4_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB4_7
 ; CHECK-NEXT:  .LBB4_3:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    b .LBB4_9
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB4_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB5_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI5_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB5_7
 ; CHECK-NEXT:  .LBB5_3:
 ; CHECK-NEXT:    vldr s0, .LCPI5_0
-; CHECK-NEXT:    b .LBB5_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB5_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB6_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vmov.f32 s0, #1.000000e+00
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB6_7
 ; CHECK-NEXT:  .LBB6_3:
 ; CHECK-NEXT:    vmov.f32 s0, #1.000000e+00
-; CHECK-NEXT:    b .LBB6_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB6_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.f32 q0, #1.000000e+00
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB7_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mvn r2, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB7_7
 ; CHECK-NEXT:  .LBB7_3:
-; CHECK-NEXT:    mvn r2, #-2147483648
-; CHECK-NEXT:    b .LBB7_9
+; CHECK-NEXT:    mvn r0, #-2147483648
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB7_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmvn.i32 q0, #0x80000000
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB8_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mvn r0, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB8_7
 ; CHECK-NEXT:  .LBB8_3:
 ; CHECK-NEXT:    mvn r0, #-2147483648
-; CHECK-NEXT:    b .LBB8_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB8_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    mvn r0, #-2147483648
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    mov r2, r12

 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB8_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lt
 ; CHECK-NEXT:    le lr, .LBB8_8
-; CHECK-NEXT:  .LBB8_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB9_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r2, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB9_7
 ; CHECK-NEXT:  .LBB9_3:
-; CHECK-NEXT:    mov.w r2, #-2147483648
-; CHECK-NEXT:    b .LBB9_9
+; CHECK-NEXT:    mov.w r0, #-2147483648
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB9_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x80000000
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB10_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r0, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB10_7
 ; CHECK-NEXT:  .LBB10_3:
 ; CHECK-NEXT:    mov.w r0, #-2147483648
-; CHECK-NEXT:    b .LBB10_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB10_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    mov.w r0, #-2147483648
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    mov r2, r12

 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB10_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, gt
 ; CHECK-NEXT:    le lr, .LBB10_8
-; CHECK-NEXT:  .LBB10_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB11_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r2, #-1
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB11_7
 ; CHECK-NEXT:  .LBB11_3:
-; CHECK-NEXT:    mov.w r2, #-1
-; CHECK-NEXT:    b .LBB11_9
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB11_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i8 q0, #0xff
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB12_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r0, #-1
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB12_7
 ; CHECK-NEXT:  .LBB12_3:
 ; CHECK-NEXT:    mov.w r0, #-1
-; CHECK-NEXT:    b .LBB12_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB12_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    mov.w r0, #-1
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    mov r2, r12

 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB12_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    le lr, .LBB12_8
-; CHECK-NEXT:  .LBB12_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB13_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB13_7
 ; CHECK-NEXT:  .LBB13_3:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    b .LBB13_9
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB13_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB14_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    b .LBB14_7
 ; CHECK-NEXT:  .LBB14_3:
 ; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    b .LBB14_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB14_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    mov r2, r12

 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB14_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    le lr, .LBB14_8
-; CHECK-NEXT:  .LBB14_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB15_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI15_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB15_7
 ; CHECK-NEXT:  .LBB15_3:
 ; CHECK-NEXT:    vldr s0, .LCPI15_0
-; CHECK-NEXT:    b .LBB15_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB15_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB16_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI16_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB16_7
 ; CHECK-NEXT:  .LBB16_3:
 ; CHECK-NEXT:    vldr s0, .LCPI16_0
-; CHECK-NEXT:    b .LBB16_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB16_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:    dls lr, lr

 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vaddva.u32 r2, q0
 ; CHECK-NEXT:    letp lr, .LBB17_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB17_4:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4

 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    vmlava.u32 r12, q1, q0
 ; CHECK-NEXT:    letp lr, .LBB18_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB18_4:
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp8.not = icmp eq i32 %n, 0
   br i1 %cmp8.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4

 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
 ; CHECK-NEXT:    vaddva.s16 r2, q0
 ; CHECK-NEXT:    letp lr, .LBB19_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB19_4:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 7
   %n.vec = and i32 %n.rnd.up, -8

 ; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
 ; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
 ; CHECK-NEXT:    vmlava.s16 r12, q1, q0
 ; CHECK-NEXT:    letp lr, .LBB20_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB20_4:
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp9.not = icmp eq i32 %n, 0
   br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 7
   %n.vec = and i32 %n.rnd.up, -8

 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
 ; CHECK-NEXT:    vaddva.u8 r2, q0
 ; CHECK-NEXT:    letp lr, .LBB21_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB21_4:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 15
   %n.vec = and i32 %n.rnd.up, -16

 ; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
 ; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
 ; CHECK-NEXT:    vmlava.u8 r12, q1, q0
 ; CHECK-NEXT:    letp lr, .LBB22_2
 ; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB22_4:
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp9.not = icmp eq i32 %n, 0
   br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 15
   %n.vec = and i32 %n.rnd.up, -16
for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret i8 %s.0.lcssa
 }
 
 define i64 @add4i64(i32* noalias nocapture readonly %x, i32 %n) {
 ; CHECK-LABEL: add4i64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    cbz r1, .LBB29_3
+; CHECK-NEXT:    cbz r1, .LBB29_4
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    mov r3, r2
 ; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:  .LBB29_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vaddlva.s32 r2, r3, q0
 ; CHECK-NEXT:    letp lr, .LBB29_2
-; CHECK-NEXT:    b .LBB29_4
-; CHECK-NEXT:  .LBB29_3:
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    mov r3, r2
-; CHECK-NEXT:  .LBB29_4: @ %for.cond.cleanup
+; CHECK-NEXT:  .LBB29_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r2
 ; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:  .LBB29_4:
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    b .LBB29_3
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4
   %trip.count.minus.1 = add i32 %n, -1
for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret i64 %s.0.lcssa
 }
 
 define i64 @mla4i64(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32 %n) {
 ; CHECK-LABEL: mla4i64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    cbz r2, .LBB30_3
+; CHECK-NEXT:    cbz r2, .LBB30_4
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    mov r3, r12
 ; CHECK-NEXT:    dlstp.32 lr, r2
 ; CHECK-NEXT:  .LBB30_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    vmlalva.s32 r12, r3, q1, q0
 ; CHECK-NEXT:    letp lr, .LBB30_2
-; CHECK-NEXT:    b .LBB30_4
-; CHECK-NEXT:  .LBB30_3:
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    mov r3, r12
-; CHECK-NEXT:  .LBB30_4: @ %for.cond.cleanup
+; CHECK-NEXT:  .LBB30_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:  .LBB30_4:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    mov r3, r12
+; CHECK-NEXT:    b .LBB30_3
 entry:
   %cmp9.not = icmp eq i32 %n, 0
   br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 3
   %n.vec = and i32 %n.rnd.up, -4
   %trip.count.minus.1 = add i32 %n, -1
for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret i64 %s.0.lcssa
 }
 
 define i64 @mla8i64(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i32 %n) {
 ; CHECK-LABEL: mla8i64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    cbz r2, .LBB31_3
+; CHECK-NEXT:    cbz r2, .LBB31_4
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    mov r3, r12
 ; CHECK-NEXT:    dlstp.16 lr, r2
 ; CHECK-NEXT:  .LBB31_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
 ; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
 ; CHECK-NEXT:    vmlalva.s16 r12, r3, q1, q0
 ; CHECK-NEXT:    letp lr, .LBB31_2
-; CHECK-NEXT:    b .LBB31_4
-; CHECK-NEXT:  .LBB31_3:
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    mov r3, r12
-; CHECK-NEXT:  .LBB31_4: @ %for.cond.cleanup
+; CHECK-NEXT:  .LBB31_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    mov r1, r3
 ; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:  .LBB31_4:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    mov r3, r12
+; CHECK-NEXT:    b .LBB31_3
 entry:
   %cmp9.not = icmp eq i32 %n, 0
   br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
 
 vector.ph:                                        ; preds = %entry
   %n.rnd.up = add i32 %n, 7
   %n.vec = and i32 %n.rnd.up, -8
   %trip.count.minus.1 = add i32 %n, -1

llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs %s -o - | FileCheck %s
 
 define void @arm_cmplx_mag_squared_f16(half* nocapture readonly %pSrc, half* nocapture %pDst, i32 %numSamples) {
 ; CHECK-LABEL: arm_cmplx_mag_squared_f16:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    beq .LBB0_8
+; CHECK-NEXT:    cbz r2, .LBB0_8
 ; CHECK-NEXT:  @ %bb.1: @ %while.body.preheader
 ; CHECK-NEXT:    cmp r2, #8
 ; CHECK-NEXT:    blo .LBB0_9
 ; CHECK-NEXT:  @ %bb.2: @ %vector.memcheck
 ; CHECK-NEXT:    add.w r3, r0, r2, lsl #2
 ; CHECK-NEXT:    cmp r3, r1
 ; CHECK-NEXT:    itt hi
 ; CHECK-NEXT:    addhi.w r3, r1, r2, lsl #1