[MBP] Enable duplicating return block to remove jump to return

[Carrot]

Summary

Sometimes LLVM generates branch to return instruction, like PR63227.

It is because in function MachineBlockPlacement::canTailDuplicateUnplacedPreds we avoid duplicating a BB into another already placed BB to prevent destroying computed layout. But if the successor BB is a return block, duplicating it will only reduce taken branches without hurt to any other branches.

[davidxl]

is the bug number referenced valid?

[Carrot]

In D153093#4435643, @davidxl wrote:

is the bug number referenced valid?

The complete link is https://github.com/llvm/llvm-project/issues/63227.

[davidxl]

lgtm

llvm/lib/CodeGen/MachineBlockPlacement.cpp

bool MachineBlockPlacement::canTailDuplicateUnplacedPreds(
   // For CFG checking.
   SmallPtrSet<const MachineBasicBlock *, 4> Successors(BB->succ_begin(),
                                                        BB->succ_end());
   for (MachineBasicBlock *Pred : Succ->predecessors()) {
     // Make sure all unplaced and unfiltered predecessors can be
     // tail-duplicated into.
     // Skip any blocks that are already placed or not in this loop.
     if (Pred == BB || (BlockFilter && !BlockFilter->count(Pred))
-        || BlockToChain[Pred] == &Chain)
+        || (BlockToChain[Pred] == &Chain && !Succ->succ_empty()))
       continue;
     if (!TailDup.canTailDuplicate(Succ, Pred)) {
       if (Successors.size() > 1 && hasSameSuccessors(*Pred, Successors))
         // This will result in a trellis after tail duplication, so we don't
         // need to copy Succ into this predecessor. In the presence
         // of a trellis tail duplication can continue to be profitable.
         // For example:
         // A            A

llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll

 ; CHECK-NEXT:    cmp w2, #15
 ; CHECK-NEXT:    b.hi .LBB5_4
 ; CHECK-NEXT:  // %bb.2:
 ; CHECK-NEXT:    mov x11, xzr
 ; CHECK-NEXT:    mov w8, wzr
 ; CHECK-NEXT:    b .LBB5_7
 ; CHECK-NEXT:  .LBB5_3:
 ; CHECK-NEXT:    mov w8, wzr
-; CHECK-NEXT:    b .LBB5_9
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB5_4: // %vector.ph
 ; CHECK-NEXT:    and x11, x10, #0xfffffff0
 ; CHECK-NEXT:    add x8, x0, #8
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    mov x12, x11
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    dup v2.8h, w9
 ; CHECK-NEXT:  .LBB5_5: // %vector.body

llvm/test/CodeGen/Thumb/branch-to-return.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv7 %s -o - | FileCheck %s
+
+; Test the branch to return in BB4 is converted to return.
+
+define i32 @foo(i32* %x, i32 %n) {
+; CHECK-LABEL: foo:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r1, #1
+; CHECK-NEXT:    blt .LBB0_4
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    bic r3, r1, #3
+; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    cmp r1, #4
+; CHECK-NEXT:    bhs .LBB0_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    b .LBB0_6
+; CHECK-NEXT:  .LBB0_3: @ %middle.block
+; CHECK-NEXT:    cmp r1, r3
+; CHECK-NEXT:    bne .LBB0_5
+; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:  .LBB0_5:
+; CHECK-NEXT:    ldr.w r0, [r12]
+; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader1
+; CHECK-NEXT:    subs r3, r1, r3
+; CHECK-NEXT:    mvn r2, #12
+; CHECK-NEXT:    and.w r1, r2, r1, lsl #2
+; CHECK-NEXT:    add r1, r12
+; CHECK-NEXT:  .LBB0_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r2, [r1], #4
+; CHECK-NEXT:    subs r3, #1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    bne .LBB0_7
+; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup
+; CHECK-NEXT:    bx lr
+entry:
+  %n.vec = and i32 %n, -4
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %n, 4
+  br i1 %min.iters.check, label %for.body.preheader1, label %middle.block
+
+middle.block:
+  %x3 = load i32, i32* %x, align 4
+  %cmp.n = icmp eq i32 %n.vec, %n
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
+
+for.body.preheader1:                              ; preds = %middle.block, %for.body.preheader
+  %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %x3, %middle.block ]
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader1, %for.body
+  %i.08 = phi i32 [ %inc, %for.body ], [ %n.vec, %for.body.preheader1 ]
+  %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
+  %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
+  %v5 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %v5, %r.07
+  %inc = add nuw nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  %r.0.lcssa = phi i32 [ 0, %entry ], [ 0, %middle.block ], [ %add, %for.body ]
+  ret i32 %r.0.lcssa
+}

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll

 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB9_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI9_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB9_6
 ; CHECK-NEXT:  .LBB9_3:
 ; CHECK-NEXT:    vldr s0, .LCPI9_0
-; CHECK-NEXT:    b .LBB9_9
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:  .LBB9_4: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r2, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vldr s0, .LCPI9_0
 ; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0

 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB10_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI10_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB10_6
 ; CHECK-NEXT:  .LBB10_3:
 ; CHECK-NEXT:    vldr s0, .LCPI10_0
-; CHECK-NEXT:    b .LBB10_9
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:  .LBB10_4: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r2, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vldr s0, .LCPI10_0
 ; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0

 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB11_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB11_6
 ; CHECK-NEXT:  .LBB11_3:
 ; CHECK-NEXT:    vldr s0, .LCPI11_0
-; CHECK-NEXT:    b .LBB11_9
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-NEXT:  .LBB11_4: @ %for.body.preheader.new
 ; CHECK-NEXT:    bic r2, r2, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    adds r4, r0, #4
 ; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    adds r3, r1, #4

llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB0_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    b .LBB0_7
 ; CHECK-NEXT:  .LBB0_3:
 ; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    b .LBB0_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB0_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    mov r2, r12
 ; CHECK-NEXT:  .LBB0_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r2], #16
 ; CHECK-NEXT:    vaddva.u32 r0, q0
 ; CHECK-NEXT:    le lr, .LBB0_5
 ; CHECK-NEXT:  @ %bb.6: @ %middle.block
 ; CHECK-NEXT:    cmp r3, r1
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
 ; CHECK-NEXT:  .LBB0_7: @ %for.body.preheader1
 ; CHECK-NEXT:    sub.w lr, r1, r3
 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:  .LBB0_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    add r0, r1
 ; CHECK-NEXT:    le lr, .LBB0_8
-; CHECK-NEXT:  .LBB0_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB2_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r2, #-1
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB2_7
 ; CHECK-NEXT:  .LBB2_3:
 ; CHECK-NEXT:    mov.w r2, #-1
-; CHECK-NEXT:    b .LBB2_9
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:  .LBB2_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i8 q0, #0xff
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:  .LBB2_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB3_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB3_7
 ; CHECK-NEXT:  .LBB3_3:
 ; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    b .LBB3_9
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:  .LBB3_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:  .LBB3_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB4_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB4_7
 ; CHECK-NEXT:  .LBB4_3:
 ; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    b .LBB4_9
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:  .LBB4_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:  .LBB4_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB5_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI5_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB5_7
 ; CHECK-NEXT:  .LBB5_3:
 ; CHECK-NEXT:    vldr s0, .LCPI5_0
-; CHECK-NEXT:    b .LBB5_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB5_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:  .LBB5_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB6_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vmov.f32 s0, #1.000000e+00
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB6_7
 ; CHECK-NEXT:  .LBB6_3:
 ; CHECK-NEXT:    vmov.f32 s0, #1.000000e+00
-; CHECK-NEXT:    b .LBB6_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB6_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.f32 q0, #1.000000e+00
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:  .LBB6_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB7_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mvn r2, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB7_7
 ; CHECK-NEXT:  .LBB7_3:
 ; CHECK-NEXT:    mvn r2, #-2147483648
-; CHECK-NEXT:    b .LBB7_9
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB7_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmvn.i32 q0, #0x80000000
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:  .LBB7_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB8_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mvn r0, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB8_7
 ; CHECK-NEXT:  .LBB8_3:
 ; CHECK-NEXT:    mvn r0, #-2147483648
-; CHECK-NEXT:    b .LBB8_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB8_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    mvn r0, #-2147483648
 ; CHECK-NEXT:    mov r2, r12
 ; CHECK-NEXT:  .LBB8_5: @ %vector.body

 ; CHECK-NEXT:    sub.w lr, r1, r3
 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:  .LBB8_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, lt
 ; CHECK-NEXT:    le lr, .LBB8_8
-; CHECK-NEXT:  .LBB8_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB9_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r2, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB9_7
 ; CHECK-NEXT:  .LBB9_3:
 ; CHECK-NEXT:    mov.w r2, #-2147483648
-; CHECK-NEXT:    b .LBB9_9
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB9_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x80000000
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:  .LBB9_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB10_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r0, #-2147483648
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB10_7
 ; CHECK-NEXT:  .LBB10_3:
 ; CHECK-NEXT:    mov.w r0, #-2147483648
-; CHECK-NEXT:    b .LBB10_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB10_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    mov.w r0, #-2147483648
 ; CHECK-NEXT:    mov r2, r12
 ; CHECK-NEXT:  .LBB10_5: @ %vector.body

 ; CHECK-NEXT:    sub.w lr, r1, r3
 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:  .LBB10_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, gt
 ; CHECK-NEXT:    le lr, .LBB10_8
-; CHECK-NEXT:  .LBB10_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB11_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r2, #-1
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB11_7
 ; CHECK-NEXT:  .LBB11_3:
 ; CHECK-NEXT:    mov.w r2, #-1
-; CHECK-NEXT:    b .LBB11_9
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB11_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i8 q0, #0xff
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:  .LBB11_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB12_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov.w r0, #-1
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    b .LBB12_7
 ; CHECK-NEXT:  .LBB12_3:
 ; CHECK-NEXT:    mov.w r0, #-1
-; CHECK-NEXT:    b .LBB12_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB12_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    mov.w r0, #-1
 ; CHECK-NEXT:    mov r2, r12
 ; CHECK-NEXT:  .LBB12_5: @ %vector.body

 ; CHECK-NEXT:    sub.w lr, r1, r3
 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:  .LBB12_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    le lr, .LBB12_8
-; CHECK-NEXT:  .LBB12_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB13_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB13_7
 ; CHECK-NEXT:  .LBB13_3:
 ; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    b .LBB13_9
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB13_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    sub.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r12, lsr #2
 ; CHECK-NEXT:    mov r2, r0
 ; CHECK-NEXT:  .LBB13_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB14_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    b .LBB14_7
 ; CHECK-NEXT:  .LBB14_3:
 ; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    b .LBB14_9
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB14_4: @ %vector.ph
 ; CHECK-NEXT:    bic r3, r1, #3
 ; CHECK-NEXT:    movs r2, #1
 ; CHECK-NEXT:    subs r0, r3, #4
 ; CHECK-NEXT:    add.w lr, r2, r0, lsr #2
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    mov r2, r12
 ; CHECK-NEXT:  .LBB14_5: @ %vector.body

 ; CHECK-NEXT:    sub.w lr, r1, r3
 ; CHECK-NEXT:    add.w r2, r12, r3, lsl #2
 ; CHECK-NEXT:  .LBB14_8: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r2], #4
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    le lr, .LBB14_8
-; CHECK-NEXT:  .LBB14_9: @ %for.cond.cleanup
+; CHECK-NEXT:  @ %bb.9: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %cmp6 = icmp sgt i32 %n, 0
   br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:                               ; preds = %entry
   %min.iters.check = icmp ult i32 %n, 4
   br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB15_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI15_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB15_7
 ; CHECK-NEXT:  .LBB15_3:
 ; CHECK-NEXT:    vldr s0, .LCPI15_0
-; CHECK-NEXT:    b .LBB15_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB15_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:  .LBB15_5: @ %vector.body

 ; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    bhs .LBB16_4
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    vldr s0, .LCPI16_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB16_7
 ; CHECK-NEXT:  .LBB16_3:
 ; CHECK-NEXT:    vldr s0, .LCPI16_0
-; CHECK-NEXT:    b .LBB16_9
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:  .LBB16_4: @ %vector.ph
 ; CHECK-NEXT:    bic r2, r1, #3
 ; CHECK-NEXT:    movs r3, #1
 ; CHECK-NEXT:    sub.w r12, r2, #4
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
 ; CHECK-NEXT:    mov r3, r0
 ; CHECK-NEXT:  .LBB16_5: @ %vector.body