[SCEV] Canonicalize X - urem X, Y patterns

[reames]

Summary

There are multiple possible ways to represent the X - urem X, Y pattern. SCEV was not canonicalizing, and thus, depending on which you analyzed, you could get different results. The sub representation appears to produce strictly inferior results in practice, so I decided to canonicalize to the Y * X/Y version.

The motivation here is that runtime unroll produces the sub X - (and X, Y-1) pattern when Y is a power of two. SCEV is thus unable to recognize that an unrolled loop exits because we don't figure out that the new unrolled step evenly divides the trip count of the unrolled loop. After instcombine runs, we convert the the andn form which SCEV recognizes, so essentially, this is just fixing a nasty pass ordering dependency.

Why this appears to minorly negatively impact hardware loop recognition on ARM, I have no idea. I definitely don't consider that a blocker. I can't even tell from the test if this is actually a regression - the test is too poorly structured to be informative.

[lebedev.ri]

llvm/lib/Analysis/ScalarEvolution.cpp

2604

What about the more general case of (-1 * urem X, Y) + X + Z --> ((-1 * urem X, Y) + X) + Z --> (Y * X/Y) + Z ?

[reames]

llvm/lib/Analysis/ScalarEvolution.cpp

2604

The pattern matching for this gets hairy. I think it's worthwhile to do (so that we can handle backedge taken count expressions in runtime unroll), but I'd strongly prefer to generalize in a separate commit with it's own review.

[lebedev.ri]

This LG. Not sure if second opinion is needed.

llvm/lib/Analysis/ScalarEvolution.cpp

2604

Yep, just highlighting that this is only the simplest case possible.

[nikic]

@dmgreen might want to look at the ARM test changes. The code change itself looks good to me though.

[efriedma]

Why this appears to minorly negatively impact hardware loop recognition on ARM, I have no idea.

We're recognizing more loops, not less. The "le" instruction indicates we've recognized a loop; we generate a generic branch "bne" when we don't. It's more instructions outside the loop because hardware loop optimization requires materializing the precise trip count. The sequence we're using to compute the trip count is messy, though. I guess the issue is that we're running SCEVExpander later than we otherwise would?

Adding a couple reviewers more familiar with MVE in case they have any further comment on this, but the ARM changes should be fine as-is.

[dmgreen]

Yeah, these look good now. Thanks!

llvm/lib/Analysis/ScalarEvolution.cpp

if (AddExpr && C && isa<SCEVConstant>(AddExpr->getOperand(0))) {
       if (PreservedFlags != SCEV::FlagAnyWrap) {
         SmallVector<const SCEV *, 4> NewOps(AddExpr->operands());
         NewOps[0] = getConstant(ConstAdd);
         return getAddExpr(NewOps, PreservedFlags);
       }
     }
   }
 
+  // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
+  if (Ops.size() == 2) {
+    const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[0]);
+    if (Mul && Mul->getNumOperands() == 2 &&
+        Mul->getOperand(0)->isAllOnesValue()) {
+      const SCEV *X;
+      const SCEV *Y;
+      if (matchURem(Mul->getOperand(1), X, Y) && X == Ops[1]) {
+        return getMulExpr(Y, getUDivExpr(X, Y));
+      }
+    }
+  }
+
   // Skip past any other cast SCEVs.
   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
     ++Idx;
 
   // If there are add operands they would be next.
   if (Idx < Ops.size()) {
     bool DeletedAdd = false;
     // If the original flags and all inlined SCEVAddExprs are NUW, use the

llvm/test/Analysis/ScalarEvolution/urem-0.ll

 }
 
 define i32 @test_sub_urem(i32 %arg) {
 ; CHECK-LABEL: 'test_sub_urem'
 ; CHECK-NEXT:  Classifying expressions for: @test_sub_urem
 ; CHECK-NEXT:    %urem = urem i32 %arg, 8
 ; CHECK-NEXT:    --> (zext i3 (trunc i32 %arg to i3) to i32) U: [0,8) S: [0,8)
 ; CHECK-NEXT:    %sub = sub i32 %arg, %urem
-; CHECK-NEXT:    --> ((-1 * (zext i3 (trunc i32 %arg to i3) to i32))<nsw> + %arg) U: full-set S: full-set
+; CHECK-NEXT:    --> (8 * (%arg /u 8))<nuw> U: [0,-7) S: [-2147483648,2147483641)
 ; CHECK-NEXT:  Determining loop execution counts for: @test_sub_urem
 ;
   %urem = urem i32 %arg, 8
   %sub = sub i32 %arg, %urem
   ret i32 %sub
 }
 
 define i32 @test_trunc_zext(i32 %arg) {
 ; CHECK-LABEL: 'test_trunc_zext'
 ; CHECK-NEXT:  Classifying expressions for: @test_trunc_zext
 ; CHECK-NEXT:    %trunc = trunc i32 %arg to i3
 ; CHECK-NEXT:    --> (trunc i32 %arg to i3) U: full-set S: full-set
 ; CHECK-NEXT:    %zext = zext i3 %trunc to i32
 ; CHECK-NEXT:    --> (zext i3 (trunc i32 %arg to i3) to i32) U: [0,8) S: [0,8)
 ; CHECK-NEXT:    %sub = sub i32 %arg, %zext
-; CHECK-NEXT:    --> ((-1 * (zext i3 (trunc i32 %arg to i3) to i32))<nsw> + %arg) U: full-set S: full-set
+; CHECK-NEXT:    --> (8 * (%arg /u 8))<nuw> U: [0,-7) S: [-2147483648,2147483641)
 ; CHECK-NEXT:  Determining loop execution counts for: @test_trunc_zext
 ;
   %trunc = trunc i32 %arg to i3
   %zext = zext i3 %trunc to i32
   %sub = sub i32 %arg, %zext
   ret i32 %sub
 }

llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fullfp16 -tail-predication=enabled %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
 ; CHECK-LABEL: fast_float_mul:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB0_11
+; CHECK-NEXT:    beq.w .LBB0_11
 ; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
 ; CHECK-NEXT:    add.w r5, r0, r3, lsl #2
 ; CHECK-NEXT:    add.w r4, r2, r3, lsl #2
 ; CHECK-NEXT:    cmp r5, r2
 ; CHECK-NEXT:    cset r12, hi
 ; CHECK-NEXT:    cmp r4, r0
 ; CHECK-NEXT:    cset lr, hi
 ; CHECK-NEXT:    cmp r5, r1

 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r2], #16
 ; CHECK-NEXT:    vmul.f32 q0, q1, q0
 ; CHECK-NEXT:    vstrw.32 q0, [r0], #16
 ; CHECK-NEXT:    letp lr, .LBB0_5
 ; CHECK-NEXT:    b .LBB0_11
 ; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader.new
-; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    subs r3, #4
 ; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    add.w lr, r5, r3, lsr #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:  .LBB0_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r5, r1, r4
 ; CHECK-NEXT:    adds r6, r2, r4
 ; CHECK-NEXT:    adds r7, r0, r4
-; CHECK-NEXT:    adds r3, #4
-; CHECK-NEXT:    vldr s0, [r5]
 ; CHECK-NEXT:    adds r4, #16
+; CHECK-NEXT:    vldr s0, [r5]
+; CHECK-NEXT:    adds r3, #4
 ; CHECK-NEXT:    vldr s2, [r6]
-; CHECK-NEXT:    cmp lr, r3
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r7]
 ; CHECK-NEXT:    vldr s0, [r5, #4]
 ; CHECK-NEXT:    vldr s2, [r6, #4]
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r7, #4]
 ; CHECK-NEXT:    vldr s0, [r5, #8]
 ; CHECK-NEXT:    vldr s2, [r6, #8]
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r7, #8]
 ; CHECK-NEXT:    vldr s0, [r5, #12]
 ; CHECK-NEXT:    vldr s2, [r6, #12]
 ; CHECK-NEXT:    vmul.f32 s0, s2, s0
 ; CHECK-NEXT:    vstr s0, [r7, #12]
-; CHECK-NEXT:    bne .LBB0_7
+; CHECK-NEXT:    le lr, .LBB0_7
 ; CHECK-NEXT:  .LBB0_8: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, r12, .LBB0_11
 ; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add.w r1, r1, r3, lsl #2
 ; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
 ; CHECK-NEXT:    add.w r0, r0, r3, lsl #2
 ; CHECK-NEXT:  .LBB0_10: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll

 define arm_aapcs_vfpcc float @half_half_mac(half* nocapture readonly %a, half* nocapture readonly %b, i32 %N) {
 ; CHECK-LABEL: half_half_mac:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    cbz r2, .LBB9_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    and r12, r2, #3
-; CHECK-NEXT:    vldr s0, .LCPI9_0
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB9_4
 ; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI9_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB9_6
 ; CHECK-NEXT:  .LBB9_3:
 ; CHECK-NEXT:    vldr s0, .LCPI9_0
 ; CHECK-NEXT:    b .LBB9_9
 ; CHECK-NEXT:  .LBB9_4: @ %for.body.preheader.new
-; CHECK-NEXT:    sub.w lr, r2, r12
+; CHECK-NEXT:    bic r2, r2, #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI9_0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:  .LBB9_5: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r5, r0, r3
 ; CHECK-NEXT:    adds r4, r1, r3
 ; CHECK-NEXT:    vldr.16 s2, [r4, #6]
 ; CHECK-NEXT:    vldr.16 s4, [r5, #6]
 ; CHECK-NEXT:    vldr.16 s6, [r5, #4]
 ; CHECK-NEXT:    vldr.16 s8, [r5, #2]
 ; CHECK-NEXT:    vmul.f16 s2, s4, s2
 ; CHECK-NEXT:    vldr.16 s4, [r4, #4]
 ; CHECK-NEXT:    vldr.16 s10, [r5]
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vmul.f16 s4, s6, s4
 ; CHECK-NEXT:    vldr.16 s6, [r4, #2]
 ; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    adds r3, #8
 ; CHECK-NEXT:    vmul.f16 s6, s8, s6
 ; CHECK-NEXT:    vldr.16 s8, [r4]
 ; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
-; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    adds r2, #4
 ; CHECK-NEXT:    vmul.f16 s8, s10, s8
-; CHECK-NEXT:    cmp lr, r2
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NEXT:    bne .LBB9_5
+; CHECK-NEXT:    le lr, .LBB9_5
 ; CHECK-NEXT:  .LBB9_6: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, r12, .LBB9_9
 ; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add.w r0, r0, r2, lsl #1
 ; CHECK-NEXT:    add.w r1, r1, r2, lsl #1
 ; CHECK-NEXT:  .LBB9_8: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s2, [r1]

 define arm_aapcs_vfpcc float @half_half_acc(half* nocapture readonly %a, half* nocapture readonly %b, i32 %N) {
 ; CHECK-LABEL: half_half_acc:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    cbz r2, .LBB10_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    and r12, r2, #3
-; CHECK-NEXT:    vldr s0, .LCPI10_0
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB10_4
 ; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI10_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB10_6
 ; CHECK-NEXT:  .LBB10_3:
 ; CHECK-NEXT:    vldr s0, .LCPI10_0
 ; CHECK-NEXT:    b .LBB10_9
 ; CHECK-NEXT:  .LBB10_4: @ %for.body.preheader.new
-; CHECK-NEXT:    sub.w lr, r2, r12
+; CHECK-NEXT:    bic r2, r2, #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI10_0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:  .LBB10_5: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r5, r0, r3
 ; CHECK-NEXT:    adds r4, r1, r3
 ; CHECK-NEXT:    vldr.16 s2, [r4, #6]
 ; CHECK-NEXT:    vldr.16 s4, [r5, #6]
 ; CHECK-NEXT:    vldr.16 s6, [r5, #4]
 ; CHECK-NEXT:    vldr.16 s8, [r5, #2]
 ; CHECK-NEXT:    vadd.f16 s2, s4, s2
 ; CHECK-NEXT:    vldr.16 s4, [r4, #4]
 ; CHECK-NEXT:    vldr.16 s10, [r5]
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vadd.f16 s4, s6, s4
 ; CHECK-NEXT:    vldr.16 s6, [r4, #2]
 ; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
-; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    adds r3, #8
 ; CHECK-NEXT:    vadd.f16 s6, s8, s6
 ; CHECK-NEXT:    vldr.16 s8, [r4]
 ; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
-; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    adds r2, #4
 ; CHECK-NEXT:    vadd.f16 s8, s10, s8
-; CHECK-NEXT:    cmp lr, r2
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NEXT:    bne .LBB10_5
+; CHECK-NEXT:    le lr, .LBB10_5
 ; CHECK-NEXT:  .LBB10_6: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, r12, .LBB10_9
 ; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add.w r0, r0, r2, lsl #1
 ; CHECK-NEXT:    add.w r1, r1, r2, lsl #1
 ; CHECK-NEXT:  .LBB10_8: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldr.16 s2, [r1]

 define arm_aapcs_vfpcc float @half_short_mac(half* nocapture readonly %a, i16* nocapture readonly %b, i32 %N) {
 ; CHECK-LABEL: half_short_mac:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    cbz r2, .LBB11_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    subs r3, r2, #1
 ; CHECK-NEXT:    and r12, r2, #3
-; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhs .LBB11_4
 ; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    b .LBB11_6
 ; CHECK-NEXT:  .LBB11_3:
 ; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    b .LBB11_9
 ; CHECK-NEXT:  .LBB11_4: @ %for.body.preheader.new
-; CHECK-NEXT:    sub.w lr, r2, r12
-; CHECK-NEXT:    adds r3, r1, #4
+; CHECK-NEXT:    bic r2, r2, #3
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI11_0
 ; CHECK-NEXT:    adds r4, r0, #4
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    adds r3, r1, #4
 ; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:  .LBB11_5: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsh.w r5, [r3, #2]
 ; CHECK-NEXT:    vldr.16 s2, [r4, #2]
 ; CHECK-NEXT:    adds r2, #4
-; CHECK-NEXT:    cmp lr, r2
 ; CHECK-NEXT:    vmov s4, r5
 ; CHECK-NEXT:    ldrsh r5, [r3], #8
 ; CHECK-NEXT:    vcvt.f16.s32 s4, s4
 ; CHECK-NEXT:    ldrsh r6, [r3, #-10]
 ; CHECK-NEXT:    vmul.f16 s2, s2, s4
 ; CHECK-NEXT:    vmov s6, r5
 ; CHECK-NEXT:    vldr.16 s4, [r4]
 ; CHECK-NEXT:    vcvt.f16.s32 s6, s6
 ; CHECK-NEXT:    ldrsh r5, [r3, #-12]
 ; CHECK-NEXT:    vmul.f16 s4, s4, s6
 ; CHECK-NEXT:    vmov s8, r6
 ; CHECK-NEXT:    vldr.16 s6, [r4, #-2]
 ; CHECK-NEXT:    vcvt.f16.s32 s8, s8
 ; CHECK-NEXT:    vmov s10, r5
 ; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
 ; CHECK-NEXT:    vmul.f16 s6, s6, s8
 ; CHECK-NEXT:    vldr.16 s8, [r4, #-4]
 ; CHECK-NEXT:    vcvt.f16.s32 s10, s10
 ; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
 ; CHECK-NEXT:    vmul.f16 s8, s8, s10
 ; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
 ; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
-; CHECK-NEXT:    add.w r4, r4, #8
+; CHECK-NEXT:    adds r4, #8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NEXT:    bne .LBB11_5
+; CHECK-NEXT:    le lr, .LBB11_5
 ; CHECK-NEXT:  .LBB11_6: @ %for.cond.cleanup.loopexit.unr-lcssa
 ; CHECK-NEXT:    wls lr, r12, .LBB11_9
 ; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
 ; CHECK-NEXT:    add.w r0, r0, r2, lsl #1
 ; CHECK-NEXT:    add.w r1, r1, r2, lsl #1
 ; CHECK-NEXT:  .LBB11_8: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrsh r2, [r1], #2

llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll

for.cond.cleanup:                                 ; preds = %middle.block, %entry
   %res.0.lcssa = phi i32 [ 0, %entry ], [ %7, %middle.block ]
   ret i32 %res.0.lcssa
 }
 
 define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_char(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
 ; CHECK-LABEL: test_vec_mul_scalar_add_char:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    ldr.w r12, [sp, #28]
-; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    ldr r4, [sp, #28]
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    beq.w .LBB5_11
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
-; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
-; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r5, r3, r4, lsl #2
+; CHECK-NEXT:    adds r6, r1, r4
 ; CHECK-NEXT:    cmp r5, r1
-; CHECK-NEXT:    add.w r4, r0, r12
-; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    add.w r7, r0, r4
+; CHECK-NEXT:    cset r12, hi
 ; CHECK-NEXT:    cmp r6, r3
 ; CHECK-NEXT:    cset r6, hi
 ; CHECK-NEXT:    cmp r5, r0
 ; CHECK-NEXT:    cset r5, hi
-; CHECK-NEXT:    cmp r4, r3
-; CHECK-NEXT:    cset r4, hi
-; CHECK-NEXT:    tst r4, r5
+; CHECK-NEXT:    cmp r7, r3
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    tst r7, r5
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    andseq.w r7, r7, r6
+; CHECK-NEXT:    andseq.w r7, r6, r12
 ; CHECK-NEXT:    beq .LBB5_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    sub.w r4, r12, #1
-; CHECK-NEXT:    and lr, r12, #3
-; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    subs r7, r4, #1
+; CHECK-NEXT:    and r12, r4, #3
+; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhs .LBB5_6
 ; CHECK-NEXT:  @ %bb.3:
-; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    mov.w r8, #0
 ; CHECK-NEXT:    b .LBB5_8
 ; CHECK-NEXT:  .LBB5_4: @ %vector.ph
-; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:    dlstp.32 lr, r4
 ; CHECK-NEXT:  .LBB5_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
 ; CHECK-NEXT:    vldrb.u32 q1, [r1], #4
 ; CHECK-NEXT:    vmlas.u32 q1, q0, r2
 ; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB5_5
 ; CHECK-NEXT:    b .LBB5_11
 ; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader.new
-; CHECK-NEXT:    sub.w r8, r12, lr
+; CHECK-NEXT:    bic r7, r4, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    subs r7, #4
 ; CHECK-NEXT:    add.w r5, r3, #8
+; CHECK-NEXT:    mov.w r8, #0
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
 ; CHECK-NEXT:    adds r6, r0, #3
 ; CHECK-NEXT:    adds r7, r1, #1
-; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:  .LBB5_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r9, [r6, #-3]
-; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    add.w r8, r8, #4
 ; CHECK-NEXT:    ldrb r4, [r7, #-1]
-; CHECK-NEXT:    cmp r8, r12
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #-8]
 ; CHECK-NEXT:    ldrb r9, [r6, #-2]
 ; CHECK-NEXT:    ldrb r4, [r7], #4
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #-4]
 ; CHECK-NEXT:    ldrb r9, [r6, #-1]
 ; CHECK-NEXT:    ldrb r4, [r7, #-3]
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5]
 ; CHECK-NEXT:    ldrb r9, [r6], #4
 ; CHECK-NEXT:    ldrb r4, [r7, #-2]
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #4]
-; CHECK-NEXT:    add.w r5, r5, #16
-; CHECK-NEXT:    bne .LBB5_7
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    le lr, .LBB5_7
 ; CHECK-NEXT:  .LBB5_8: @ %for.cond.cleanup.loopexit.unr-lcssa
-; CHECK-NEXT:    wls lr, lr, .LBB5_11
+; CHECK-NEXT:    wls lr, r12, .LBB5_11
 ; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
-; CHECK-NEXT:    add r0, r12
-; CHECK-NEXT:    add r1, r12
-; CHECK-NEXT:    add.w r3, r3, r12, lsl #2
+; CHECK-NEXT:    add r0, r8
+; CHECK-NEXT:    add r1, r8
+; CHECK-NEXT:    add.w r3, r3, r8, lsl #2
 ; CHECK-NEXT:  .LBB5_10: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r7, [r0], #1
 ; CHECK-NEXT:    ldrb r6, [r1], #1
 ; CHECK-NEXT:    smlabb r7, r6, r7, r2
 ; CHECK-NEXT:    str r7, [r3], #4
 ; CHECK-NEXT:    le lr, .LBB5_10
 ; CHECK-NEXT:  .LBB5_11: @ %for.cond.cleanup

 for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
 define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_uchar(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
 ; CHECK-LABEL: test_vec_mul_scalar_add_uchar:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    ldr.w r12, [sp, #28]
-; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    ldr r4, [sp, #28]
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    beq.w .LBB7_11
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
-; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
-; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r5, r3, r4, lsl #2
+; CHECK-NEXT:    adds r6, r1, r4
 ; CHECK-NEXT:    cmp r5, r1
-; CHECK-NEXT:    add.w r4, r0, r12
-; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    add.w r7, r0, r4
+; CHECK-NEXT:    cset r12, hi
 ; CHECK-NEXT:    cmp r6, r3
 ; CHECK-NEXT:    cset r6, hi
 ; CHECK-NEXT:    cmp r5, r0
 ; CHECK-NEXT:    cset r5, hi
-; CHECK-NEXT:    cmp r4, r3
-; CHECK-NEXT:    cset r4, hi
-; CHECK-NEXT:    tst r4, r5
+; CHECK-NEXT:    cmp r7, r3
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    tst r7, r5
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    andseq.w r7, r7, r6
+; CHECK-NEXT:    andseq.w r7, r6, r12
 ; CHECK-NEXT:    beq .LBB7_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    sub.w r4, r12, #1
-; CHECK-NEXT:    and lr, r12, #3
-; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    subs r7, r4, #1
+; CHECK-NEXT:    and r12, r4, #3
+; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhs .LBB7_6
 ; CHECK-NEXT:  @ %bb.3:
-; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    mov.w r8, #0
 ; CHECK-NEXT:    b .LBB7_8
 ; CHECK-NEXT:  .LBB7_4: @ %vector.ph
-; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:    dlstp.32 lr, r4
 ; CHECK-NEXT:  .LBB7_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u32 q0, [r0], #4
 ; CHECK-NEXT:    vldrb.u32 q1, [r1], #4
 ; CHECK-NEXT:    vmlas.u32 q1, q0, r2
 ; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB7_5
 ; CHECK-NEXT:    b .LBB7_11
 ; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader.new
-; CHECK-NEXT:    sub.w r8, r12, lr
+; CHECK-NEXT:    bic r7, r4, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    subs r7, #4
 ; CHECK-NEXT:    add.w r5, r3, #8
+; CHECK-NEXT:    mov.w r8, #0
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
 ; CHECK-NEXT:    adds r6, r0, #3
 ; CHECK-NEXT:    adds r7, r1, #1
-; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:  .LBB7_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r9, [r6, #-3]
-; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    add.w r8, r8, #4
 ; CHECK-NEXT:    ldrb r4, [r7, #-1]
-; CHECK-NEXT:    cmp r8, r12
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #-8]
 ; CHECK-NEXT:    ldrb r9, [r6, #-2]
 ; CHECK-NEXT:    ldrb r4, [r7], #4
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #-4]
 ; CHECK-NEXT:    ldrb r9, [r6, #-1]
 ; CHECK-NEXT:    ldrb r4, [r7, #-3]
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5]
 ; CHECK-NEXT:    ldrb r9, [r6], #4
 ; CHECK-NEXT:    ldrb r4, [r7, #-2]
 ; CHECK-NEXT:    smlabb r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #4]
-; CHECK-NEXT:    add.w r5, r5, #16
-; CHECK-NEXT:    bne .LBB7_7
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    le lr, .LBB7_7
 ; CHECK-NEXT:  .LBB7_8: @ %for.cond.cleanup.loopexit.unr-lcssa
-; CHECK-NEXT:    wls lr, lr, .LBB7_11
+; CHECK-NEXT:    wls lr, r12, .LBB7_11
 ; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
-; CHECK-NEXT:    add r0, r12
-; CHECK-NEXT:    add r1, r12
-; CHECK-NEXT:    add.w r3, r3, r12, lsl #2
+; CHECK-NEXT:    add r0, r8
+; CHECK-NEXT:    add r1, r8
+; CHECK-NEXT:    add.w r3, r3, r8, lsl #2
 ; CHECK-NEXT:  .LBB7_10: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldrb r7, [r0], #1
 ; CHECK-NEXT:    ldrb r6, [r1], #1
 ; CHECK-NEXT:    smlabb r7, r6, r7, r2
 ; CHECK-NEXT:    str r7, [r3], #4
 ; CHECK-NEXT:    le lr, .LBB7_10
 ; CHECK-NEXT:  .LBB7_11: @ %for.cond.cleanup

 for.cond.cleanup:                                 ; preds = %vector.body, %entry
   ret void
 }
 
 define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_int(i32* nocapture readonly %a, i32* nocapture readonly %b, i32 %c, i32* nocapture %res, i32 %N) {
 ; CHECK-LABEL: test_vec_mul_scalar_add_int:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    ldr.w r12, [sp, #28]
-; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    ldr r4, [sp, #28]
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    beq.w .LBB9_11
 ; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
-; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
-; CHECK-NEXT:    add.w r6, r1, r12, lsl #2
+; CHECK-NEXT:    add.w r5, r3, r4, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r4, lsl #2
 ; CHECK-NEXT:    cmp r5, r1
-; CHECK-NEXT:    add.w r4, r0, r12, lsl #2
-; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    add.w r7, r0, r4, lsl #2
+; CHECK-NEXT:    cset r12, hi
 ; CHECK-NEXT:    cmp r6, r3
 ; CHECK-NEXT:    cset r6, hi
 ; CHECK-NEXT:    cmp r5, r0
 ; CHECK-NEXT:    cset r5, hi
-; CHECK-NEXT:    cmp r4, r3
-; CHECK-NEXT:    cset r4, hi
-; CHECK-NEXT:    tst r4, r5
+; CHECK-NEXT:    cmp r7, r3
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    tst r7, r5
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    andseq.w r7, r7, r6
+; CHECK-NEXT:    andseq.w r7, r6, r12
 ; CHECK-NEXT:    beq .LBB9_4
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT:    sub.w r4, r12, #1
-; CHECK-NEXT:    and lr, r12, #3
-; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    subs r7, r4, #1
+; CHECK-NEXT:    and r12, r4, #3
+; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhs .LBB9_6
 ; CHECK-NEXT:  @ %bb.3:
-; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    mov.w r8, #0
 ; CHECK-NEXT:    b .LBB9_8
 ; CHECK-NEXT:  .LBB9_4: @ %vector.ph
-; CHECK-NEXT:    dlstp.32 lr, r12
+; CHECK-NEXT:    dlstp.32 lr, r4
 ; CHECK-NEXT:  .LBB9_5: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    vmlas.u32 q1, q0, r2
 ; CHECK-NEXT:    vstrw.32 q1, [r3], #16
 ; CHECK-NEXT:    letp lr, .LBB9_5
 ; CHECK-NEXT:    b .LBB9_11
 ; CHECK-NEXT:  .LBB9_6: @ %for.body.preheader.new
-; CHECK-NEXT:    sub.w r8, r12, lr
+; CHECK-NEXT:    bic r7, r4, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    subs r7, #4
 ; CHECK-NEXT:    add.w r5, r3, #8
+; CHECK-NEXT:    mov.w r8, #0
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
 ; CHECK-NEXT:    add.w r6, r0, #8
 ; CHECK-NEXT:    add.w r7, r1, #8
-; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:  .LBB9_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r9, [r6, #-8]
-; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    add.w r8, r8, #4
 ; CHECK-NEXT:    ldr r4, [r7, #-8]
-; CHECK-NEXT:    cmp r8, r12
 ; CHECK-NEXT:    mla r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #-8]
 ; CHECK-NEXT:    ldr r9, [r6, #-4]
 ; CHECK-NEXT:    ldr r4, [r7, #-4]
 ; CHECK-NEXT:    mla r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #-4]
 ; CHECK-NEXT:    ldr.w r9, [r6]
 ; CHECK-NEXT:    ldr r4, [r7]
 ; CHECK-NEXT:    mla r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5]
 ; CHECK-NEXT:    ldr.w r9, [r6, #4]
-; CHECK-NEXT:    add.w r6, r6, #16
+; CHECK-NEXT:    adds r6, #16
 ; CHECK-NEXT:    ldr r4, [r7, #4]
-; CHECK-NEXT:    add.w r7, r7, #16
+; CHECK-NEXT:    adds r7, #16
 ; CHECK-NEXT:    mla r4, r4, r9, r2
 ; CHECK-NEXT:    str r4, [r5, #4]
-; CHECK-NEXT:    add.w r5, r5, #16
-; CHECK-NEXT:    bne .LBB9_7
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    le lr, .LBB9_7
 ; CHECK-NEXT:  .LBB9_8: @ %for.cond.cleanup.loopexit.unr-lcssa
-; CHECK-NEXT:    wls lr, lr, .LBB9_11
+; CHECK-NEXT:    wls lr, r12, .LBB9_11
 ; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
-; CHECK-NEXT:    add.w r0, r0, r12, lsl #2
-; CHECK-NEXT:    add.w r1, r1, r12, lsl #2
-; CHECK-NEXT:    add.w r3, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r0, r0, r8, lsl #2
+; CHECK-NEXT:    add.w r1, r1, r8, lsl #2
+; CHECK-NEXT:    add.w r3, r3, r8, lsl #2
 ; CHECK-NEXT:  .LBB9_10: @ %for.body.epil
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r7, [r0], #4
 ; CHECK-NEXT:    ldr r6, [r1], #4
 ; CHECK-NEXT:    mla r7, r6, r7, r2
 ; CHECK-NEXT:    str r7, [r3], #4
 ; CHECK-NEXT:    le lr, .LBB9_10
 ; CHECK-NEXT:  .LBB9_11: @ %for.cond.cleanup

llvm/test/Transforms/HardwareLoops/ARM/structure.ll

 ; CHECK: call i32 @llvm.start.loop.iterations.i32(i32 %N)
 ; CHECK: call i32 @llvm.loop.decrement.reg.i32(
 
 ; TODO: We should be able to support the unrolled loop body.
 ; CHECK-UNROLL-LABEL: unroll_inc_int
 ; CHECK-UNROLL:     [[PREHEADER:.LBB[0-9_]+]]: @ %for.body.preheader
 ; CHECK-UNROLL-NOT: dls
 ; CHECK-UNROLL:     [[LOOP:.LBB[0-9_]+]]: @ %for.body
-; CHECK-UNROLL-NOT: le lr, [[LOOP]]
-; CHECK-UNROLL:     bne [[LOOP]]
+; CHECK-UNROLL:     le lr, [[LOOP]]
 ; CHECK-UNROLL:     wls lr, r12, [[EXIT:.LBB[0-9_]+]]
 ; CHECK-UNROLL:     [[EPIL:.LBB[0-9_]+]]:
 ; CHECK-UNROLL:     le lr, [[EPIL]]
 ; CHECK-UNROLL-NEXT: [[EXIT]]
 
 define void @unroll_inc_int(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
 entry:
   %cmp8 = icmp sgt i32 %N, 0

 ; CHECK: call { i32, i1 } @llvm.test.start.loop.iterations.i32(i32 %N)
 ; CHECK: call i32 @llvm.loop.decrement.reg.i32(
 
 ; TODO: We should be able to support the unrolled loop body.
 ; CHECK-UNROLL-LABEL: unroll_inc_unsigned
 ; CHECK-UNROLL:     [[PREHEADER:.LBB[0-9_]+]]: @ %for.body.preheader
 ; CHECK-UNROLL-NOT: dls
 ; CHECK-UNROLL:     [[LOOP:.LBB[0-9_]+]]: @ %for.body
-; CHECK-UNROLL-NOT: le lr, [[LOOP]]
-; CHECK-UNROLL:     bne [[LOOP]]
+; CHECK-UNROLL:     le lr, [[LOOP]]
 ; CHECK-UNROLL:     wls lr, r12, [[EPIL_EXIT:.LBB[0-9_]+]]
 ; CHECK-UNROLL: [[EPIL:.LBB[0-9_]+]]:
 ; CHECK-UNROLL:     le lr, [[EPIL]]
 ; CHECK-UNROLL: [[EPIL_EXIT]]:
 ; CHECK-UNROLL:     pop
 define void @unroll_inc_unsigned(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
 entry:
   %cmp8 = icmp eq i32 %N, 0

llvm/test/Transforms/LoopUnroll/runtime-loop5.ll

 ; UNROLL-4-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i3, i3* [[A]], i64 [[INDVARS_IV_NEXT_1]]
 ; UNROLL-4-NEXT:    [[TMP4:%.*]] = load i3, i3* [[ARRAYIDX_2]], align 1
 ; UNROLL-4-NEXT:    [[ADD_2:%.*]] = add nsw i3 [[TMP4]], [[ADD_1]]
 ; UNROLL-4-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT_1]], 1
 ; UNROLL-4-NEXT:    [[NITER_NEXT_2:%.*]] = add nuw nsw i3 [[NITER_NEXT_1]], 1
 ; UNROLL-4-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i3, i3* [[A]], i64 [[INDVARS_IV_NEXT_2]]
 ; UNROLL-4-NEXT:    [[TMP5:%.*]] = load i3, i3* [[ARRAYIDX_3]], align 1
 ; UNROLL-4-NEXT:    [[ADD_3]] = add nsw i3 [[TMP5]], [[ADD_2]]
-; UNROLL-4-NEXT:    [[INDVARS_IV_NEXT_3]] = add i64 [[INDVARS_IV_NEXT_2]], 1
+; UNROLL-4-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV_NEXT_2]], 1
 ; UNROLL-4-NEXT:    [[NITER_NEXT_3]] = add i3 [[NITER_NEXT_2]], 1
 ; UNROLL-4-NEXT:    [[NITER_NCMP_3:%.*]] = icmp eq i3 [[NITER_NEXT_3]], [[UNROLL_ITER]]
 ; UNROLL-4-NEXT:    br i1 [[NITER_NCMP_3]], label [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; UNROLL-4:       for.end.loopexit.unr-lcssa.loopexit:
 ; UNROLL-4-NEXT:    [[ADD_LCSSA_PH_PH:%.*]] = phi i3 [ [[ADD_3]], [[FOR_BODY]] ]
 ; UNROLL-4-NEXT:    [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3]], [[FOR_BODY]] ]
 ; UNROLL-4-NEXT:    [[SUM_02_UNR_PH:%.*]] = phi i3 [ [[ADD_3]], [[FOR_BODY]] ]
 ; UNROLL-4-NEXT:    br label [[FOR_END_LOOPEXIT_UNR_LCSSA]]