[CPG][ARM] Optimize towards branch on zero in codegenprepare

[dmgreen]

Summary

This adds a simple fold into codegenprepare that converts comparison of branches towards comparison with zero if possible. For example:

  %c = icmp ult %x, 8
  br %c, bla, blb
  %tc = lshr %x, 3
becomes
  %tc = lshr %x, 3
  %c = icmp eq %tc, 0
  br %c, bla, blb

As a first order approximation, this can reduce the number of instructions needed to perform the branch as the shift is (often) needed anyway. At the moment this does not effect very much, as llvm tends to prefer the opposite form. But it can protect against regressions from commits like rG9423f78240a2.

Simple cases of Add and Sub are added along with Shift, equally as the comparison to zero can often be folded with cpsr flags.

[lebedev.ri]

This should go into DAGCombine.

[nikic]

@lebedev.ri This is a cross-block optimization, it cannot go into DAGCombine.

[SjoerdMeijer]

Looks like a good optimisation to me. Perhaps wait a day in case others want to comment.

[nikic]

llvm/lib/CodeGen/CodeGenPrepare.cpp

7710

This is going to assert for large (> 64-bit) icmps.

7731

Style nit: Cmp->isEquality()

[dmgreen]

Thanks. I've switched it to use APInt and added a couple of extra tests.

llvm/include/llvm/CodeGen/TargetLowering.h

public:
   virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const {
     return 1;
   }
 
   /// Return true if instruction generated for equality comparison is folded
   /// with instruction generated for signed comparison.
   virtual bool isEqualityCmpFoldedWithSignedCmp() const { return true; }
 
+  /// Return true if the heuristic to prefer icmp eq zero should be used in code
+  /// gen prepare.
+  virtual bool preferZeroCompareBranch() const { return false; }
+
   /// Return true if it is safe to transform an integer-domain bitwise operation
   /// into the equivalent floating-point operation. This should be set to true
   /// if the target has IEEE-754-compliant fabs/fneg operations for the input
   /// type.
   virtual bool hasBitPreservingFPLogic(EVT VT) const {
     return false;
   }
 

llvm/lib/CodeGen/CodeGenPrepare.cpp

static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI,
   // After unmerging, verify that GEPIOp is actually only used in SrcBlock (not
   // alive on IndirectBr edges).
   assert(find_if(GEPIOp->users(), [&](User *Usr) {
         return cast<Instruction>(Usr)->getParent() != SrcBlock;
       }) == GEPIOp->users().end() && "GEPIOp is used outside SrcBlock");
   return true;
 }
 
+static bool optimizeBranch(BranchInst *Branch, const TargetLowering &TLI) {
+  // Try and convert
+  //  %c = icmp ult %x, 8
+  //  br %c, bla, blb
+  //  %tc = lshr %x, 3
+  // to
+  //  %tc = lshr %x, 3
+  //  %c = icmp eq %tc, 0
+  //  br %c, bla, blb
+  // Creating the cmp to zero can be better for the backend, especially if the
+  // lshr produces flags that can be used automatically.
+  if (!TLI.preferZeroCompareBranch() || !Branch->isConditional())
+    return false;
+
+  ICmpInst *Cmp = dyn_cast<ICmpInst>(Branch->getCondition());
+  if (!Cmp || !isa<ConstantInt>(Cmp->getOperand(1)) || !Cmp->hasOneUse())
+    return false;
+
+  Value *X = Cmp->getOperand(0);
+  APInt CmpC = cast<ConstantInt>(Cmp->getOperand(1))->getValue();
+
+  for (auto *U : X->users()) {
+    Instruction *UI = dyn_cast<Instruction>(U);
+    // A quick dominance check
+    if (!UI ||
+        (UI->getParent() != Branch->getParent() &&
+         UI->getParent() != Branch->getSuccessor(0) &&
+         UI->getParent() != Branch->getSuccessor(1)) ||
+        (UI->getParent() != Branch->getParent() &&
+         !UI->getParent()->getSinglePredecessor()))
+      continue;
+
+    if (CmpC.isPowerOf2() && Cmp->getPredicate() == ICmpInst::ICMP_ULT &&
+        match(UI, m_Shr(m_Specific(X), m_SpecificInt(CmpC.logBase2())))) {
+      IRBuilder<> Builder(Branch);
+      if (UI->getParent() != Branch->getParent())
+        UI->moveBefore(Branch);
+      Value *NewCmp = Builder.CreateCmp(ICmpInst::ICMP_EQ, UI,
+                                        ConstantInt::get(UI->getType(), 0));
+      LLVM_DEBUG(dbgs() << "Converting " << *Cmp << "\n");
+      LLVM_DEBUG(dbgs() << " to compare on zero: " << *NewCmp << "\n");
+      Cmp->replaceAllUsesWith(NewCmp);
+      return true;
+    }
+    if (Cmp->isEquality() &&
+        (match(UI, m_Add(m_Specific(X), m_SpecificInt(-CmpC))) ||
+         match(UI, m_Sub(m_Specific(X), m_SpecificInt(CmpC))))) {
+      IRBuilder<> Builder(Branch);
+      if (UI->getParent() != Branch->getParent())
+        UI->moveBefore(Branch);
+      Value *NewCmp = Builder.CreateCmp(Cmp->getPredicate(), UI,
+                                        ConstantInt::get(UI->getType(), 0));
+      LLVM_DEBUG(dbgs() << "Converting " << *Cmp << "\n");
+      LLVM_DEBUG(dbgs() << " to compare on zero: " << *NewCmp << "\n");
+      Cmp->replaceAllUsesWith(NewCmp);
+      return true;
+    }
+  }
+  return false;
+}
+
 bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) {
   // Bail out if we inserted the instruction to prevent optimizations from
   // stepping on each other's toes.
   if (InsertedInsts.count(I))
     return false;
 
   // TODO: Move into the switch on opcode below here.
   if (PHINode *P = dyn_cast<PHINode>(I)) {
bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) {
   case Instruction::Select:
     return optimizeSelectInst(cast<SelectInst>(I));
   case Instruction::ShuffleVector:
     return optimizeShuffleVectorInst(cast<ShuffleVectorInst>(I));
   case Instruction::Switch:
     return optimizeSwitchInst(cast<SwitchInst>(I));
   case Instruction::ExtractElement:
     return optimizeExtractElementInst(cast<ExtractElementInst>(I));
+  case Instruction::Br:
+    return optimizeBranch(cast<BranchInst>(I), *TLI);
   }
 
   return false;
 }
 
 /// Given an OR instruction, check to see if this is a bitreverse
 /// idiom. If so, insert the new intrinsic and return true.
 bool CodeGenPrepare::makeBitReverse(Instruction &I) {

llvm/lib/Target/ARM/ARMISelLowering.h

public:
 
     /// createFastISel - This method returns a target specific FastISel object,
     /// or null if the target does not support "fast" ISel.
     FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
                              const TargetLibraryInfo *libInfo) const override;
 
     Sched::Preference getSchedulingPreference(SDNode *N) const override;
 
+    bool preferZeroCompareBranch() const override { return true; }
+
     bool
     isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
 
     /// isFPImmLegal - Returns true if the target can instruction select the
     /// specified FP immediate natively. If false, the legalizer will
     /// materialize the FP immediate as a load from a constant pool.
     bool isFPImmLegal(const APFloat &Imm, EVT VT,

llvm/test/CodeGen/ARM/branch-on-zero.ll

 
 while.end:                                        ; preds = %while.body, %entry
   ret i32 0
 }
 
 define i32 @test_lshr2(i32* nocapture %x, i32* nocapture readonly %y, i32 %n) {
 ; CHECK-V6M-LABEL: test_lshr2:
 ; CHECK-V6M:       @ %bb.0: @ %entry
-; CHECK-V6M-NEXT:    cmp r2, #4
-; CHECK-V6M-NEXT:    blo .LBB1_3
-; CHECK-V6M-NEXT:  @ %bb.1: @ %while.body.preheader
 ; CHECK-V6M-NEXT:    lsrs r2, r2, #2
-; CHECK-V6M-NEXT:  .LBB1_2: @ %while.body
+; CHECK-V6M-NEXT:    beq .LBB1_2
+; CHECK-V6M-NEXT:  .LBB1_1: @ %while.body
 ; CHECK-V6M-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-V6M-NEXT:    ldm r1!, {r3}
 ; CHECK-V6M-NEXT:    lsls r3, r3, #1
 ; CHECK-V6M-NEXT:    stm r0!, {r3}
 ; CHECK-V6M-NEXT:    subs r2, r2, #1
-; CHECK-V6M-NEXT:    bne .LBB1_2
-; CHECK-V6M-NEXT:  .LBB1_3: @ %while.end
+; CHECK-V6M-NEXT:    bne .LBB1_1
+; CHECK-V6M-NEXT:  .LBB1_2: @ %while.end
 ; CHECK-V6M-NEXT:    movs r0, #0
 ; CHECK-V6M-NEXT:    bx lr
 ;
 ; CHECK-V7M-LABEL: test_lshr2:
 ; CHECK-V7M:       @ %bb.0: @ %entry
-; CHECK-V7M-NEXT:    cmp r2, #4
-; CHECK-V7M-NEXT:    blo .LBB1_3
+; CHECK-V7M-NEXT:    lsrs r2, r2, #2
+; CHECK-V7M-NEXT:    beq .LBB1_3
 ; CHECK-V7M-NEXT:  @ %bb.1: @ %while.body.preheader
 ; CHECK-V7M-NEXT:    subs r1, #4
 ; CHECK-V7M-NEXT:    subs r0, #4
-; CHECK-V7M-NEXT:    lsrs r2, r2, #2
 ; CHECK-V7M-NEXT:  .LBB1_2: @ %while.body
 ; CHECK-V7M-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-V7M-NEXT:    ldr r3, [r1, #4]!
 ; CHECK-V7M-NEXT:    subs r2, #1
 ; CHECK-V7M-NEXT:    lsl.w r3, r3, #1
 ; CHECK-V7M-NEXT:    str r3, [r0, #4]!
 ; CHECK-V7M-NEXT:    bne .LBB1_2
 ; CHECK-V7M-NEXT:  .LBB1_3: @ %while.end
 ; CHECK-V7M-NEXT:    movs r0, #0
 ; CHECK-V7M-NEXT:    bx lr
 ;
 ; CHECK-V81M-LABEL: test_lshr2:
 ; CHECK-V81M:       @ %bb.0: @ %entry
 ; CHECK-V81M-NEXT:    .save {r7, lr}
 ; CHECK-V81M-NEXT:    push {r7, lr}
-; CHECK-V81M-NEXT:    cmp r2, #4
-; CHECK-V81M-NEXT:    blo .LBB1_3
-; CHECK-V81M-NEXT:  @ %bb.1: @ %while.body.preheader
-; CHECK-V81M-NEXT:    lsr.w lr, r2, #2
-; CHECK-V81M-NEXT:  .LBB1_2: @ %while.body
+; CHECK-V81M-NEXT:    lsrs r2, r2, #2
+; CHECK-V81M-NEXT:    wls lr, r2, .LBB1_2
+; CHECK-V81M-NEXT:  .LBB1_1: @ %while.body
 ; CHECK-V81M-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-V81M-NEXT:    ldr r2, [r1], #4
 ; CHECK-V81M-NEXT:    lsls r2, r2, #1
 ; CHECK-V81M-NEXT:    str r2, [r0], #4
-; CHECK-V81M-NEXT:    le lr, .LBB1_2
-; CHECK-V81M-NEXT:  .LBB1_3: @ %while.end
+; CHECK-V81M-NEXT:    le lr, .LBB1_1
+; CHECK-V81M-NEXT:  .LBB1_2: @ %while.end
 ; CHECK-V81M-NEXT:    movs r0, #0
 ; CHECK-V81M-NEXT:    pop {r7, pc}
 ;
 ; CHECK-V7A-LABEL: test_lshr2:
 ; CHECK-V7A:       @ %bb.0: @ %entry
-; CHECK-V7A-NEXT:    cmp r2, #4
-; CHECK-V7A-NEXT:    blo .LBB1_3
+; CHECK-V7A-NEXT:    mov r3, #0
+; CHECK-V7A-NEXT:    cmp r3, r2, lsr #2
+; CHECK-V7A-NEXT:    beq .LBB1_3
 ; CHECK-V7A-NEXT:  @ %bb.1: @ %while.body.preheader
 ; CHECK-V7A-NEXT:    lsr r2, r2, #2
 ; CHECK-V7A-NEXT:  .LBB1_2: @ %while.body
 ; CHECK-V7A-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-V7A-NEXT:    ldr r3, [r1], #4
 ; CHECK-V7A-NEXT:    subs r2, r2, #1
 ; CHECK-V7A-NEXT:    lsl r3, r3, #1
 ; CHECK-V7A-NEXT:    str r3, [r0], #4

llvm/test/CodeGen/Thumb2/2010-02-11-phi-cycle.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=thumbv7-none-eabi | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32-n32"
 
 define i32 @test(i32 %n) nounwind {
 ; CHECK-LABEL: test:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    cmp r0, #1
+; CHECK-NEXT:    subs r4, r0, #1
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r4, pc}
-; CHECK-NEXT:  .LBB0_1: @ %bb.nph
-; CHECK-NEXT:    subs r4, r0, #1
-; CHECK-NEXT:  .LBB0_2: @ %bb
+; CHECK-NEXT:  .LBB0_1: @ %bb
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    bl f
 ; CHECK-NEXT:    bl g
 ; CHECK-NEXT:    subs r4, #1
-; CHECK-NEXT:    bne .LBB0_2
-; CHECK-NEXT:  @ %bb.3: @ %return
+; CHECK-NEXT:    bne .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %return
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = icmp eq i32 %n, 1                          ; <i1> [#uses=1]
   br i1 %0, label %return, label %bb.nph
 
 bb.nph:                                           ; preds = %entry
   %tmp = add i32 %n, -1                           ; <i32> [#uses=1]
   br label %bb

 }
 
 define i32 @test_dead_cycle(i32 %n) nounwind {
 ; also check for duplicate induction variables (radar 7645034)
 ; CHECK-LABEL: test_dead_cycle:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    cmp r0, #1
+; CHECK-NEXT:    subs r4, r0, #1
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r4, pc}
-; CHECK-NEXT:  .LBB1_1: @ %bb.nph
-; CHECK-NEXT:    subs r4, r0, #1
-; CHECK-NEXT:    b .LBB1_3
-; CHECK-NEXT:  .LBB1_2: @ %bb2
-; CHECK-NEXT:    @ in Loop: Header=BB1_3 Depth=1
-; CHECK-NEXT:    subs r4, #1
-; CHECK-NEXT:    beq .LBB1_5
-; CHECK-NEXT:  .LBB1_3: @ %bb
+; CHECK-NEXT:  .LBB1_1: @ %bb
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmp r4, #2
-; CHECK-NEXT:    blt .LBB1_2
-; CHECK-NEXT:  @ %bb.4: @ %bb1
-; CHECK-NEXT:    @ in Loop: Header=BB1_3 Depth=1
+; CHECK-NEXT:    blt .LBB1_3
+; CHECK-NEXT:  @ %bb.2: @ %bb1
+; CHECK-NEXT:    @ in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    bl f
 ; CHECK-NEXT:    bl g
-; CHECK-NEXT:    b .LBB1_2
-; CHECK-NEXT:  .LBB1_5: @ %return
+; CHECK-NEXT:  .LBB1_3: @ %bb2
+; CHECK-NEXT:    @ in Loop: Header=BB1_1 Depth=1
+; CHECK-NEXT:    subs r4, #1
+; CHECK-NEXT:    bne .LBB1_1
+; CHECK-NEXT:  @ %bb.4: @ %return
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %0 = icmp eq i32 %n, 1                          ; <i1> [#uses=1]
   br i1 %0, label %return, label %bb.nph
 
 bb.nph:                                           ; preds = %entry
   %tmp = add i32 %n, -1                           ; <i32> [#uses=2]
   br label %bb

llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -codegenprepare < %s | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv8.1m.main-none-eabi"
 
 define i32 @lshr3_then(i32 %a) {
 ; CHECK-LABEL: @lshr3_then(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A:%.*]], 8
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = lshr i32 [[A:%.*]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    ret i32 0
 ; CHECK:       else:
-; CHECK-NEXT:    [[L:%.*]] = lshr i32 [[A]], 3
 ; CHECK-NEXT:    ret i32 [[L]]
 ;
 entry:
   %c = icmp ult i32 %a, 8
   br i1 %c, label %then, label %else
 
 then:
   ret i32 0
 
 else:
   %l = lshr i32 %a, 3
   ret i32 %l
 }
 
 define i32 @lshr5_else(i32 %a) {
 ; CHECK-LABEL: @lshr5_else(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A:%.*]], 32
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = lshr i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    [[L:%.*]] = lshr i32 [[A]], 5
 ; CHECK-NEXT:    ret i32 [[L]]
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %c = icmp ult i32 %a, 32
   br i1 %c, label %then, label %else
 
 then:
   %l = lshr i32 %a, 5
   ret i32 %l
 
 else:
   ret i32 0
 }
 
 define i32 @lshr2_entry(i32 %a) {
 ; CHECK-LABEL: @lshr2_entry(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L:%.*]] = lshr i32 [[A:%.*]], 1
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], 2
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    ret i32 [[L]]
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %l = lshr i32 %a, 1
   %c = icmp ult i32 %a, 2

 
 else:
   ret i32 0
 }
 
 define i32 @ashr5_else(i32 %a) {
 ; CHECK-LABEL: @ashr5_else(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A:%.*]], 32
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = ashr i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    [[L:%.*]] = ashr i32 [[A]], 5
 ; CHECK-NEXT:    ret i32 [[L]]
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %c = icmp ult i32 %a, 32
   br i1 %c, label %then, label %else
 

 else:
   %l = add i32 %a, 10
   ret i32 %l
 }
 
 define i32 @addm10_then(i32 %a) {
 ; CHECK-LABEL: @addm10_then(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[A:%.*]], 10
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = add i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    [[L:%.*]] = add i32 [[A]], -10
 ; CHECK-NEXT:    ret i32 [[L]]
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %c = icmp eq i32 %a, 10
   br i1 %c, label %then, label %else
 

 
 else:
   ret i32 0
 }
 
 define i32 @sub10_else(i32 %a) {
 ; CHECK-LABEL: @sub10_else(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[A:%.*]], 10
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = sub i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    ret i32 0
 ; CHECK:       else:
-; CHECK-NEXT:    [[L:%.*]] = sub i32 [[A]], 10
 ; CHECK-NEXT:    ret i32 [[L]]
 ;
 entry:
   %c = icmp eq i32 %a, 10
   br i1 %c, label %then, label %else
 
 then:
   ret i32 0
 
 else:
   %l = sub i32 %a, 10
   ret i32 %l
 }
 
 define i32 @subm10_then(i32 %a) {
 ; CHECK-LABEL: @subm10_then(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[A:%.*]], -10
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = sub i32 [[A:%.*]], -10
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    [[L:%.*]] = sub i32 [[A]], -10
 ; CHECK-NEXT:    ret i32 [[L]]
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %c = icmp eq i32 %a, -10
   br i1 %c, label %then, label %else
 
 then:
   %l = sub i32 %a, -10
   ret i32 %l
 
 else:
   ret i32 0
 }
 
 define i64 @lshr64(i64 %a) {
 ; CHECK-LABEL: @lshr64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[A:%.*]], 1099511627776
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = lshr i64 [[A:%.*]], 40
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    ret i64 0
 ; CHECK:       else:
-; CHECK-NEXT:    [[L:%.*]] = lshr i64 [[A]], 40
 ; CHECK-NEXT:    ret i64 [[L]]
 ;
 entry:
   %c = icmp ult i64 %a, 1099511627776
   br i1 %c, label %then, label %else
 
 then:
   ret i64 0
 
 else:
   %l = lshr i64 %a, 40
   ret i64 %l
 }
 
 define i128 @lshr128(i128 %a) {
 ; CHECK-LABEL: @lshr128(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i128 [[A:%.*]], 36893488147419103232
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = lshr i128 [[A:%.*]], 65
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i128 [[L]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    ret i128 0
 ; CHECK:       else:
-; CHECK-NEXT:    [[L:%.*]] = lshr i128 [[A]], 65
 ; CHECK-NEXT:    ret i128 [[L]]
 ;
 entry:
   %c = icmp ult i128 %a, 36893488147419103232
   br i1 %c, label %then, label %else
 
 then:
   ret i128 0