[LoopIdiom] Freeze BitPos if !isGuaranteedNotToBeUndefOrPoison

[StephenFan]

Summary

Fixes: https://github.com/llvm/llvm-project/issues/62873

[nikic]

llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp

2403

Maybe it would be more robust to RAUW all uses instead?

[StephenFan]

RAUW.

[nikic]

LGTM

llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp

2403

Add if (!InsertPtr) return false; here, there are some rare edge cases where there is no insertion point.

[StephenFan]

Return false if InsertPt is nullptr

llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp

if (TTI->getArithmeticInstrCost(Instruction::Shl, Ty, CostKind) >
       TargetTransformInfo::TCC_Basic) {
     LLVM_DEBUG(dbgs() << DEBUG_TYPE " Shift is too costly, not beneficial\n");
     return MadeChange;
   }
 
   // Ok, transform appears worthwhile.
   MadeChange = true;
 
+  if (!isGuaranteedNotToBeUndefOrPoison(BitPos)) {
+    // BitMask may be computed from BitPos, Freeze BitPos so we can increase
+    // it's use count.
+    Instruction *InsertPt = nullptr;
+    if (auto *BitPosI = dyn_cast<Instruction>(BitPos))
+      InsertPt = BitPosI->getInsertionPointAfterDef();
+    else
+      InsertPt = &*DT->getRoot()->getFirstNonPHIOrDbgOrAlloca();
+    if (!InsertPt)
+      return false;
+    FreezeInst *BitPosFrozen =
+        new FreezeInst(BitPos, BitPos->getName() + ".fr", InsertPt);
+    BitPos->replaceUsesWithIf(BitPosFrozen, [BitPosFrozen](Use &U) {
+      return U.getUser() != BitPosFrozen;
+    });
+    BitPos = BitPosFrozen;
+  }
+
   // Step 1: Compute the loop trip count.
 
   Value *LowBitMask = Builder.CreateAdd(BitMask, Constant::getAllOnesValue(Ty),
                                         BitPos->getName() + ".lowbitmask");
   Value *Mask =
       Builder.CreateOr(LowBitMask, BitMask, BitPos->getName() + ".mask");
   Value *XMasked = Builder.CreateAnd(X, Mask, X->getName() + ".masked");
   CallInst *XMaskedNumLeadingZeros = Builder.CreateIntrinsic(

llvm/test/Transforms/LoopIdiom/X86/left-shift-until-bittest.ll

 ;     x <<= 1;
 ; .. which is an uncountable loop.
 ; We should transform it into it's countable form.
 
 ; Most basic example.
 define i32 @p0_i32(i32 %x, i32 %bit) {
 ; ALL-LABEL: @p0_i32(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG16:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG16:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META9:![0-9]+]], metadata !DIExpression()), !dbg [[DBG16]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG17:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG17]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG17]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG17:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG17]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG17]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG17]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG17]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG17]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG17]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG17]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG17]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG17]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG17]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG18:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG17]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG17]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META11:![0-9]+]], metadata !DIExpression()), !dbg [[DBG17]]

 end:
   ret i32 %x.curr
 }
 
 ; Same, but in some other bit width.
 define i16 @p1_i16(i16 %x, i16 %bit) {
 ; LZCNT-LABEL: @p1_i16(
 ; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i16 1, [[BIT:%.*]], !dbg [[DBG32:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR:%.*]] = freeze i16 [[BIT:%.*]]
+; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i16 1, [[BIT_FR]], !dbg [[DBG32:![0-9]+]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i16 [[BITMASK]], metadata [[META26:![0-9]+]], metadata !DIExpression()), !dbg [[DBG32]]
-; LZCNT-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i16 [[BITMASK]], -1, !dbg [[DBG33:![0-9]+]]
-; LZCNT-NEXT:    [[BIT_MASK:%.*]] = or i16 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG33]]
-; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i16 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG33]]
+; LZCNT-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i16 [[BITMASK]], -1, !dbg [[DBG33:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR_MASK:%.*]] = or i16 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG33]]
+; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i16 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG33]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X_MASKED]], i1 true), !dbg [[DBG33]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i16 16, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG33]]
 ; LZCNT-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i16 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG33]]
-; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i16 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG33]]
+; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i16 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG33]]
 ; LZCNT-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i16 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG33]]
 ; LZCNT-NEXT:    [[X_CURR:%.*]] = shl i16 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG33]]
 ; LZCNT-NEXT:    [[X_NEXT:%.*]] = shl i16 [[X_CURR]], 1, !dbg [[DBG33]]
 ; LZCNT-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG34:![0-9]+]]
 ; LZCNT:       loop:
 ; LZCNT-NEXT:    [[LOOP_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG33]]
 ; LZCNT-NEXT:    [[TMP0:%.*]] = phi i16 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG33]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i16 [[TMP0]], metadata [[META28:![0-9]+]], metadata !DIExpression()), !dbg [[DBG33]]

 end:
   ret i16 %x.curr
 }
 
 ; We don't particularly care whether %x.curr or %x.curr will live-out.
 define i32 @p2_different_liveout(i32 %x, i32 %bit) {
 ; ALL-LABEL: @p2_different_liveout(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG47:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG47:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META42:![0-9]+]], metadata !DIExpression()), !dbg [[DBG47]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG48:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG48]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG48]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG48:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG48]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG48]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG48]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG48]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG48]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG48]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG48]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG48]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG48]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG48]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG49:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG48]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG48]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META43:![0-9]+]], metadata !DIExpression()), !dbg [[DBG48]]
end:
   store i32 %x.next, ptr %p1
   ret void
 }
 
 ; All no-wrap flags can be kept on the shift.
 define void @p5_nuw(i32 %x, i32 %bit, ptr %p0, ptr %p1) {
 ; ALL-LABEL: @p5_nuw(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG92:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG92:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META87:![0-9]+]], metadata !DIExpression()), !dbg [[DBG92]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG93:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG93]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG93]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG93:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG93]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG93]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG93]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG93]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG93]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG93]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG93]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG93]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl nuw i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG93]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl nuw i32 [[X]], [[LOOP_TRIPCOUNT]], !dbg [[DBG93]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG94:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG93]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG93]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META88:![0-9]+]], metadata !DIExpression()), !dbg [[DBG93]]

 end:
   store i32 %x.curr, ptr %p0
   store i32 %x.next, ptr %p1
   ret void
 }
 define void @p6_nsw(i32 %x, i32 %bit, ptr %p0, ptr %p1) {
 ; ALL-LABEL: @p6_nsw(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG109:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG109:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META104:![0-9]+]], metadata !DIExpression()), !dbg [[DBG109]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG110:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG110]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG110]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG110:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG110]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG110]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG110]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG110]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG110]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG110]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG110]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG110]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl nsw i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG110]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl nsw i32 [[X]], [[LOOP_TRIPCOUNT]], !dbg [[DBG110]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG111:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG110]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG110]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META105:![0-9]+]], metadata !DIExpression()), !dbg [[DBG110]]

 end:
   store i32 %x.curr, ptr %p0
   store i32 %x.next, ptr %p1
   ret void
 }
 define void @p7_nuwnsw(i32 %x, i32 %bit, ptr %p0, ptr %p1) {
 ; ALL-LABEL: @p7_nuwnsw(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG126:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG126:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META121:![0-9]+]], metadata !DIExpression()), !dbg [[DBG126]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG127:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG127]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG127]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG127:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG127]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG127]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG127]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG127]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG127]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG127]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG127]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG127]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl nuw nsw i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG127]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl nuw nsw i32 [[X]], [[LOOP_TRIPCOUNT]], !dbg [[DBG127]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG128:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG127]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG127]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META122:![0-9]+]], metadata !DIExpression()), !dbg [[DBG127]]
end:
   store i32 %x.curr, ptr %p0
   store i32 %x.next, ptr %p1
   ret void
 }
 
 define void @p10_x_is_not_one(i32 %bit, ptr %p0, ptr %p1) {
 ; ALL-LABEL: @p10_x_is_not_one(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG171:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG171:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META166:![0-9]+]], metadata !DIExpression()), !dbg [[DBG171]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG172:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG172]]
-; ALL-NEXT:    [[DOTMASKED:%.*]] = and i32 2, [[BIT_MASK]], !dbg [[DBG172]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG172:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG172]]
+; ALL-NEXT:    [[DOTMASKED:%.*]] = and i32 2, [[BIT_FR_MASK]], !dbg [[DBG172]]
 ; ALL-NEXT:    [[DOTMASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[DOTMASKED]], i1 true), !dbg [[DBG172]]
 ; ALL-NEXT:    [[DOTMASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[DOTMASKED_NUMLEADINGZEROS]], !dbg [[DBG172]]
 ; ALL-NEXT:    [[DOTMASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[DOTMASKED_NUMACTIVEBITS]], -1, !dbg [[DBG172]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[DOTMASKED_LEADINGONEPOS]], !dbg [[DBG172]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[DOTMASKED_LEADINGONEPOS]], !dbg [[DBG172]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG172]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 2, [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG172]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG172]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG173:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG172]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ 2, [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG172]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META167:![0-9]+]], metadata !DIExpression()), !dbg [[DBG172]]
end:
   store i32 %x.next, ptr %p1
   ret void
 }
 
 ; Check that loop backedge's cmp-br order is correctly handled
 define i32 @p11(i32 %x, i32 %bit) {
 ; ALL-LABEL: @p11(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG188:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG188:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META183:![0-9]+]], metadata !DIExpression()), !dbg [[DBG188]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG189:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG189]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG189]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG189:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG189]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG189]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG189]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG189]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG189]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG189]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG189]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG189]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG189]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG189]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG190:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG189]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG189]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META184:![0-9]+]], metadata !DIExpression()), !dbg [[DBG189]]

 end:
   ret i32 %x.curr
 }
 
 ; `and` is commutative, so ensure that order is irrelevant
 define i32 @p12(i32 %x, i32 %bit) {
 ; ALL-LABEL: @p12(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG203:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG203:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META198:![0-9]+]], metadata !DIExpression()), !dbg [[DBG203]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG204:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG204]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG204]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG204:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG204]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG204]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG204]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG204]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG204]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG204]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG204]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG204]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG204]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG204]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG205:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG204]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG204]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META199:![0-9]+]], metadata !DIExpression()), !dbg [[DBG204]]
end:
   ret i32 %x.curr
 }
 
 ; PHI node does not have any particular order for it's incomings,
 ; but check that the other order still works.
 define i32 @p13(i32 %x, i32 %bit) {
 ; ALL-LABEL: @p13(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG218:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG218:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META213:![0-9]+]], metadata !DIExpression()), !dbg [[DBG218]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG219:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG219]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG219]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG219:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG219]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG219]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG219]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG219]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG219]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG219]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG219]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG219]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG219]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG219]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG220:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG219]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[TMP1:%.*]], [[LOOP]] ], [ [[X]], [[ENTRY]] ], !dbg [[DBG219]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META214:![0-9]+]], metadata !DIExpression()), !dbg [[DBG219]]

 
 ;-------------------------------------------------------------------------------
 ; Tests with extra cruft.
 
 ; If loop body has any extra instructions we don't want to deal with it.
 define i32 @n29(i32 %x, i32 %bit) {
 ; ALL-LABEL: @n29(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG448:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG448:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META443:![0-9]+]], metadata !DIExpression()), !dbg [[DBG448]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG449:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG449]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG449]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG449:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG449]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG449]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG449]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG449]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG449]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG449]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG449]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG449]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG449]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG449]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG450:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG449]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG449]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META444:![0-9]+]], metadata !DIExpression()), !dbg [[DBG449]]

 end:
   ret i32 %x.curr
 }
 
 ; In-loop instructions should not have uses outside of the loop.
 define i32 @n31(i32 %x, i32 %bit) {
 ; ALL-LABEL: @n31(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG476:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG476:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META471:![0-9]+]], metadata !DIExpression()), !dbg [[DBG476]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG477:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG477]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG477]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG477:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG477]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG477]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG477]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG477]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG477]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG477]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG477]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG477]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG477]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG477]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG478:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG477]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG477]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META472:![0-9]+]], metadata !DIExpression()), !dbg [[DBG477]]

 
 end:
   call void @use32(i32 %x.curr.bitmasked)
   ret i32 %x.curr
 }
 define i32 @n32(i32 %x, i32 %bit) {
 ; ALL-LABEL: @n32(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], !dbg [[DBG492:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR:%.*]] = freeze i32 [[BIT:%.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT_FR]], !dbg [[DBG492:![0-9]+]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], metadata [[META487:![0-9]+]], metadata !DIExpression()), !dbg [[DBG492]]
-; ALL-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG493:![0-9]+]]
-; ALL-NEXT:    [[BIT_MASK:%.*]] = or i32 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG493]]
-; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG493]]
+; ALL-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i32 [[BITMASK]], -1, !dbg [[DBG493:![0-9]+]]
+; ALL-NEXT:    [[BIT_FR_MASK:%.*]] = or i32 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG493]]
+; ALL-NEXT:    [[X_MASKED:%.*]] = and i32 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG493]]
 ; ALL-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X_MASKED]], i1 true), !dbg [[DBG493]]
 ; ALL-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i32 32, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG493]]
 ; ALL-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i32 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG493]]
-; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG493]]
+; ALL-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i32 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG493]]
 ; ALL-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i32 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG493]]
 ; ALL-NEXT:    [[X_CURR:%.*]] = shl i32 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG493]]
 ; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X_CURR]], 1, !dbg [[DBG493]]
 ; ALL-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG494:![0-9]+]]
 ; ALL:       loop:
 ; ALL-NEXT:    [[LOOP_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG493]]
 ; ALL-NEXT:    [[TMP0:%.*]] = phi i32 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG493]]
 ; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[TMP0]], metadata [[META488:![0-9]+]], metadata !DIExpression()), !dbg [[DBG493]]

 end:
   ret i32 %x.curr
 }
 
 ; Various weird bit widths
 define void @t35_i1(i1 %x, i1 %bit, ptr %p0, ptr %p1) {
 ; LZCNT-LABEL: @t35_i1(
 ; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i1 true, [[BIT:%.*]], !dbg [[DBG539:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR:%.*]] = freeze i1 [[BIT:%.*]]
+; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i1 true, [[BIT_FR]], !dbg [[DBG539:![0-9]+]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i1 [[BITMASK]], metadata [[META534:![0-9]+]], metadata !DIExpression()), !dbg [[DBG539]]
-; LZCNT-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i1 [[BITMASK]], true, !dbg [[DBG540:![0-9]+]]
-; LZCNT-NEXT:    [[BIT_MASK:%.*]] = or i1 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG540]]
-; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i1 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG540]]
+; LZCNT-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i1 [[BITMASK]], true, !dbg [[DBG540:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR_MASK:%.*]] = or i1 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG540]]
+; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i1 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG540]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i1 @llvm.ctlz.i1(i1 [[X_MASKED]], i1 true), !dbg [[DBG540]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i1 true, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG540]]
 ; LZCNT-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add i1 [[X_MASKED_NUMACTIVEBITS]], true, !dbg [[DBG540]]
-; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i1 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG540]]
+; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i1 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG540]]
 ; LZCNT-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i1 [[LOOP_BACKEDGETAKENCOUNT]], true, !dbg [[DBG540]]
 ; LZCNT-NEXT:    [[X_CURR:%.*]] = shl i1 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG540]]
 ; LZCNT-NEXT:    [[X_NEXT:%.*]] = shl i1 [[X_CURR]], true, !dbg [[DBG540]]
 ; LZCNT-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG541:![0-9]+]]
 ; LZCNT:       loop:
 ; LZCNT-NEXT:    [[LOOP_IV:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG540]]
 ; LZCNT-NEXT:    [[TMP0:%.*]] = phi i1 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG540]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i1 [[TMP0]], metadata [[META535:![0-9]+]], metadata !DIExpression()), !dbg [[DBG540]]

 end:
   store i1 %x.curr, ptr %p0
   store i1 %x.next, ptr %p1
   ret void
 }
 define void @t36_i2(i2 %x, i2 %bit, ptr %p0, ptr %p1) {
 ; LZCNT-LABEL: @t36_i2(
 ; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i2 1, [[BIT:%.*]], !dbg [[DBG556:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR:%.*]] = freeze i2 [[BIT:%.*]]
+; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i2 1, [[BIT_FR]], !dbg [[DBG556:![0-9]+]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i2 [[BITMASK]], metadata [[META551:![0-9]+]], metadata !DIExpression()), !dbg [[DBG556]]
-; LZCNT-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i2 [[BITMASK]], -1, !dbg [[DBG557:![0-9]+]]
-; LZCNT-NEXT:    [[BIT_MASK:%.*]] = or i2 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG557]]
-; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i2 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG557]]
+; LZCNT-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i2 [[BITMASK]], -1, !dbg [[DBG557:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR_MASK:%.*]] = or i2 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG557]]
+; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i2 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG557]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i2 @llvm.ctlz.i2(i2 [[X_MASKED]], i1 true), !dbg [[DBG557]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw i2 -2, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG557]]
 ; LZCNT-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add i2 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG557]]
-; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i2 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG557]]
+; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i2 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG557]]
 ; LZCNT-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw i2 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG557]]
 ; LZCNT-NEXT:    [[X_CURR:%.*]] = shl i2 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG557]]
 ; LZCNT-NEXT:    [[X_NEXT:%.*]] = shl i2 [[X_CURR]], 1, !dbg [[DBG557]]
 ; LZCNT-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG558:![0-9]+]]
 ; LZCNT:       loop:
 ; LZCNT-NEXT:    [[LOOP_IV:%.*]] = phi i2 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG557]]
 ; LZCNT-NEXT:    [[TMP0:%.*]] = phi i2 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG557]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i2 [[TMP0]], metadata [[META552:![0-9]+]], metadata !DIExpression()), !dbg [[DBG557]]

 end:
   store i2 %x.curr, ptr %p0
   store i2 %x.next, ptr %p1
   ret void
 }
 define void @t37_i3(i3 %x, i3 %bit, ptr %p0, ptr %p1) {
 ; LZCNT-LABEL: @t37_i3(
 ; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i3 1, [[BIT:%.*]], !dbg [[DBG573:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR:%.*]] = freeze i3 [[BIT:%.*]]
+; LZCNT-NEXT:    [[BITMASK:%.*]] = shl i3 1, [[BIT_FR]], !dbg [[DBG573:![0-9]+]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i3 [[BITMASK]], metadata [[META568:![0-9]+]], metadata !DIExpression()), !dbg [[DBG573]]
-; LZCNT-NEXT:    [[BIT_LOWBITMASK:%.*]] = add i3 [[BITMASK]], -1, !dbg [[DBG574:![0-9]+]]
-; LZCNT-NEXT:    [[BIT_MASK:%.*]] = or i3 [[BIT_LOWBITMASK]], [[BITMASK]], !dbg [[DBG574]]
-; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i3 [[X:%.*]], [[BIT_MASK]], !dbg [[DBG574]]
+; LZCNT-NEXT:    [[BIT_FR_LOWBITMASK:%.*]] = add i3 [[BITMASK]], -1, !dbg [[DBG574:![0-9]+]]
+; LZCNT-NEXT:    [[BIT_FR_MASK:%.*]] = or i3 [[BIT_FR_LOWBITMASK]], [[BITMASK]], !dbg [[DBG574]]
+; LZCNT-NEXT:    [[X_MASKED:%.*]] = and i3 [[X:%.*]], [[BIT_FR_MASK]], !dbg [[DBG574]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMLEADINGZEROS:%.*]] = call i3 @llvm.ctlz.i3(i3 [[X_MASKED]], i1 true), !dbg [[DBG574]]
 ; LZCNT-NEXT:    [[X_MASKED_NUMACTIVEBITS:%.*]] = sub nuw nsw i3 3, [[X_MASKED_NUMLEADINGZEROS]], !dbg [[DBG574]]
 ; LZCNT-NEXT:    [[X_MASKED_LEADINGONEPOS:%.*]] = add nsw i3 [[X_MASKED_NUMACTIVEBITS]], -1, !dbg [[DBG574]]
-; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i3 [[BIT]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG574]]
+; LZCNT-NEXT:    [[LOOP_BACKEDGETAKENCOUNT:%.*]] = sub nuw nsw i3 [[BIT_FR]], [[X_MASKED_LEADINGONEPOS]], !dbg [[DBG574]]
 ; LZCNT-NEXT:    [[LOOP_TRIPCOUNT:%.*]] = add nuw nsw i3 [[LOOP_BACKEDGETAKENCOUNT]], 1, !dbg [[DBG574]]
 ; LZCNT-NEXT:    [[X_CURR:%.*]] = shl i3 [[X]], [[LOOP_BACKEDGETAKENCOUNT]], !dbg [[DBG574]]
 ; LZCNT-NEXT:    [[X_NEXT:%.*]] = shl i3 [[X_CURR]], 1, !dbg [[DBG574]]
 ; LZCNT-NEXT:    br label [[LOOP:%.*]], !dbg [[DBG575:![0-9]+]]
 ; LZCNT:       loop:
 ; LZCNT-NEXT:    [[LOOP_IV:%.*]] = phi i3 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_IV_NEXT:%.*]], [[LOOP]] ], !dbg [[DBG574]]
 ; LZCNT-NEXT:    [[TMP0:%.*]] = phi i3 [ [[X]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ], !dbg [[DBG574]]
 ; LZCNT-NEXT:    call void @llvm.dbg.value(metadata i3 [[TMP0]], metadata [[META569:![0-9]+]], metadata !DIExpression()), !dbg [[DBG574]]