[IntegerDivision][AMDGPU] Use CreateLogicalOr to block poison propagation.

[craig.topper]

Summary

There are two ctlz intrinsics here with the zero_is_poison flag
set. There are also two comparisons that check if either of the
inputs the ctlzs are zero. We need to use a logical or to block
the poison from the ctlz if either of the inputs is zero.

[nikic]

llvm/lib/Transforms/Utils/IntegerDivision.cpp

217

I think you actually need to freeze the inputs here. Otherwise an undef value could be picked to be non-zero here and zero in the ctlz call.

[craig.topper]

Add Freeze

[nikic]

llvm/lib/Transforms/Utils/IntegerDivision.cpp

233

I don't think we need this logical or, because both inputs are non-poison with the freeze.

238

We *do* need this one, because Ret0_4 may be poison (through SR).

241

We also need this one, because RetDividend may be poison (through SR).

[aqjune]

llvm/lib/Transforms/Utils/IntegerDivision.cpp

136

Should Dividend and Divisor here be frozen to address the comment?

[craig.topper]

llvm/lib/Transforms/Utils/IntegerDivision.cpp

136

I don't understand the question. I froze them on line 256 and 257. Do you think they should be frozen earlier int he function or in a different basic block?

[arsenm]

llvm/test/CodeGen/AMDGPU/srem64.ll

1023–1028

These are all small improvements

[nikic]

There are some failing unit tests.

[aqjune]

llvm/lib/Transforms/Utils/IntegerDivision.cpp

136

Oops, I missed the updates at 256, 257, sorry for this.

[craig.topper]

Change one CreateLogicalOr back to CreateOr.
Update unit test I missed before

[craig.topper]

Rebase

llvm/lib/Transforms/Utils/IntegerDivision.cpp

static Value *generateSignedRemainderCode(Value *Dividend, Value *Divisor,
   // ;   %divisor_sgn  = ashr i32 %divisor, 31
   // ;   %dvd_xor      = xor i32 %dividend, %dividend_sgn
   // ;   %dvs_xor      = xor i32 %divisor, %divisor_sgn
   // ;   %u_dividend   = sub i32 %dvd_xor, %dividend_sgn
   // ;   %u_divisor    = sub i32 %dvs_xor, %divisor_sgn
   // ;   %urem         = urem i32 %dividend, %divisor
   // ;   %xored        = xor i32 %urem, %dividend_sgn
   // ;   %srem         = sub i32 %xored, %dividend_sgn
+  Dividend            = Builder.CreateFreeze(Dividend);
+  Divisor             = Builder.CreateFreeze(Divisor);
   Value *DividendSign = Builder.CreateAShr(Dividend, Shift);
   Value *DivisorSign  = Builder.CreateAShr(Divisor, Shift);
   Value *DvdXor       = Builder.CreateXor(Dividend, DividendSign);
   Value *DvsXor       = Builder.CreateXor(Divisor, DivisorSign);
   Value *UDividend    = Builder.CreateSub(DvdXor, DividendSign);
   Value *UDivisor     = Builder.CreateSub(DvsXor, DivisorSign);
   Value *URem         = Builder.CreateURem(UDividend, UDivisor);
   Value *Xored        = Builder.CreateXor(URem, DividendSign);
static Value *generatedUnsignedRemainderCode(Value *Dividend, Value *Divisor,
                                              IRBuilder<> &Builder) {
   // Remainder = Dividend - Quotient*Divisor
 
   // Following instructions are generated for both i32 and i64
 
   // ;   %quotient  = udiv i32 %dividend, %divisor
   // ;   %product   = mul i32 %divisor, %quotient
   // ;   %remainder = sub i32 %dividend, %product
+  Dividend         = Builder.CreateFreeze(Dividend);
+  Divisor          = Builder.CreateFreeze(Divisor);
   Value *Quotient  = Builder.CreateUDiv(Dividend, Divisor);
   Value *Product   = Builder.CreateMul(Divisor, Quotient);
   Value *Remainder = Builder.CreateSub(Dividend, Product);
 
   if (Instruction *UDiv = dyn_cast<Instruction>(Quotient))
     Builder.SetInsertPoint(UDiv);
 
   return Remainder;
static Value *generateSignedDivisionCode(Value *Dividend, Value *Divisor,
   // ;   %tmp2   = xor i32 %tmp, %dividend
   // ;   %u_dvnd = sub nsw i32 %tmp2, %tmp
   // ;   %tmp3   = xor i32 %tmp1, %divisor
   // ;   %u_dvsr = sub nsw i32 %tmp3, %tmp1
   // ;   %q_sgn  = xor i32 %tmp1, %tmp
   // ;   %q_mag  = udiv i32 %u_dvnd, %u_dvsr
   // ;   %tmp4   = xor i32 %q_mag, %q_sgn
   // ;   %q      = sub i32 %tmp4, %q_sgn
+  Dividend      = Builder.CreateFreeze(Dividend);
+  Divisor       = Builder.CreateFreeze(Divisor);
   Value *Tmp    = Builder.CreateAShr(Dividend, Shift);
   Value *Tmp1   = Builder.CreateAShr(Divisor, Shift);
   Value *Tmp2   = Builder.CreateXor(Tmp, Dividend);
   Value *U_Dvnd = Builder.CreateSub(Tmp2, Tmp);
   Value *Tmp3   = Builder.CreateXor(Tmp1, Divisor);
   Value *U_Dvsr = Builder.CreateSub(Tmp3, Tmp1);
   Value *Q_Sgn  = Builder.CreateXor(Tmp1, Tmp);
   Value *Q_Mag  = Builder.CreateUDiv(U_Dvnd, U_Dvsr);
   Value *Tmp4   = Builder.CreateXor(Q_Mag, Q_Sgn);
   Value *Q      = Builder.CreateSub(Tmp4, Q_Sgn);
 
   if (Instruction *UDiv = dyn_cast<Instruction>(Q_Mag))
     Builder.SetInsertPoint(UDiv);
 
   return Q;
 }
 
 /// Generates code to divide two unsigned scalar 32-bit or 64-bit integers.
 /// Returns the quotient, rounded towards 0. Builder's insert point should
 /// point where the caller wants code generated, e.g. at the udiv instruction.
 static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
                                            IRBuilder<> &Builder) {
   // The basic algorithm can be found in the compiler-rt project's
   // implementation of __udivsi3.c. Here, we do a lower-level IR based approach
   // that's been hand-tuned to lessen the amount of control flow involved.
 
   // Some helper values
   IntegerType *DivTy = cast<IntegerType>(Dividend->getType());
   unsigned BitWidth = DivTy->getBitWidth();
 
static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
   SpecialCases->getTerminator()->eraseFromParent();
 
   // Same instructions are generated for both i32 (msb 31) and i64 (msb 63).
 
   // First off, check for special cases: dividend or divisor is zero, divisor
   // is greater than dividend, and divisor is 1.
   // ; special-cases:
   // ;   %ret0_1      = icmp eq i32 %divisor, 0
   // ;   %ret0_2      = icmp eq i32 %dividend, 0
   // ;   %ret0_3      = or i1 %ret0_1, %ret0_2
   // ;   %tmp0        = tail call i32 @llvm.ctlz.i32(i32 %divisor, i1 true)
   // ;   %tmp1        = tail call i32 @llvm.ctlz.i32(i32 %dividend, i1 true)
   // ;   %sr          = sub nsw i32 %tmp0, %tmp1
   // ;   %ret0_4      = icmp ugt i32 %sr, 31
   // ;   %ret0        = select i1 %ret0_3, i1 true, i1 %ret0_4
   // ;   %retDividend = icmp eq i32 %sr, 31
   // ;   %retVal      = select i1 %ret0, i32 0, i32 %dividend
   // ;   %earlyRet    = select i1 %ret0, i1 true, %retDividend
   // ;   br i1 %earlyRet, label %end, label %bb1
   Builder.SetInsertPoint(SpecialCases);
+  Divisor            = Builder.CreateFreeze(Divisor);
+  Dividend           = Builder.CreateFreeze(Dividend);
   Value *Ret0_1      = Builder.CreateICmpEQ(Divisor, Zero);
   Value *Ret0_2      = Builder.CreateICmpEQ(Dividend, Zero);
   Value *Ret0_3      = Builder.CreateOr(Ret0_1, Ret0_2);
   Value *Tmp0 = Builder.CreateCall(CTLZ, {Divisor, True});
   Value *Tmp1 = Builder.CreateCall(CTLZ, {Dividend, True});
   Value *SR          = Builder.CreateSub(Tmp0, Tmp1);
   Value *Ret0_4      = Builder.CreateICmpUGT(SR, MSB);
   Value *Ret0        = Builder.CreateLogicalOr(Ret0_3, Ret0_4);
   Value *RetDividend = Builder.CreateICmpEQ(SR, MSB);
   Value *RetVal      = Builder.CreateSelect(Ret0, Zero, Dividend);
   Value *EarlyRet    = Builder.CreateLogicalOr(Ret0, RetDividend);
   Builder.CreateCondBr(EarlyRet, End, BB1);
 
   // ; bb1:                                             ; preds = %special-cases
   // ;   %sr_1     = add i32 %sr, 1
   // ;   %tmp2     = sub i32 31, %sr
   // ;   %q        = shl i32 %dividend, %tmp2
   // ;   %skipLoop = icmp eq i32 %sr_1, 0
   // ;   br i1 %skipLoop, label %loop-exit, label %preheader

llvm/test/CodeGen/AMDGPU/sdiv64.ll

 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    buffer_store_short v1, off, s[0:3], 0 offset:4
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_sdiv24_48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s7
-; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
-; GCN-IR-NEXT:    s_sext_i32_i16 s3, s3
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 24
+; GCN-IR-NEXT:    s_sext_i32_i16 s1, s1
+; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[6:7], 24
+; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[0:1], 24
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[2:3], 16
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[0:1], 16
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 16
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
-; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[2:3], 24
-; GCN-IR-NEXT:    s_ashr_i32 s2, s3, 31
+; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[6:7], 16
+; GCN-IR-NEXT:    s_ashr_i32 s2, s7, 31
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[0:1], s[8:9]
 ; GCN-IR-NEXT:    s_mov_b32 s3, s2
 ; GCN-IR-NEXT:    s_sub_u32 s12, s6, s0
 ; GCN-IR-NEXT:    s_subb_u32 s13, s7, s0
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[2:3], s[10:11]
 ; GCN-IR-NEXT:    s_sub_u32 s6, s6, s2
 ; GCN-IR-NEXT:    s_subb_u32 s7, s7, s2
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[6:7], 0

llvm/test/CodeGen/AMDGPU/srem64.ll

 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem33_64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[6:7], 31
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[0:1], 31
-; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
-; GCN-IR-NEXT:    s_ashr_i32 s6, s1, 31
+; GCN-IR-NEXT:    s_ashr_i64 s[6:7], s[0:1], 31
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    s_ashr_i32 s10, s7, 31
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
-; GCN-IR-NEXT:    s_mov_b32 s7, s6
+; GCN-IR-NEXT:    s_mov_b32 s11, s10
 ; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[2:3], s[0:1]
-; GCN-IR-NEXT:    s_xor_b64 s[8:9], s[8:9], s[6:7]
+; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[6:7], s[10:11]
 ; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
 ; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
-; GCN-IR-NEXT:    s_sub_u32 s8, s8, s6
-; GCN-IR-NEXT:    s_subb_u32 s9, s9, s6
+; GCN-IR-NEXT:    s_sub_u32 s8, s6, s10
+; GCN-IR-NEXT:    s_subb_u32 s9, s7, s10
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[8:9], 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[2:3], 0
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
 ; GCN-IR-NEXT:    s_or_b64 s[14:15], s[10:11], s[12:13]
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s8
 ; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s9
 ; GCN-IR-NEXT:    s_min_u32 s12, s10, s11
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
 ; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
 ; GCN-IR-NEXT:    s_min_u32 s16, s10, s11
 ; GCN-IR-NEXT:    s_sub_u32 s10, s12, s16
 ; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[10:11], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[20:21], s[10:11], 63
 ; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[18:19]
 ; GCN-IR-NEXT:    s_or_b64 s[18:19], s[14:15], s[20:21]
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_mov_b32 s13, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s14, s10, 1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s11, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[14:15], 0
 ; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[2:3], s10
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_4

 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GCN-NEXT:    buffer_store_short v1, off, s[0:3], 0 offset:4
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_srem24_48:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-IR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_sext_i32_i16 s7, s7
-; GCN-IR-NEXT:    s_sext_i32_i16 s3, s3
-; GCN-IR-NEXT:    s_ashr_i32 s0, s7, 31
-; GCN-IR-NEXT:    s_ashr_i32 s12, s3, 31
-; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[6:7], 24
-; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[2:3], 24
+; GCN-IR-NEXT:    s_sext_i32_i16 s1, s1
+; GCN-IR-NEXT:    s_ashr_i64 s[2:3], s[6:7], 24
+; GCN-IR-NEXT:    s_ashr_i64 s[0:1], s[0:1], 24
+; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[2:3], 16
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[0:1], 16
+; GCN-IR-NEXT:    s_ashr_i32 s0, s3, 31
+; GCN-IR-NEXT:    s_ashr_i32 s12, s7, 31
+; GCN-IR-NEXT:    s_ashr_i64 s[8:9], s[2:3], 16
+; GCN-IR-NEXT:    s_ashr_i64 s[10:11], s[6:7], 16
 ; GCN-IR-NEXT:    s_mov_b32 s1, s0
 ; GCN-IR-NEXT:    s_mov_b32 s13, s12
 ; GCN-IR-NEXT:    s_xor_b64 s[2:3], s[8:9], s[0:1]
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[10:11], s[12:13]
 ; GCN-IR-NEXT:    s_sub_u32 s2, s2, s0
 ; GCN-IR-NEXT:    s_subb_u32 s3, s3, s0
 ; GCN-IR-NEXT:    s_sub_u32 s6, s6, s12
 ; GCN-IR-NEXT:    s_subb_u32 s7, s7, s12

llvm/test/CodeGen/AMDGPU/udiv64.ll

 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-IR-NEXT:    s_and_b32 s3, s7, 0xffff
 ; GCN-IR-NEXT:    s_and_b32 s2, s6, 0xff000000
 ; GCN-IR-NEXT:    s_and_b32 s1, s1, 0xffff
 ; GCN-IR-NEXT:    s_and_b32 s0, s0, 0xff000000
 ; GCN-IR-NEXT:    s_lshr_b64 s[8:9], s[2:3], 24
-; GCN-IR-NEXT:    s_lshr_b64 s[2:3], s[0:1], 24
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
+; GCN-IR-NEXT:    s_lshr_b64 s[0:1], s[0:1], 24
+; GCN-IR-NEXT:    s_and_b32 s9, s9, 0xffff
+; GCN-IR-NEXT:    s_and_b32 s1, s1, 0xffff
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[0:1], 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[8:9], 0
-; GCN-IR-NEXT:    s_mov_b64 s[0:1], 0
+; GCN-IR-NEXT:    s_mov_b64 s[2:3], 0
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[6:7], s[10:11]
-; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
+; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s0
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
-; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
+; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s1
 ; GCN-IR-NEXT:    s_min_u32 s10, s6, s7
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s8
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s9
 ; GCN-IR-NEXT:    s_min_u32 s14, s6, s7
 ; GCN-IR-NEXT:    s_sub_u32 s6, s10, s14
 ; GCN-IR-NEXT:    s_subb_u32 s7, 0, 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[6:7], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[6:7], 63
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[16:17]
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[12:13], s[18:19]
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
 ; GCN-IR-NEXT:    s_add_u32 s12, s6, 1
 ; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[12:13], 0
 ; GCN-IR-NEXT:    s_sub_i32 s6, 63, s6
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[8:9], s6
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[8:9], s12
-; GCN-IR-NEXT:    s_add_u32 s15, s2, -1
-; GCN-IR-NEXT:    s_addc_u32 s16, s3, -1
-; GCN-IR-NEXT:    s_not_b64 s[0:1], s[10:11]
-; GCN-IR-NEXT:    s_add_u32 s8, s0, s14
-; GCN-IR-NEXT:    s_addc_u32 s9, s1, 0
+; GCN-IR-NEXT:    s_add_u32 s15, s0, -1
+; GCN-IR-NEXT:    s_addc_u32 s16, s1, -1
+; GCN-IR-NEXT:    s_not_b64 s[2:3], s[10:11]
+; GCN-IR-NEXT:    s_add_u32 s8, s2, s14
+; GCN-IR-NEXT:    s_addc_u32 s9, s3, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    s_mov_b32 s1, 0
+; GCN-IR-NEXT:    s_mov_b32 s3, 0
 ; GCN-IR-NEXT:  .LBB7_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GCN-IR-NEXT:    s_lshl_b64 s[12:13], s[12:13], 1
-; GCN-IR-NEXT:    s_lshr_b32 s0, s7, 31
+; GCN-IR-NEXT:    s_lshr_b32 s2, s7, 31
 ; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[6:7], 1
-; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[0:1]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[2:3]
 ; GCN-IR-NEXT:    s_or_b64 s[6:7], s[10:11], s[6:7]
-; GCN-IR-NEXT:    s_sub_u32 s0, s15, s12
-; GCN-IR-NEXT:    s_subb_u32 s0, s16, s13
-; GCN-IR-NEXT:    s_ashr_i32 s10, s0, 31
+; GCN-IR-NEXT:    s_sub_u32 s2, s15, s12
+; GCN-IR-NEXT:    s_subb_u32 s2, s16, s13
+; GCN-IR-NEXT:    s_ashr_i32 s10, s2, 31
 ; GCN-IR-NEXT:    s_mov_b32 s11, s10
-; GCN-IR-NEXT:    s_and_b32 s0, s10, 1
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], s[2:3]
+; GCN-IR-NEXT:    s_and_b32 s2, s10, 1
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[10:11], s[0:1]
 ; GCN-IR-NEXT:    s_sub_u32 s12, s12, s10
 ; GCN-IR-NEXT:    s_subb_u32 s13, s13, s11
 ; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
 ; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[8:9], 0
-; GCN-IR-NEXT:    s_mov_b64 s[10:11], s[0:1]
+; GCN-IR-NEXT:    s_mov_b64 s[10:11], s[2:3]
 ; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_3
 ; GCN-IR-NEXT:  .LBB7_4: ; %Flow3
-; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[6:7], 1
-; GCN-IR-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-IR-NEXT:    s_lshl_b64 s[0:1], s[6:7], 1
+; GCN-IR-NEXT:    s_or_b64 s[0:1], s[2:3], s[0:1]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-IR-NEXT:    s_branch .LBB7_6
 ; GCN-IR-NEXT:  .LBB7_5:
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s9
 ; GCN-IR-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[12:13]
 ; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
 ; GCN-IR-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[12:13]

llvm/test/CodeGen/X86/expand-large-div-rem-sdiv129.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-- -expand-large-div-rem -expand-div-rem-bits 128 < %s | FileCheck %s
 
 define void @sdiv129(i129* %ptr, i129* %out) nounwind {
 ; CHECK-LABEL: @sdiv129(
 ; CHECK-NEXT:  _udiv-special-cases:
 ; CHECK-NEXT:    [[A:%.*]] = load i129, i129* [[PTR:%.*]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i129 [[TMP0]], [[A]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i129 [[TMP1]], [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = xor i129 0, [[TMP0]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i129 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = or i1 false, [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i129 @llvm.ctlz.i129(i129 3, i1 true)
-; CHECK-NEXT:    [[TMP7:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP2]], i1 true)
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i129 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp ugt i129 [[TMP8]], 128
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP5]], i1 true, i1 [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i129 [[TMP8]], 128
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP10]], i129 0, i129 [[TMP2]]
-; CHECK-NEXT:    [[TMP13:%.*]] = select i1 [[TMP10]], i1 true, i1 [[TMP11]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i129 [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i129 3
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i129 [[TMP0]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i129 [[TMP1]], 128
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i129 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i129 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i129 [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i129 [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i129 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = freeze i129 [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = freeze i129 [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i129 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i129 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP9]], i1 true)
+; CHECK-NEXT:    [[TMP15:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP10]], i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i129 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ugt i129 [[TMP16]], 128
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP13]], i1 true, i1 [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i129 [[TMP16]], 128
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP18]], i129 0, i129 [[TMP10]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP18]], i1 true, i1 [[TMP19]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
 ; CHECK:       udiv-loop-exit:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP29:%.*]], [[UDIV_DO_WHILE:%.*]] ]
-; CHECK-NEXT:    [[TMP15:%.*]] = phi i129 [ [[TMP37:%.*]], [[UDIV_BB1]] ], [ [[TMP26:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = shl i129 [[TMP15]], 1
-; CHECK-NEXT:    [[TMP17:%.*]] = or i129 [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP37:%.*]], [[UDIV_DO_WHILE:%.*]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i129 [ [[TMP46:%.*]], [[UDIV_BB1]] ], [ [[TMP34:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = shl i129 [[TMP23]], 1
+; CHECK-NEXT:    [[TMP25:%.*]] = or i129 [[TMP22]], [[TMP24]]
 ; CHECK-NEXT:    br label [[UDIV_END]]
 ; CHECK:       udiv-do-while:
-; CHECK-NEXT:    [[TMP18:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP29]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP19:%.*]] = phi i129 [ [[TMP35:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP32:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP20:%.*]] = phi i129 [ [[TMP34:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP31:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP21:%.*]] = phi i129 [ [[TMP37]], [[UDIV_PREHEADER]] ], [ [[TMP26]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = shl i129 [[TMP20]], 1
-; CHECK-NEXT:    [[TMP23:%.*]] = lshr i129 [[TMP21]], 128
-; CHECK-NEXT:    [[TMP24:%.*]] = or i129 [[TMP22]], [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = shl i129 [[TMP21]], 1
-; CHECK-NEXT:    [[TMP26]] = or i129 [[TMP18]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = sub i129 2, [[TMP24]]
-; CHECK-NEXT:    [[TMP28:%.*]] = ashr i129 [[TMP27]], 128
-; CHECK-NEXT:    [[TMP29]] = and i129 [[TMP28]], 1
-; CHECK-NEXT:    [[TMP30:%.*]] = and i129 [[TMP28]], 3
-; CHECK-NEXT:    [[TMP31]] = sub i129 [[TMP24]], [[TMP30]]
-; CHECK-NEXT:    [[TMP32]] = add i129 [[TMP19]], -1
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp eq i129 [[TMP32]], 0
-; CHECK-NEXT:    br i1 [[TMP33]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
+; CHECK-NEXT:    [[TMP26:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP37]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = phi i129 [ [[TMP44:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP40:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP28:%.*]] = phi i129 [ [[TMP42:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP39:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP29:%.*]] = phi i129 [ [[TMP46]], [[UDIV_PREHEADER]] ], [ [[TMP34]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP30:%.*]] = shl i129 [[TMP28]], 1
+; CHECK-NEXT:    [[TMP31:%.*]] = lshr i129 [[TMP29]], 128
+; CHECK-NEXT:    [[TMP32:%.*]] = or i129 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = shl i129 [[TMP29]], 1
+; CHECK-NEXT:    [[TMP34]] = or i129 [[TMP26]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = sub i129 [[TMP43:%.*]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = ashr i129 [[TMP35]], 128
+; CHECK-NEXT:    [[TMP37]] = and i129 [[TMP36]], 1
+; CHECK-NEXT:    [[TMP38:%.*]] = and i129 [[TMP36]], [[TMP9]]
+; CHECK-NEXT:    [[TMP39]] = sub i129 [[TMP32]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40]] = add i129 [[TMP27]], -1
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp eq i129 [[TMP40]], 0
+; CHECK-NEXT:    br i1 [[TMP41]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-preheader:
-; CHECK-NEXT:    [[TMP34]] = lshr i129 [[TMP2]], [[TMP35]]
+; CHECK-NEXT:    [[TMP42]] = lshr i129 [[TMP10]], [[TMP44]]
+; CHECK-NEXT:    [[TMP43]] = add i129 [[TMP9]], -1
 ; CHECK-NEXT:    br label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-bb1:
-; CHECK-NEXT:    [[TMP35]] = add i129 [[TMP8]], 1
-; CHECK-NEXT:    [[TMP36:%.*]] = sub i129 128, [[TMP8]]
-; CHECK-NEXT:    [[TMP37]] = shl i129 [[TMP2]], [[TMP36]]
-; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i129 [[TMP35]], 0
-; CHECK-NEXT:    br i1 [[TMP38]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
+; CHECK-NEXT:    [[TMP44]] = add i129 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP45:%.*]] = sub i129 128, [[TMP16]]
+; CHECK-NEXT:    [[TMP46]] = shl i129 [[TMP10]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp eq i129 [[TMP44]], 0
+; CHECK-NEXT:    br i1 [[TMP47]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
 ; CHECK:       udiv-end:
-; CHECK-NEXT:    [[TMP39:%.*]] = phi i129 [ [[TMP17]], [[UDIV_LOOP_EXIT]] ], [ [[TMP12]], [[_UDIV_SPECIAL_CASES:%.*]] ]
-; CHECK-NEXT:    [[TMP40:%.*]] = xor i129 [[TMP39]], [[TMP3]]
-; CHECK-NEXT:    [[TMP41:%.*]] = sub i129 [[TMP40]], [[TMP3]]
-; CHECK-NEXT:    store i129 [[TMP41]], i129* [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[TMP48:%.*]] = phi i129 [ [[TMP25]], [[UDIV_LOOP_EXIT]] ], [ [[TMP20]], [[_UDIV_SPECIAL_CASES:%.*]] ]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor i129 [[TMP48]], [[TMP8]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sub i129 [[TMP49]], [[TMP8]]
+; CHECK-NEXT:    store i129 [[TMP50]], i129* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %a = load i129, i129* %ptr
   %res = sdiv i129 %a, 3
   store i129 %res, i129* %out
   ret void
 }

llvm/test/CodeGen/X86/expand-large-div-rem-srem129.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-- -expand-large-div-rem -expand-div-rem-bits 128 < %s | FileCheck %s
 
 define void @test(i129* %ptr, i129* %out) nounwind {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  _udiv-special-cases:
 ; CHECK-NEXT:    [[A:%.*]] = load i129, i129* [[PTR:%.*]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i129 [[A]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i129 [[TMP1]], [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i129 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = or i1 false, [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i129 @llvm.ctlz.i129(i129 3, i1 true)
-; CHECK-NEXT:    [[TMP6:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP2]], i1 true)
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i129 [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i129 [[TMP7]], 128
-; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP4]], i1 true, i1 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i129 [[TMP7]], 128
-; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP9]], i129 0, i129 [[TMP2]]
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP9]], i1 true, i1 [[TMP10]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i129 [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i129 3
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i129 [[TMP0]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i129 [[TMP1]], 128
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i129 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i129 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i129 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i129 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = freeze i129 [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = freeze i129 [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = freeze i129 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = freeze i129 [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i129 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i129 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP10]], i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP11]], i1 true)
+; CHECK-NEXT:    [[TMP17:%.*]] = sub i129 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ugt i129 [[TMP17]], 128
+; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP14]], i1 true, i1 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i129 [[TMP17]], 128
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP19]], i129 0, i129 [[TMP11]]
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP19]], i1 true, i1 [[TMP20]]
+; CHECK-NEXT:    br i1 [[TMP22]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
 ; CHECK:       udiv-loop-exit:
-; CHECK-NEXT:    [[TMP13:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP28:%.*]], [[UDIV_DO_WHILE:%.*]] ]
-; CHECK-NEXT:    [[TMP14:%.*]] = phi i129 [ [[TMP36:%.*]], [[UDIV_BB1]] ], [ [[TMP25:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP15:%.*]] = shl i129 [[TMP14]], 1
-; CHECK-NEXT:    [[TMP16:%.*]] = or i129 [[TMP13]], [[TMP15]]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP38:%.*]], [[UDIV_DO_WHILE:%.*]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = phi i129 [ [[TMP47:%.*]], [[UDIV_BB1]] ], [ [[TMP35:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = shl i129 [[TMP24]], 1
+; CHECK-NEXT:    [[TMP26:%.*]] = or i129 [[TMP23]], [[TMP25]]
 ; CHECK-NEXT:    br label [[UDIV_END]]
 ; CHECK:       udiv-do-while:
-; CHECK-NEXT:    [[TMP17:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP28]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP18:%.*]] = phi i129 [ [[TMP34:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP31:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP19:%.*]] = phi i129 [ [[TMP33:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP30:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP20:%.*]] = phi i129 [ [[TMP36]], [[UDIV_PREHEADER]] ], [ [[TMP25]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP21:%.*]] = shl i129 [[TMP19]], 1
-; CHECK-NEXT:    [[TMP22:%.*]] = lshr i129 [[TMP20]], 128
-; CHECK-NEXT:    [[TMP23:%.*]] = or i129 [[TMP21]], [[TMP22]]
-; CHECK-NEXT:    [[TMP24:%.*]] = shl i129 [[TMP20]], 1
-; CHECK-NEXT:    [[TMP25]] = or i129 [[TMP17]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = sub i129 2, [[TMP23]]
-; CHECK-NEXT:    [[TMP27:%.*]] = ashr i129 [[TMP26]], 128
-; CHECK-NEXT:    [[TMP28]] = and i129 [[TMP27]], 1
-; CHECK-NEXT:    [[TMP29:%.*]] = and i129 [[TMP27]], 3
-; CHECK-NEXT:    [[TMP30]] = sub i129 [[TMP23]], [[TMP29]]
-; CHECK-NEXT:    [[TMP31]] = add i129 [[TMP18]], -1
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i129 [[TMP31]], 0
-; CHECK-NEXT:    br i1 [[TMP32]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
+; CHECK-NEXT:    [[TMP27:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP38]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP28:%.*]] = phi i129 [ [[TMP45:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP41:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP29:%.*]] = phi i129 [ [[TMP43:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP40:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP30:%.*]] = phi i129 [ [[TMP47]], [[UDIV_PREHEADER]] ], [ [[TMP35]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP31:%.*]] = shl i129 [[TMP29]], 1
+; CHECK-NEXT:    [[TMP32:%.*]] = lshr i129 [[TMP30]], 128
+; CHECK-NEXT:    [[TMP33:%.*]] = or i129 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = shl i129 [[TMP30]], 1
+; CHECK-NEXT:    [[TMP35]] = or i129 [[TMP27]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = sub i129 [[TMP44:%.*]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = ashr i129 [[TMP36]], 128
+; CHECK-NEXT:    [[TMP38]] = and i129 [[TMP37]], 1
+; CHECK-NEXT:    [[TMP39:%.*]] = and i129 [[TMP37]], [[TMP10]]
+; CHECK-NEXT:    [[TMP40]] = sub i129 [[TMP33]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41]] = add i129 [[TMP28]], -1
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp eq i129 [[TMP41]], 0
+; CHECK-NEXT:    br i1 [[TMP42]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-preheader:
-; CHECK-NEXT:    [[TMP33]] = lshr i129 [[TMP2]], [[TMP34]]
+; CHECK-NEXT:    [[TMP43]] = lshr i129 [[TMP11]], [[TMP45]]
+; CHECK-NEXT:    [[TMP44]] = add i129 [[TMP10]], -1
 ; CHECK-NEXT:    br label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-bb1:
-; CHECK-NEXT:    [[TMP34]] = add i129 [[TMP7]], 1
-; CHECK-NEXT:    [[TMP35:%.*]] = sub i129 128, [[TMP7]]
-; CHECK-NEXT:    [[TMP36]] = shl i129 [[TMP2]], [[TMP35]]
-; CHECK-NEXT:    [[TMP37:%.*]] = icmp eq i129 [[TMP34]], 0
-; CHECK-NEXT:    br i1 [[TMP37]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
+; CHECK-NEXT:    [[TMP45]] = add i129 [[TMP17]], 1
+; CHECK-NEXT:    [[TMP46:%.*]] = sub i129 128, [[TMP17]]
+; CHECK-NEXT:    [[TMP47]] = shl i129 [[TMP11]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp eq i129 [[TMP45]], 0
+; CHECK-NEXT:    br i1 [[TMP48]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
 ; CHECK:       udiv-end:
-; CHECK-NEXT:    [[TMP38:%.*]] = phi i129 [ [[TMP16]], [[UDIV_LOOP_EXIT]] ], [ [[TMP11]], [[_UDIV_SPECIAL_CASES:%.*]] ]
-; CHECK-NEXT:    [[TMP39:%.*]] = mul i129 3, [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = sub i129 [[TMP2]], [[TMP39]]
-; CHECK-NEXT:    [[TMP41:%.*]] = xor i129 [[TMP40]], [[TMP0]]
-; CHECK-NEXT:    [[TMP42:%.*]] = sub i129 [[TMP41]], [[TMP0]]
-; CHECK-NEXT:    store i129 [[TMP42]], i129* [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[TMP49:%.*]] = phi i129 [ [[TMP26]], [[UDIV_LOOP_EXIT]] ], [ [[TMP21]], [[_UDIV_SPECIAL_CASES:%.*]] ]
+; CHECK-NEXT:    [[TMP50:%.*]] = mul i129 [[TMP9]], [[TMP49]]
+; CHECK-NEXT:    [[TMP51:%.*]] = sub i129 [[TMP8]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = xor i129 [[TMP51]], [[TMP2]]
+; CHECK-NEXT:    [[TMP53:%.*]] = sub i129 [[TMP52]], [[TMP2]]
+; CHECK-NEXT:    store i129 [[TMP53]], i129* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %a = load i129, i129* %ptr
   %res = srem i129 %a, 3
   store i129 %res, i129* %out
   ret void
 }

llvm/test/CodeGen/X86/expand-large-div-rem-udiv129.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-- -expand-large-div-rem -expand-div-rem-bits 128 < %s | FileCheck %s
 
 define void @test(i129* %ptr, i129* %out) nounwind {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  _udiv-special-cases:
 ; CHECK-NEXT:    [[A:%.*]] = load i129, i129* [[PTR:%.*]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i129 [[A]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = or i1 false, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i129 @llvm.ctlz.i129(i129 3, i1 true)
-; CHECK-NEXT:    [[TMP3:%.*]] = call i129 @llvm.ctlz.i129(i129 [[A]], i1 true)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i129 [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ugt i129 [[TMP4]], 128
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP1]], i1 true, i1 [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i129 [[TMP4]], 128
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP6]], i129 0, i129 [[A]]
-; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP6]], i1 true, i1 [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i129 3
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i129 [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i129 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i129 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP0]], i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP1]], i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i129 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i129 [[TMP7]], 128
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP4]], i1 true, i1 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i129 [[TMP7]], 128
+; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP9]], i129 0, i129 [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP9]], i1 true, i1 [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
 ; CHECK:       udiv-loop-exit:
-; CHECK-NEXT:    [[TMP10:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP25:%.*]], [[UDIV_DO_WHILE:%.*]] ]
-; CHECK-NEXT:    [[TMP11:%.*]] = phi i129 [ [[TMP33:%.*]], [[UDIV_BB1]] ], [ [[TMP22:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = shl i129 [[TMP11]], 1
-; CHECK-NEXT:    [[TMP13:%.*]] = or i129 [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP28:%.*]], [[UDIV_DO_WHILE:%.*]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi i129 [ [[TMP37:%.*]], [[UDIV_BB1]] ], [ [[TMP25:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP15:%.*]] = shl i129 [[TMP14]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = or i129 [[TMP13]], [[TMP15]]
 ; CHECK-NEXT:    br label [[UDIV_END]]
 ; CHECK:       udiv-do-while:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP25]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP15:%.*]] = phi i129 [ [[TMP31:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP28:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = phi i129 [ [[TMP30:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP27:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = phi i129 [ [[TMP33]], [[UDIV_PREHEADER]] ], [ [[TMP22]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP18:%.*]] = shl i129 [[TMP16]], 1
-; CHECK-NEXT:    [[TMP19:%.*]] = lshr i129 [[TMP17]], 128
-; CHECK-NEXT:    [[TMP20:%.*]] = or i129 [[TMP18]], [[TMP19]]
-; CHECK-NEXT:    [[TMP21:%.*]] = shl i129 [[TMP17]], 1
-; CHECK-NEXT:    [[TMP22]] = or i129 [[TMP14]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = sub i129 2, [[TMP20]]
-; CHECK-NEXT:    [[TMP24:%.*]] = ashr i129 [[TMP23]], 128
-; CHECK-NEXT:    [[TMP25]] = and i129 [[TMP24]], 1
-; CHECK-NEXT:    [[TMP26:%.*]] = and i129 [[TMP24]], 3
-; CHECK-NEXT:    [[TMP27]] = sub i129 [[TMP20]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28]] = add i129 [[TMP15]], -1
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i129 [[TMP28]], 0
-; CHECK-NEXT:    br i1 [[TMP29]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
+; CHECK-NEXT:    [[TMP17:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP28]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = phi i129 [ [[TMP35:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP31:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP19:%.*]] = phi i129 [ [[TMP33:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP30:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = phi i129 [ [[TMP37]], [[UDIV_PREHEADER]] ], [ [[TMP25]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = shl i129 [[TMP19]], 1
+; CHECK-NEXT:    [[TMP22:%.*]] = lshr i129 [[TMP20]], 128
+; CHECK-NEXT:    [[TMP23:%.*]] = or i129 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = shl i129 [[TMP20]], 1
+; CHECK-NEXT:    [[TMP25]] = or i129 [[TMP17]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = sub i129 [[TMP34:%.*]], [[TMP23]]
+; CHECK-NEXT:    [[TMP27:%.*]] = ashr i129 [[TMP26]], 128
+; CHECK-NEXT:    [[TMP28]] = and i129 [[TMP27]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = and i129 [[TMP27]], [[TMP0]]
+; CHECK-NEXT:    [[TMP30]] = sub i129 [[TMP23]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31]] = add i129 [[TMP18]], -1
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i129 [[TMP31]], 0
+; CHECK-NEXT:    br i1 [[TMP32]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-preheader:
-; CHECK-NEXT:    [[TMP30]] = lshr i129 [[A]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33]] = lshr i129 [[TMP1]], [[TMP35]]
+; CHECK-NEXT:    [[TMP34]] = add i129 [[TMP0]], -1
 ; CHECK-NEXT:    br label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-bb1:
-; CHECK-NEXT:    [[TMP31]] = add i129 [[TMP4]], 1
-; CHECK-NEXT:    [[TMP32:%.*]] = sub i129 128, [[TMP4]]
-; CHECK-NEXT:    [[TMP33]] = shl i129 [[A]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i129 [[TMP31]], 0
-; CHECK-NEXT:    br i1 [[TMP34]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
+; CHECK-NEXT:    [[TMP35]] = add i129 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP36:%.*]] = sub i129 128, [[TMP7]]
+; CHECK-NEXT:    [[TMP37]] = shl i129 [[TMP1]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i129 [[TMP35]], 0
+; CHECK-NEXT:    br i1 [[TMP38]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
 ; CHECK:       udiv-end:
-; CHECK-NEXT:    [[TMP35:%.*]] = phi i129 [ [[TMP13]], [[UDIV_LOOP_EXIT]] ], [ [[TMP8]], [[_UDIV_SPECIAL_CASES:%.*]] ]
-; CHECK-NEXT:    store i129 [[TMP35]], i129* [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[TMP39:%.*]] = phi i129 [ [[TMP16]], [[UDIV_LOOP_EXIT]] ], [ [[TMP11]], [[_UDIV_SPECIAL_CASES:%.*]] ]
+; CHECK-NEXT:    store i129 [[TMP39]], i129* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %a = load i129, i129* %ptr
   %res = udiv i129 %a, 3
   store i129 %res, i129* %out
   ret void
 }

llvm/test/CodeGen/X86/expand-large-div-rem-urem129.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-- -expand-large-div-rem -expand-div-rem-bits 128 < %s | FileCheck %s
 
 define void @test(i129* %ptr, i129* %out) nounwind {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  _udiv-special-cases:
 ; CHECK-NEXT:    [[A:%.*]] = load i129, i129* [[PTR:%.*]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i129 [[A]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = or i1 false, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i129 @llvm.ctlz.i129(i129 3, i1 true)
-; CHECK-NEXT:    [[TMP3:%.*]] = call i129 @llvm.ctlz.i129(i129 [[A]], i1 true)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i129 [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ugt i129 [[TMP4]], 128
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP1]], i1 true, i1 [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i129 [[TMP4]], 128
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP6]], i129 0, i129 [[A]]
-; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP6]], i1 true, i1 [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i129 [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i129 3
+; CHECK-NEXT:    [[TMP2:%.*]] = freeze i129 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = freeze i129 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i129 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i129 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP2]], i1 true)
+; CHECK-NEXT:    [[TMP8:%.*]] = call i129 @llvm.ctlz.i129(i129 [[TMP3]], i1 true)
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i129 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ugt i129 [[TMP9]], 128
+; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP6]], i1 true, i1 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i129 [[TMP9]], 128
+; CHECK-NEXT:    [[TMP13:%.*]] = select i1 [[TMP11]], i129 0, i129 [[TMP3]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP11]], i1 true, i1 [[TMP12]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[UDIV_END:%.*]], label [[UDIV_BB1:%.*]]
 ; CHECK:       udiv-loop-exit:
-; CHECK-NEXT:    [[TMP10:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP25:%.*]], [[UDIV_DO_WHILE:%.*]] ]
-; CHECK-NEXT:    [[TMP11:%.*]] = phi i129 [ [[TMP33:%.*]], [[UDIV_BB1]] ], [ [[TMP22:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = shl i129 [[TMP11]], 1
-; CHECK-NEXT:    [[TMP13:%.*]] = or i129 [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = phi i129 [ 0, [[UDIV_BB1]] ], [ [[TMP30:%.*]], [[UDIV_DO_WHILE:%.*]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = phi i129 [ [[TMP39:%.*]], [[UDIV_BB1]] ], [ [[TMP27:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = shl i129 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = or i129 [[TMP15]], [[TMP17]]
 ; CHECK-NEXT:    br label [[UDIV_END]]
 ; CHECK:       udiv-do-while:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP25]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP15:%.*]] = phi i129 [ [[TMP31:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP28:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = phi i129 [ [[TMP30:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP27:%.*]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = phi i129 [ [[TMP33]], [[UDIV_PREHEADER]] ], [ [[TMP22]], [[UDIV_DO_WHILE]] ]
-; CHECK-NEXT:    [[TMP18:%.*]] = shl i129 [[TMP16]], 1
-; CHECK-NEXT:    [[TMP19:%.*]] = lshr i129 [[TMP17]], 128
-; CHECK-NEXT:    [[TMP20:%.*]] = or i129 [[TMP18]], [[TMP19]]
-; CHECK-NEXT:    [[TMP21:%.*]] = shl i129 [[TMP17]], 1
-; CHECK-NEXT:    [[TMP22]] = or i129 [[TMP14]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = sub i129 2, [[TMP20]]
-; CHECK-NEXT:    [[TMP24:%.*]] = ashr i129 [[TMP23]], 128
-; CHECK-NEXT:    [[TMP25]] = and i129 [[TMP24]], 1
-; CHECK-NEXT:    [[TMP26:%.*]] = and i129 [[TMP24]], 3
-; CHECK-NEXT:    [[TMP27]] = sub i129 [[TMP20]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28]] = add i129 [[TMP15]], -1
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i129 [[TMP28]], 0
-; CHECK-NEXT:    br i1 [[TMP29]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
+; CHECK-NEXT:    [[TMP19:%.*]] = phi i129 [ 0, [[UDIV_PREHEADER:%.*]] ], [ [[TMP30]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = phi i129 [ [[TMP37:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP33:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = phi i129 [ [[TMP35:%.*]], [[UDIV_PREHEADER]] ], [ [[TMP32:%.*]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i129 [ [[TMP39]], [[UDIV_PREHEADER]] ], [ [[TMP27]], [[UDIV_DO_WHILE]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = shl i129 [[TMP21]], 1
+; CHECK-NEXT:    [[TMP24:%.*]] = lshr i129 [[TMP22]], 128
+; CHECK-NEXT:    [[TMP25:%.*]] = or i129 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shl i129 [[TMP22]], 1
+; CHECK-NEXT:    [[TMP27]] = or i129 [[TMP19]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = sub i129 [[TMP36:%.*]], [[TMP25]]
+; CHECK-NEXT:    [[TMP29:%.*]] = ashr i129 [[TMP28]], 128
+; CHECK-NEXT:    [[TMP30]] = and i129 [[TMP29]], 1
+; CHECK-NEXT:    [[TMP31:%.*]] = and i129 [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP32]] = sub i129 [[TMP25]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33]] = add i129 [[TMP20]], -1
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i129 [[TMP33]], 0
+; CHECK-NEXT:    br i1 [[TMP34]], label [[UDIV_LOOP_EXIT:%.*]], label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-preheader:
-; CHECK-NEXT:    [[TMP30]] = lshr i129 [[A]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35]] = lshr i129 [[TMP3]], [[TMP37]]
+; CHECK-NEXT:    [[TMP36]] = add i129 [[TMP2]], -1
 ; CHECK-NEXT:    br label [[UDIV_DO_WHILE]]
 ; CHECK:       udiv-bb1:
-; CHECK-NEXT:    [[TMP31]] = add i129 [[TMP4]], 1
-; CHECK-NEXT:    [[TMP32:%.*]] = sub i129 128, [[TMP4]]
-; CHECK-NEXT:    [[TMP33]] = shl i129 [[A]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i129 [[TMP31]], 0
-; CHECK-NEXT:    br i1 [[TMP34]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
+; CHECK-NEXT:    [[TMP37]] = add i129 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP38:%.*]] = sub i129 128, [[TMP9]]
+; CHECK-NEXT:    [[TMP39]] = shl i129 [[TMP3]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp eq i129 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[TMP40]], label [[UDIV_LOOP_EXIT]], label [[UDIV_PREHEADER]]
 ; CHECK:       udiv-end:
-; CHECK-NEXT:    [[TMP35:%.*]] = phi i129 [ [[TMP13]], [[UDIV_LOOP_EXIT]] ], [ [[TMP8]], [[_UDIV_SPECIAL_CASES:%.*]] ]
-; CHECK-NEXT:    [[TMP36:%.*]] = mul i129 3, [[TMP35]]
-; CHECK-NEXT:    [[TMP37:%.*]] = sub i129 [[A]], [[TMP36]]
-; CHECK-NEXT:    store i129 [[TMP37]], i129* [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[TMP41:%.*]] = phi i129 [ [[TMP18]], [[UDIV_LOOP_EXIT]] ], [ [[TMP13]], [[_UDIV_SPECIAL_CASES:%.*]] ]
+; CHECK-NEXT:    [[TMP42:%.*]] = mul i129 [[TMP1]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = sub i129 [[TMP0]], [[TMP42]]
+; CHECK-NEXT:    store i129 [[TMP43]], i129* [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %a = load i129, i129* %ptr
   %res = urem i129 %a, 3
   store i129 %res, i129* %out
   ret void
 }

llvm/unittests/Transforms/Utils/IntegerDivisionTest.cpp

TEST(IntegerDivision, SDiv) {
   Value *B = &*AI++;
 
   Value *Div = Builder.CreateSDiv(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::SDiv);
 
   Value *Ret = Builder.CreateRet(Div);
 
   expandDivision(cast<BinaryOperator>(Div));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::AShr);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Quotient = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Quotient && Quotient->getOpcode() == Instruction::Sub);
 }
 
 TEST(IntegerDivision, UDiv) {
   LLVMContext C;
   Module M("test division", C);
TEST(IntegerDivision, UDiv) {
   Value *B = &*AI++;
 
   Value *Div = Builder.CreateUDiv(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::UDiv);
 
   Value *Ret = Builder.CreateRet(Div);
 
   expandDivision(cast<BinaryOperator>(Div));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::ICmp);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Quotient = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Quotient && Quotient->getOpcode() == Instruction::PHI);
 }
 
 TEST(IntegerDivision, SRem) {
   LLVMContext C;
   Module M("test remainder", C);
TEST(IntegerDivision, SRem) {
   Value *B = &*AI++;
 
   Value *Rem = Builder.CreateSRem(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::SRem);
 
   Value *Ret = Builder.CreateRet(Rem);
 
   expandRemainder(cast<BinaryOperator>(Rem));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::AShr);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Remainder = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Remainder && Remainder->getOpcode() == Instruction::Sub);
 }
 
 TEST(IntegerDivision, URem) {
   LLVMContext C;
   Module M("test remainder", C);
TEST(IntegerDivision, URem) {
   Value *B = &*AI++;
 
   Value *Rem = Builder.CreateURem(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::URem);
 
   Value *Ret = Builder.CreateRet(Rem);
 
   expandRemainder(cast<BinaryOperator>(Rem));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::ICmp);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Remainder = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Remainder && Remainder->getOpcode() == Instruction::Sub);
 }
 
 
 TEST(IntegerDivision, SDiv64) {
   LLVMContext C;
TEST(IntegerDivision, SDiv64) {
   Value *B = &*AI++;
 
   Value *Div = Builder.CreateSDiv(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::SDiv);
 
   Value *Ret = Builder.CreateRet(Div);
 
   expandDivision(cast<BinaryOperator>(Div));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::AShr);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Quotient = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Quotient && Quotient->getOpcode() == Instruction::Sub);
 }
 
 TEST(IntegerDivision, UDiv64) {
   LLVMContext C;
   Module M("test division", C);
TEST(IntegerDivision, UDiv64) {
   Value *B = &*AI++;
 
   Value *Div = Builder.CreateUDiv(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::UDiv);
 
   Value *Ret = Builder.CreateRet(Div);
 
   expandDivision(cast<BinaryOperator>(Div));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::ICmp);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Quotient = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Quotient && Quotient->getOpcode() == Instruction::PHI);
 }
 
 TEST(IntegerDivision, SRem64) {
   LLVMContext C;
   Module M("test remainder", C);
TEST(IntegerDivision, SRem64) {
   Value *B = &*AI++;
 
   Value *Rem = Builder.CreateSRem(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::SRem);
 
   Value *Ret = Builder.CreateRet(Rem);
 
   expandRemainder(cast<BinaryOperator>(Rem));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::AShr);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Remainder = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Remainder && Remainder->getOpcode() == Instruction::Sub);
 }
 
 TEST(IntegerDivision, URem64) {
   LLVMContext C;
   Module M("test remainder", C);
TEST(IntegerDivision, URem64) {
   Value *B = &*AI++;
 
   Value *Rem = Builder.CreateURem(A, B);
   EXPECT_TRUE(BB->front().getOpcode() == Instruction::URem);
 
   Value *Ret = Builder.CreateRet(Rem);
 
   expandRemainder(cast<BinaryOperator>(Rem));
-  EXPECT_TRUE(BB->front().getOpcode() == Instruction::ICmp);
+  EXPECT_TRUE(BB->front().getOpcode() == Instruction::Freeze);
 
   Instruction* Remainder = dyn_cast<Instruction>(cast<User>(Ret)->getOperand(0));
   EXPECT_TRUE(Remainder && Remainder->getOpcode() == Instruction::Sub);
 }
 
 }