[DAG] computeKnownBits - add ISD::MULHS/MULHU/SMUL_LOHI/UMUL_LOHI handling

[RKSimon]

Summary

Reuse the existing KnownBits multiplication code to handle the 'extend + multiply + extract high bits' pattern for multiply-high ops.

Noticed while looking at the codegen for D88785 / D98587 - the patch helps division-by-constant expansion code in particular, which suggests that we might have some further KnownBits div/rem cases we could handle - but this was far easier to implement.

[lebedev.ri]

Can you please split this into KnownBits patch and SelectionDAG patch?

[RKSimon]

In D98857#2634345, @lebedev.ri wrote:

Can you please split this into KnownBits patch and SelectionDAG patch?

Sure

[foad]

llvm/include/llvm/Support/KnownBits.h

299

Typo "extened"

302

Zero-extended?

[RKSimon]

rebase on top of D98866

[nagisa]

I think you may be able to quite easily add a case for S/UMUL_LOHI here as well (but unsure if its necessary, just noticing that its not explicitly handled here)

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

2983

What does this (and a similar one below) comment signify?

[RKSimon]

In D98857#2636074, @nagisa wrote:

I think you may be able to quite easily add a case for S/UMUL_LOHI here as well (but unsure if its necessary, just noticing that its not explicitly handled here)

I'm happy to do that (and MULHS NumSignBits) as followup patches.

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

2983

It was of more use when I did the expansion here instead of inside KnownBits - I'll drop it

[RKSimon]

remove old comment

[RKSimon]

Added *MUL_LOHI handling as the diff was relatively benign

[foad]

The code changes look good to me, and the effect on the the AMDGPU tests is nice.

llvm/include/llvm/Support/KnownBits.h

public:
 
   /// Compute known bits resulting from adding LHS and RHS.
   static KnownBits computeForAddSub(bool Add, bool NSW, const KnownBits &LHS,
                                     KnownBits RHS);
 
   /// Compute known bits resulting from multiplying LHS and RHS.
   static KnownBits computeForMul(const KnownBits &LHS, const KnownBits &RHS);
 
+  /// Compute known bits from sign-extened multiply-hi.
+  static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS);
+
+  /// Compute known bits from sign-extened multiply-hi.
+  static KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS);
+
   /// Compute known bits for udiv(LHS, RHS).
   static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS);
 
   /// Compute known bits for urem(LHS, RHS).
   static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS);
 
   /// Compute known bits for srem(LHS, RHS).
   static KnownBits srem(const KnownBits &LHS, const KnownBits &RHS);

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

case ISD::XOR:
     Known ^= Known2;
     break;
   case ISD::MUL: {
     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
     Known = KnownBits::computeForMul(Known, Known2);
     break;
   }
+  case ISD::MULHU: {
+    // mulhu(x,y) -> trunc(srl(mul(zext(x),zext(y)),bw)).
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    Known = KnownBits::mulhu(Known, Known2);
+    break;
+  }
+  case ISD::MULHS: {
+    // mulhs(x,y) -> trunc(srl(mul(sext(x),sext(y)),bw)).
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    Known = KnownBits::mulhs(Known, Known2);
+    break;
+  }
   case ISD::UDIV: {
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
     Known = KnownBits::udiv(Known, Known2);
     break;
   }
   case ISD::SELECT:
   case ISD::VSELECT:

llvm/lib/Support/KnownBits.cpp

KnownBits KnownBits::computeForMul(const KnownBits &LHS, const KnownBits &RHS) {
 
   KnownBits Res(BitWidth);
   Res.Zero.setHighBits(LeadZ);
   Res.Zero |= (~BottomKnown).getLoBits(ResultBitsKnown);
   Res.One = BottomKnown.getLoBits(ResultBitsKnown);
   return Res;
 }
 
+KnownBits KnownBits::mulhs(const KnownBits &LHS, const KnownBits &RHS) {
+  unsigned BitWidth = LHS.getBitWidth();
+  assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() &&
+         !RHS.hasConflict() && "Operand mismatch");
+  KnownBits WideLHS = LHS.sext(2 * BitWidth);
+  KnownBits WideRHS = RHS.sext(2 * BitWidth);
+  return computeForMul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth);
+}
+
+KnownBits KnownBits::mulhu(const KnownBits &LHS, const KnownBits &RHS) {
+  unsigned BitWidth = LHS.getBitWidth();
+  assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() &&
+         !RHS.hasConflict() && "Operand mismatch");
+  KnownBits WideLHS = LHS.zext(2 * BitWidth);
+  KnownBits WideRHS = RHS.zext(2 * BitWidth);
+  return computeForMul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth);
+}
+
 KnownBits KnownBits::udiv(const KnownBits &LHS, const KnownBits &RHS) {
   unsigned BitWidth = LHS.getBitWidth();
   assert(!LHS.hasConflict() && !RHS.hasConflict());
   KnownBits Known(BitWidth);
 
   // For the purposes of computing leading zeros we can conservatively
   // treat a udiv as a logical right shift by the power of 2 known to
   // be less than the denominator.

llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll

 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz BB3_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    s_mul_i32 s6, s6, 5
 ; GFX7LESS-NEXT:    s_mov_b32 s10, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_mov_b32 s8, s2
 ; GFX7LESS-NEXT:    s_mov_b32 s9, s3
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT:    s_mul_i32 s3, s2, 5
-; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s2, 5
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7LESS-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    buffer_wbinvl1
 ; GFX7LESS-NEXT:  BB3_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000

 ; GFX89-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX89-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX89-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX89-NEXT:    s_cbranch_execz BB3_2
 ; GFX89-NEXT:  ; %bb.1:
 ; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX89-NEXT:    s_mov_b32 s8, s2
 ; GFX89-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
-; GFX89-NEXT:    v_mul_hi_u32_u24_e64 v2, s2, 5
 ; GFX89-NEXT:    s_mul_i32 s2, s2, 5
 ; GFX89-NEXT:    s_mov_b32 s11, 0xf000
 ; GFX89-NEXT:    s_mov_b32 s10, -1
 ; GFX89-NEXT:    s_mov_b32 s9, s3
 ; GFX89-NEXT:    v_mov_b32_e32 v1, s2
+; GFX89-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX89-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX89-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
 ; GFX89-NEXT:    s_waitcnt vmcnt(0)
 ; GFX89-NEXT:    buffer_wbinvl1_vol
 ; GFX89-NEXT:  BB3_2:
 ; GFX89-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX89-NEXT:    v_readfirstlane_b32 s2, v1

 ; GCN64-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GCN64-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GCN64-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
 ; GCN64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GCN64-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN64-NEXT:    s_cbranch_execz BB3_2
 ; GCN64-NEXT:  ; %bb.1:
 ; GCN64-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GCN64-NEXT:    v_mov_b32_e32 v2, 0
+; GCN64-NEXT:    s_mul_i32 s6, s6, 5
 ; GCN64-NEXT:    s_mov_b32 s11, 0x31016000
-; GCN64-NEXT:    s_mul_i32 s7, s6, 5
-; GCN64-NEXT:    v_mul_hi_u32_u24_e64 v2, s6, 5
-; GCN64-NEXT:    v_mov_b32_e32 v1, s7
+; GCN64-NEXT:    v_mov_b32_e32 v1, s6
 ; GCN64-NEXT:    s_mov_b32 s10, -1
 ; GCN64-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN64-NEXT:    s_mov_b32 s8, s2
 ; GCN64-NEXT:    s_mov_b32 s9, s3
 ; GCN64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN64-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN64-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
 ; GCN64-NEXT:    s_waitcnt vmcnt(0)

 ; GCN32-NEXT:    s_mov_b32 s5, exec_lo
 ; GCN32-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GCN32-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
 ; GCN32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GCN32-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; GCN32-NEXT:    s_cbranch_execz BB3_2
 ; GCN32-NEXT:  ; %bb.1:
 ; GCN32-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GCN32-NEXT:    v_mov_b32_e32 v2, 0
+; GCN32-NEXT:    s_mul_i32 s5, s5, 5
 ; GCN32-NEXT:    s_mov_b32 s11, 0x31016000
-; GCN32-NEXT:    s_mul_i32 s6, s5, 5
-; GCN32-NEXT:    v_mul_hi_u32_u24_e64 v2, s5, 5
-; GCN32-NEXT:    v_mov_b32_e32 v1, s6
+; GCN32-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN32-NEXT:    s_mov_b32 s10, -1
 ; GCN32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN32-NEXT:    s_mov_b32 s8, s2
 ; GCN32-NEXT:    s_mov_b32 s9, s3
 ; GCN32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN32-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN32-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
 ; GCN32-NEXT:    s_waitcnt vmcnt(0)

 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz BB9_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    s_mul_i32 s6, s6, 5
 ; GFX7LESS-NEXT:    s_mov_b32 s10, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_mov_b32 s8, s2
 ; GFX7LESS-NEXT:    s_mov_b32 s9, s3
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT:    s_mul_i32 s3, s2, 5
-; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s2, 5
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7LESS-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    buffer_wbinvl1
 ; GFX7LESS-NEXT:  BB9_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000

 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB9_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s8, s2
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s2, 5
 ; GFX8-NEXT:    s_mul_i32 s2, s2, 5
 ; GFX8-NEXT:    s_mov_b32 s11, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s10, -1
 ; GFX8-NEXT:    s_mov_b32 s9, s3
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
 ; GFX8-NEXT:  BB9_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    v_readfirstlane_b32 s5, v2
 ; GFX8-NEXT:    v_readfirstlane_b32 s4, v1

 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB9_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s8, s2
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s2, 5
 ; GFX9-NEXT:    s_mul_i32 s2, s2, 5
 ; GFX9-NEXT:    s_mov_b32 s11, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s10, -1
 ; GFX9-NEXT:    s_mov_b32 s9, s3
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:  BB9_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_readfirstlane_b32 s5, v2
 ; GFX9-NEXT:    v_readfirstlane_b32 s4, v1

 ; GCN64-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GCN64-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GCN64-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
 ; GCN64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GCN64-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN64-NEXT:    s_cbranch_execz BB9_2
 ; GCN64-NEXT:  ; %bb.1:
 ; GCN64-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GCN64-NEXT:    v_mov_b32_e32 v2, 0
+; GCN64-NEXT:    s_mul_i32 s6, s6, 5
 ; GCN64-NEXT:    s_mov_b32 s11, 0x31016000
-; GCN64-NEXT:    s_mul_i32 s7, s6, 5
-; GCN64-NEXT:    v_mul_hi_u32_u24_e64 v2, s6, 5
-; GCN64-NEXT:    v_mov_b32_e32 v1, s7
+; GCN64-NEXT:    v_mov_b32_e32 v1, s6
 ; GCN64-NEXT:    s_mov_b32 s10, -1
 ; GCN64-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN64-NEXT:    s_mov_b32 s8, s2
 ; GCN64-NEXT:    s_mov_b32 s9, s3
 ; GCN64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN64-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN64-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
 ; GCN64-NEXT:    s_waitcnt vmcnt(0)

 ; GCN32-NEXT:    s_mov_b32 s5, exec_lo
 ; GCN32-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GCN32-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
 ; GCN32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GCN32-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; GCN32-NEXT:    s_cbranch_execz BB9_2
 ; GCN32-NEXT:  ; %bb.1:
 ; GCN32-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GCN32-NEXT:    v_mov_b32_e32 v2, 0
+; GCN32-NEXT:    s_mul_i32 s5, s5, 5
 ; GCN32-NEXT:    s_mov_b32 s11, 0x31016000
-; GCN32-NEXT:    s_mul_i32 s6, s5, 5
-; GCN32-NEXT:    v_mul_hi_u32_u24_e64 v2, s5, 5
-; GCN32-NEXT:    v_mov_b32_e32 v1, s6
+; GCN32-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN32-NEXT:    s_mov_b32 s10, -1
 ; GCN32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN32-NEXT:    s_mov_b32 s8, s2
 ; GCN32-NEXT:    s_mov_b32 s9, s3
 ; GCN32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN32-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN32-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
 ; GCN32-NEXT:    s_waitcnt vmcnt(0)

llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll

 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz BB5_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX7LESS-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX7LESS-NEXT:    s_mul_i32 s5, s4, 5
-; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7LESS-NEXT:    s_mov_b32 m0, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:  BB5_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1

 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB5_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX8-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:  BB5_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)

 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB5_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX9-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:  BB5_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v1

 ; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-NEXT:    s_cbranch_execz BB5_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1064-NEXT:    s_mul_i32 s5, s4, 5
-; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  BB5_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]

 ; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB5_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_mul_i32 s3, s3, 5
 ; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1032-NEXT:    s_mul_i32 s4, s3, 5
-; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  BB5_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2

 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz BB11_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX7LESS-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX7LESS-NEXT:    s_mul_i32 s5, s4, 5
-; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7LESS-NEXT:    s_mov_b32 m0, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:  BB11_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1

 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB11_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX8-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:  BB11_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)

 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB11_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX9-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:  BB11_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s3, v2

 ; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-NEXT:    s_cbranch_execz BB11_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1064-NEXT:    s_mul_i32 s5, s4, 5
-; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  BB11_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]

 ; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB11_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_mul_i32 s3, s3, 5
 ; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1032-NEXT:    s_mul_i32 s4, s3, 5
-; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  BB11_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2

llvm/test/CodeGen/AMDGPU/sdiv64.ll

 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, 24, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
-; GCN-NEXT:    v_mul_hi_u32 v0, 0, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, 24, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, 24, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s8, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, s9, v0
 ; GCN-NEXT:    v_mov_b32_e32 v5, s9
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-NEXT:    v_mul_lo_u32 v3, s8, v0
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4

 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v14, v10, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v12, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v13, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
 ; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v4, v6, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
 ; GCN-NEXT:    v_mul_lo_u32 v5, v4, 24
-; GCN-NEXT:    v_mul_hi_u32 v7, 24, v4
-; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, 24, v3
+; GCN-NEXT:    v_mul_hi_u32 v6, 24, v4
 ; GCN-NEXT:    v_mul_hi_u32 v4, 0, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v13, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v13, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v4, v12, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
 ; GCN-NEXT:    v_mul_lo_u32 v7, v1, v3
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
 ; GCN-NEXT:    v_mul_lo_u32 v6, v0, v3
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5

 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v12, vcc
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v13, v7, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
 ; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v4, v6, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    s_mov_b32 s4, 0x8000
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, s4, v3
-; GCN-NEXT:    v_mul_hi_u32 v6, s4, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v7, 15, v4
-; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GCN-NEXT:    v_mul_hi_u32 v3, s4, v3
+; GCN-NEXT:    v_mul_hi_u32 v5, s4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 15, v4
 ; GCN-NEXT:    v_mul_hi_u32 v4, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v13, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v13, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v4, v12, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
 ; GCN-NEXT:    v_mul_lo_u32 v7, v1, v3
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
 ; GCN-NEXT:    v_mul_lo_u32 v6, v0, v3
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5

llvm/test/CodeGen/AMDGPU/srem64.ll

 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, 24, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
-; GCN-NEXT:    v_mul_hi_u32 v0, 0, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, 24, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, 24, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s8, v0
 ; GCN-NEXT:    v_mul_lo_u32 v3, s9, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s8, v0
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1

 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v13, v9, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v11, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v12, v6, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v5, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, 24, v2
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
-; GCN-NEXT:    v_mul_hi_u32 v2, 0, v2
+; GCN-NEXT:    v_mul_hi_u32 v2, 24, v2
+; GCN-NEXT:    v_mul_hi_u32 v5, 24, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v12, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v12, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0, v2
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v11, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
 ; GCN-NEXT:    v_mul_lo_u32 v5, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v0, v2
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3

 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v11, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v12, v6, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v5, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    s_mov_b32 s4, 0x8000
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v4, s4, v2
-; GCN-NEXT:    v_mul_hi_u32 v5, s4, v3
-; GCN-NEXT:    v_lshlrev_b32_e32 v6, 15, v3
-; GCN-NEXT:    v_mul_hi_u32 v2, 0, v2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v2, s4, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, s4, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 15, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v12, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v12, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0, v2
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v11, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
 ; GCN-NEXT:    v_mul_lo_u32 v5, v1, v2
 ; GCN-NEXT:    v_mul_lo_u32 v2, v0, v2
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3

llvm/test/CodeGen/AMDGPU/udiv64.ll

 ; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
 ; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
 ; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
 ; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
 ; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GCN-NEXT:    v_mul_hi_u32 v8, v3, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; GCN-NEXT:    v_mul_hi_u32 v10, v3, v4
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v8, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v10, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
 ; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
 ; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
 ; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
 ; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
 ; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
 ; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
 ; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
 ; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v3, v1, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, 24
 ; GCN-NEXT:    v_mul_hi_u32 v0, v0, 24
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, 24
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, 24
 ; GCN-NEXT:    v_mov_b32_e32 v5, s7
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, s7, v0
 ; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GCN-NEXT:    v_mul_lo_u32 v3, s6, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
 ; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 24, v3
 ; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
 ; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s6, v3
 ; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s7, v4
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s6, v5
 ; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
 ; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s7, v4
 ; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
 ; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v2, s[0:1], 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
 ;
 ; GCN-IR-LABEL: s_test_udiv_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-IR-NEXT:    s_waitcnt lgkmcnt(0)

llvm/test/CodeGen/AMDGPU/urem-seteq-illegal-types.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-mesa3d < %s | FileCheck %s
 
 define i1 @test_urem_odd(i13 %X) nounwind {
 ; CHECK-LABEL: test_urem_odd:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x1fff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0xcccccccd
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 2, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, 5
+; CHECK-NEXT:    v_mul_u32_u24_e32 v1, 5, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %urem = urem i13 %X, 5
   %cmp = icmp eq i13 %urem, 0
   ret i1 %cmp
 }
 
 define i1 @test_urem_even(i27 %X) nounwind {
 ; CHECK-LABEL: test_urem_even:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_and_b32_e32 v1, 0x7ffffff, v0
 ; CHECK-NEXT:    v_bfe_u32 v0, v0, 1, 26
 ; CHECK-NEXT:    s_mov_b32 s4, 0x92492493
 ; CHECK-NEXT:    v_mul_hi_u32 v0, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v0, 2, v0
-; CHECK-NEXT:    v_mul_lo_u32 v0, v0, 14
+; CHECK-NEXT:    v_mul_u32_u24_e32 v0, 14, v0
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v1, v0
 ; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %urem = urem i27 %X, 14
   %cmp = icmp eq i27 %urem, 0
   ret i1 %cmp
 }
 
 define i1 @test_urem_odd_setne(i4 %X) nounwind {
 ; CHECK-LABEL: test_urem_odd_setne:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_and_b32_e32 v0, 15, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0xcccccccd
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 2, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, 5
+; CHECK-NEXT:    v_mul_u32_u24_e32 v1, 5, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %urem = urem i4 %X, 5
   %cmp = icmp ne i4 %urem, 0
   ret i1 %cmp
 }
 
 define i1 @test_urem_negative_odd(i9 %X) nounwind {
 ; CHECK-LABEL: test_urem_negative_odd:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x1ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0x2050c9f9
-; CHECK-NEXT:    s_movk_i32 s5, 0x1fb
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 6, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, s5
+; CHECK-NEXT:    v_mul_u32_u24_e32 v1, 0x1fb, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %urem = urem i9 %X, -5
   %cmp = icmp ne i9 %urem, 0
   ret i1 %cmp
 }

llvm/test/CodeGen/AMDGPU/urem64.ll

 ; GCN-NEXT:    v_mul_lo_u32 v4, s2, v3
 ; GCN-NEXT:    v_mul_lo_u32 v7, s3, v0
 ; GCN-NEXT:    v_mul_lo_u32 v6, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
 ; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
 ; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GCN-NEXT:    v_mul_hi_u32 v8, v3, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; GCN-NEXT:    v_mul_hi_u32 v10, v3, v4
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v8, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v10, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v2, v6, vcc
 ; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v3, v5, s[0:1]
 ; GCN-NEXT:    v_mul_lo_u32 v6, s2, v4
 ; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
 ; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
 ; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
 ; GCN-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
 ; GCN-NEXT:    v_mul_lo_u32 v10, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v12, v0, v6
 ; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
 ; GCN-NEXT:    v_mul_hi_u32 v9, v4, v7
 ; GCN-NEXT:    v_mul_lo_u32 v7, v4, v7
 ; GCN-NEXT:    v_mul_hi_u32 v8, v4, v6
 ; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
 ; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v2, v12, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v3, v1, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, 24
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, 24
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v3, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v1, s6, v1
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, 24
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, 24
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, 24
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, s7, v0
 ; GCN-NEXT:    v_mul_hi_u32 v2, s6, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, s7, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s6, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, s7
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
-; GCN-NEXT:    v_mov_b32_e32 v3, s7
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
 ; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s6, v0
 ; GCN-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
 ; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s7, v5
 ; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
 ; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
 ; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s6, v4

llvm/test/CodeGen/ARM/select-imm.ll

entry:
   ret i1 %cmp
 }
 
 define i1 @t11() {
 ; ARM-LABEL: t11:
 ; ARM:       @ %bb.0: @ %entry
 ; ARM-NEXT:    .pad #4
 ; ARM-NEXT:    sub sp, sp, #4
-; ARM-NEXT:    ldr r0, .LCPI10_0
-; ARM-NEXT:    mov r1, #33
-; ARM-NEXT:    umull r2, r3, r1, r0
-; ARM-NEXT:    lsr r0, r3, #3
-; ARM-NEXT:    add r0, r0, r0, lsl #2
-; ARM-NEXT:    sub r0, r1, r0, lsl #1
-; ARM-NEXT:    ldr r1, [sp]
-; ARM-NEXT:    and r1, r1, #-33554432
-; ARM-NEXT:    orr r0, r1, r0
-; ARM-NEXT:    mov r1, #255
+; ARM-NEXT:    ldr r0, [sp]
+; ARM-NEXT:    mov r1, #40960
+; ARM-NEXT:    orr r1, r1, #-33554432
 ; ARM-NEXT:    orr r0, r0, #40960
-; ARM-NEXT:    orr r1, r1, #3840
-; ARM-NEXT:    str r0, [sp]
 ; ARM-NEXT:    and r0, r0, r1
-; ARM-NEXT:    sub r0, r0, #3
-; ARM-NEXT:    rsbs r1, r0, #0
-; ARM-NEXT:    adc r0, r0, r1
+; ARM-NEXT:    orr r0, r0, #3
+; ARM-NEXT:    str r0, [sp]
+; ARM-NEXT:    mov r0, #1
 ; ARM-NEXT:    add sp, sp, #4
 ; ARM-NEXT:    mov pc, lr
-; ARM-NEXT:    .p2align 2
-; ARM-NEXT:  @ %bb.1:
-; ARM-NEXT:  .LCPI10_0:
-; ARM-NEXT:    .long 3435973837 @ 0xcccccccd
 ;
 ; ARMT2-LABEL: t11:
 ; ARMT2:       @ %bb.0: @ %entry
 ; ARMT2-NEXT:    .pad #4
 ; ARMT2-NEXT:    sub sp, sp, #4
 ; ARMT2-NEXT:    ldr r1, [sp]
 ; ARMT2-NEXT:    mov r0, #33
 ; ARMT2-NEXT:    movw r2, #52429

llvm/test/CodeGen/PowerPC/urem-seteq-illegal-types.ll

 ; PPC-NEXT:    blr
 ;
 ; PPC64LE-LABEL: test_urem_odd_setne:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lis 4, -13108
 ; PPC64LE-NEXT:    clrlwi 3, 3, 28
 ; PPC64LE-NEXT:    ori 4, 4, 52429
 ; PPC64LE-NEXT:    mulhwu 4, 3, 4
-; PPC64LE-NEXT:    rlwinm 5, 4, 0, 0, 29
 ; PPC64LE-NEXT:    srwi 4, 4, 2
-; PPC64LE-NEXT:    add 4, 4, 5
+; PPC64LE-NEXT:    rlwimi 4, 4, 2, 28, 29
 ; PPC64LE-NEXT:    sub 3, 3, 4
 ; PPC64LE-NEXT:    cntlzw 3, 3
 ; PPC64LE-NEXT:    not 3, 3
 ; PPC64LE-NEXT:    rlwinm 3, 3, 27, 31, 31
 ; PPC64LE-NEXT:    blr
   %urem = urem i4 %X, 5
   %cmp = icmp ne i4 %urem, 0
   ret i1 %cmp

llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    .cfi_offset %esi, -20
 ; CHECK-NEXT:    .cfi_offset %edi, -16
 ; CHECK-NEXT:    .cfi_offset %ebx, -12
 ; CHECK-NEXT:    .cfi_offset %ebp, -8
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %bl
 ; CHECK-NEXT:    testb $1, %bl
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.1: ## %bb116.i
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.2: ## %bb52.i.i
 ; CHECK-NEXT:    testb $1, %bl
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.3: ## %bb142.i
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.4:
 ; CHECK-NEXT:    movl L_.str89$non_lazy_ptr, %edi
 ; CHECK-NEXT:    movb $1, %bh
-; CHECK-NEXT:    movl $274877907, %ebp ## imm = 0x10624DD3
+; CHECK-NEXT:    movl L_.str$non_lazy_ptr, %ebp
 ; CHECK-NEXT:    jmp LBB0_5
-; CHECK-NEXT:  LBB0_23: ## %bb7806
+; CHECK-NEXT:  LBB0_21: ## %bb7806
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp16:
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $1, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll __ZN12wxStringBase6appendEmw
 ; CHECK-NEXT:  Ltmp17:
 ; CHECK-NEXT:  LBB0_5: ## %bb3261
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpl $37, 0
-; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:    jne LBB0_25
 ; CHECK-NEXT:  ## %bb.6: ## %bb3306
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp0:
 ; CHECK-NEXT:    movl %edi, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll __ZN12wxStringBaseaSEPKw
 ; CHECK-NEXT:  Ltmp1:
 ; CHECK-NEXT:  ## %bb.7: ## %bb3314
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    movl 0, %eax
 ; CHECK-NEXT:    cmpl $121, %eax
-; CHECK-NEXT:    ja LBB0_27
+; CHECK-NEXT:    ja LBB0_25
 ; CHECK-NEXT:  ## %bb.8: ## %bb3314
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    jmpl *LJTI0_0(,%eax,4)
 ; CHECK-NEXT:  LBB0_10: ## %bb5809
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:    jne LBB0_25
 ; CHECK-NEXT:  ## %bb.11: ## %bb5809
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    testb %bh, %bh
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.12: ## %bb91.i8504
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    testb $1, %bl
 ; CHECK-NEXT:    je LBB0_14
 ; CHECK-NEXT:  ## %bb.13: ## %bb155.i8541
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp4:
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll _gmtime_r
 ; CHECK-NEXT:  Ltmp5:
 ; CHECK-NEXT:  LBB0_14: ## %bb182.i8560
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    testb $1, %bl
 ; CHECK-NEXT:    je LBB0_15
-; CHECK-NEXT:  ## %bb.17: ## %bb278.i8617
+; CHECK-NEXT:  ## %bb.16: ## %bb278.i8617
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    je LBB0_19
-; CHECK-NEXT:  ## %bb.18: ## %bb440.i8663
+; CHECK-NEXT:    je LBB0_18
+; CHECK-NEXT:  ## %bb.17: ## %bb440.i8663
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp6:
 ; CHECK-NEXT:    movl L_.str4$non_lazy_ptr, %eax
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl L_.str33$non_lazy_ptr, %eax
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl L__ZZNK10wxDateTime5GetTmERKNS_8TimeZoneEE12__FUNCTION__$non_lazy_ptr, %eax
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    movl L_.str$non_lazy_ptr, %eax
-; CHECK-NEXT:    movl %eax, (%esp)
+; CHECK-NEXT:    movl %ebp, (%esp)
 ; CHECK-NEXT:    movl $1717, {{[0-9]+}}(%esp) ## imm = 0x6B5
 ; CHECK-NEXT:    calll __Z10wxOnAssertPKwiPKcS0_S0_
 ; CHECK-NEXT:  Ltmp7:
-; CHECK-NEXT:  LBB0_19: ## %bb448.i8694
-; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    jmp LBB0_20
+; CHECK-NEXT:    jmp LBB0_18
 ; CHECK-NEXT:  LBB0_15: ## %bb187.i8591
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    jne LBB0_27
-; CHECK-NEXT:  ## %bb.16: ## %bb265.i8606
+; CHECK-NEXT:    jne LBB0_25
+; CHECK-NEXT:  LBB0_18: ## %invcont5814
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    imull %ebp
-; CHECK-NEXT:    movl %edx, %eax
-; CHECK-NEXT:    shrl $31, %eax
-; CHECK-NEXT:    shrl $6, %edx
-; CHECK-NEXT:    addl %eax, %edx
-; CHECK-NEXT:    imull $1000, %edx, %eax ## imm = 0x3E8
-; CHECK-NEXT:    negl %eax
-; CHECK-NEXT:  LBB0_20: ## %invcont5814
-; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    movzwl %ax, %eax
 ; CHECK-NEXT:  Ltmp8:
-; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll __ZN8wxString6FormatEPKwz
 ; CHECK-NEXT:    subl $4, %esp
 ; CHECK-NEXT:  Ltmp9:
-; CHECK-NEXT:  ## %bb.21: ## %invcont5831
+; CHECK-NEXT:  ## %bb.19: ## %invcont5831
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp10:
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll __ZN12wxStringBase10ConcatSelfEmPKwm
 ; CHECK-NEXT:  Ltmp11:
 ; CHECK-NEXT:    jmp LBB0_5
 ; CHECK-NEXT:  LBB0_9: ## %bb5657
 ; CHECK-NEXT:  Ltmp13:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl %eax, (%esp)
 ; CHECK-NEXT:    calll __ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE
 ; CHECK-NEXT:  Ltmp14:
-; CHECK-NEXT:    jmp LBB0_27
-; CHECK-NEXT:  LBB0_22: ## %bb5968
+; CHECK-NEXT:    jmp LBB0_25
+; CHECK-NEXT:  LBB0_20: ## %bb5968
 ; CHECK-NEXT:  Ltmp2:
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll __ZN8wxString6FormatEPKwz
 ; CHECK-NEXT:    subl $4, %esp
 ; CHECK-NEXT:  Ltmp3:
-; CHECK-NEXT:  LBB0_27: ## %bb115.critedge.i
+; CHECK-NEXT:  LBB0_25: ## %bb115.critedge.i
 ; CHECK-NEXT:    movl %esi, %eax
 ; CHECK-NEXT:    addl $28, %esp
 ; CHECK-NEXT:    popl %esi
 ; CHECK-NEXT:    popl %edi
 ; CHECK-NEXT:    popl %ebx
 ; CHECK-NEXT:    popl %ebp
 ; CHECK-NEXT:    retl $4
-; CHECK-NEXT:  LBB0_25: ## %lpad.loopexit.split-lp
+; CHECK-NEXT:  LBB0_23: ## %lpad.loopexit.split-lp
 ; CHECK-NEXT:  Ltmp15:
-; CHECK-NEXT:    jmp LBB0_27
-; CHECK-NEXT:  LBB0_26: ## %lpad8185
+; CHECK-NEXT:    jmp LBB0_25
+; CHECK-NEXT:  LBB0_24: ## %lpad8185
 ; CHECK-NEXT:  Ltmp12:
-; CHECK-NEXT:    jmp LBB0_27
-; CHECK-NEXT:  LBB0_24: ## %lpad.loopexit
+; CHECK-NEXT:    jmp LBB0_25
+; CHECK-NEXT:  LBB0_22: ## %lpad.loopexit
 ; CHECK-NEXT:  Ltmp18:
-; CHECK-NEXT:    jmp LBB0_27
+; CHECK-NEXT:    jmp LBB0_25
 ; CHECK-NEXT:  Lfunc_end0:
 entry:
   br i1 %foo, label %bb116.i, label %bb115.critedge.i
 bb115.critedge.i:   ; preds = %entry
   ret void
 bb116.i:    ; preds = %entry
   br i1 %foo, label %bb52.i.i, label %bb3118
 bb3118:   ; preds = %bb116.i

llvm/test/CodeGen/X86/combine-udiv.ll

 ; XOP-NEXT:    retq
   %div = udiv <16 x i8> %x, <i8 -64, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   ret <16 x i8> %div
 }
 
 define <8 x i16> @pr38477(<8 x i16> %a0) {
 ; SSE2-LABEL: pr38477:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,4957,57457,4103,16385,35545,2048,2115]
-; SSE2-NEXT:    pmulhuw %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psubw %xmm2, %xmm1
-; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    paddw %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,65535,65535,0,65535]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,4957,57457,4103,16385,35545,2048,2115]
+; SSE2-NEXT:    pmulhuw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,65535,65535,65535,65535,65535,65535]
-; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psubw %xmm1, %xmm0
+; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,65535,0,65535]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: pr38477:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,4957,57457,4103,16385,35545,2048,2115]
 ; SSE41-NEXT:    pmulhuw %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    psubw %xmm2, %xmm1

llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll

 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; SSE2-NEXT:    psrld $2, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [6,6,6,6]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    psubd %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: p4_vector_urem_by_const__splat:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
 ; SSE4-NEXT:    pmuludq %xmm2, %xmm1
 ; SSE4-NEXT:    pmuludq %xmm0, %xmm2
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; SSE4-NEXT:    psrld $2, %xmm2
-; SSE4-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
 ; SSE4-NEXT:    psubd %xmm2, %xmm0
 ; SSE4-NEXT:    pxor %xmm1, %xmm1
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX2-LABEL: p4_vector_urem_by_const__splat:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
 ; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
 ; AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
-; AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmaddwd {{.*}}(%rip), %xmm1, %xmm1
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 128, i32 128, i32 128> ; clearly a power-of-two or zero
   %t1 = urem <4 x i32> %t0, <i32 6, i32 6, i32 6, i32 6> ; '6' is clearly not a power of two
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 0, i32 0>
   ret <4 x i1> %t2
 }
 
 define <4 x i1> @p5_vector_urem_by_const__nonsplat(<4 x i32> %x, <4 x i32> %y) {
 ; SSE2-LABEL: p5_vector_urem_by_const__nonsplat:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    psrld $1, %xmm1
-; SSE2-NEXT:    psrld $2, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [3,5,6,9]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]
-; SSE2-NEXT:    pmuludq %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    psrld $2, %xmm1
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[1,2]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2,3,1]
+; SSE2-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    psubd %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: p5_vector_urem_by_const__nonsplat:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177]
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE4-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE4-NEXT:    pmuludq %xmm0, %xmm1
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; SSE4-NEXT:    movdqa %xmm1, %xmm2
 ; SSE4-NEXT:    psrld $2, %xmm2
 ; SSE4-NEXT:    psrld $1, %xmm1
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5],xmm1[6,7]
-; SSE4-NEXT:    pmulld {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    pmaddwd {{.*}}(%rip), %xmm1
 ; SSE4-NEXT:    psubd %xmm1, %xmm0
 ; SSE4-NEXT:    pxor %xmm1, %xmm1
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX2-LABEL: p5_vector_urem_by_const__nonsplat:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177]
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
 ; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
 ; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmaddwd {{.*}}(%rip), %xmm1, %xmm1
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 2, i32 4, i32 8>
   %t1 = urem <4 x i32> %t0, <i32 3, i32 5, i32 6, i32 9>
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 0, i32 0>
   ret <4 x i1> %t2

 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; SSE2-NEXT:    psrld $2, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [6,6,6,6]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    psubd %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE4-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
 ; SSE4-NEXT:    pmuludq %xmm2, %xmm1
 ; SSE4-NEXT:    pmuludq %xmm0, %xmm2
 ; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; SSE4-NEXT:    psrld $2, %xmm2
-; SSE4-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
 ; SSE4-NEXT:    psubd %xmm2, %xmm0
 ; SSE4-NEXT:    pxor %xmm1, %xmm1
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX2-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
 ; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
 ; AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
-; AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmaddwd {{.*}}(%rip), %xmm1, %xmm1
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 128, i32 128, i32 128> ; clearly a power-of-two or zero
   %t1 = urem <4 x i32> %t0, <i32 6, i32 6, i32 6, i32 6> ; '6' is clearly not a power of two
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 undef, i32 0>
   ret <4 x i1> %t2

llvm/unittests/Support/KnownBitsTest.cpp

ForeachKnownBits(Bits, [&](const KnownBits &Known2) {
       KnownAnd.One.setAllBits();
       KnownBits KnownOr(KnownAnd);
       KnownBits KnownXor(KnownAnd);
       KnownBits KnownUMax(KnownAnd);
       KnownBits KnownUMin(KnownAnd);
       KnownBits KnownSMax(KnownAnd);
       KnownBits KnownSMin(KnownAnd);
       KnownBits KnownMul(KnownAnd);
+      KnownBits KnownMulHS(KnownAnd);
+      KnownBits KnownMulHU(KnownAnd);
       KnownBits KnownUDiv(KnownAnd);
       KnownBits KnownURem(KnownAnd);
       KnownBits KnownSRem(KnownAnd);
       KnownBits KnownShl(KnownAnd);
       KnownBits KnownLShr(KnownAnd);
       KnownBits KnownAShr(KnownAnd);
 
       ForeachNumInKnownBits(Known1, [&](const APInt &N1) {
ForeachKnownBits(Bits, [&](const KnownBits &Known2) {
           Res = APIntOps::smin(N1, N2);
           KnownSMin.One &= Res;
           KnownSMin.Zero &= ~Res;
 
           Res = N1 * N2;
           KnownMul.One &= Res;
           KnownMul.Zero &= ~Res;
 
+          Res = (N1.sext(2 * Bits) * N2.sext(2 * Bits)).extractBits(Bits, Bits);
+          KnownMulHS.One &= Res;
+          KnownMulHS.Zero &= ~Res;
+
+          Res = (N1.zext(2 * Bits) * N2.zext(2 * Bits)).extractBits(Bits, Bits);
+          KnownMulHU.One &= Res;
+          KnownMulHU.Zero &= ~Res;
+
           if (!N2.isNullValue()) {
             Res = N1.udiv(N2);
             KnownUDiv.One &= Res;
             KnownUDiv.Zero &= ~Res;
 
             Res = N1.urem(N2);
             KnownURem.One &= Res;
             KnownURem.Zero &= ~Res;
ForeachKnownBits(Bits, [&](const KnownBits &Known2) {
       KnownBits ComputedSMax = KnownBits::smax(Known1, Known2);
       EXPECT_EQ(KnownSMax.Zero, ComputedSMax.Zero);
       EXPECT_EQ(KnownSMax.One, ComputedSMax.One);
 
       KnownBits ComputedSMin = KnownBits::smin(Known1, Known2);
       EXPECT_EQ(KnownSMin.Zero, ComputedSMin.Zero);
       EXPECT_EQ(KnownSMin.One, ComputedSMin.One);
 
-      // ComputedMul is conservatively correct, but not guaranteed to be
+      // The following are conservatively correct, but not guaranteed to be
       // precise.
       KnownBits ComputedMul = KnownBits::computeForMul(Known1, Known2);
       EXPECT_TRUE(ComputedMul.Zero.isSubsetOf(KnownMul.Zero));
       EXPECT_TRUE(ComputedMul.One.isSubsetOf(KnownMul.One));
 
+      KnownBits ComputedMulHS = KnownBits::mulhs(Known1, Known2);
+      EXPECT_TRUE(ComputedMulHS.Zero.isSubsetOf(KnownMulHS.Zero));
+      EXPECT_TRUE(ComputedMulHS.One.isSubsetOf(KnownMulHS.One));
+
+      KnownBits ComputedMulHU = KnownBits::mulhu(Known1, Known2);
+      EXPECT_TRUE(ComputedMulHU.Zero.isSubsetOf(KnownMulHU.Zero));
+      EXPECT_TRUE(ComputedMulHU.One.isSubsetOf(KnownMulHU.One));
+
       KnownBits ComputedUDiv = KnownBits::udiv(Known1, Known2);
       EXPECT_TRUE(ComputedUDiv.Zero.isSubsetOf(KnownUDiv.Zero));
       EXPECT_TRUE(ComputedUDiv.One.isSubsetOf(KnownUDiv.One));
 
       KnownBits ComputedURem = KnownBits::urem(Known1, Known2);
       EXPECT_TRUE(ComputedURem.Zero.isSubsetOf(KnownURem.Zero));
       EXPECT_TRUE(ComputedURem.One.isSubsetOf(KnownURem.One));