[AMDGPU] Implement widening multiplies with v_mad_i64_i32/v_mad_u64_u32

[foad]

Summary

Select SelectionDAG ops smul_lohi/umul_lohi to
v_mad_i64_i32/v_mad_u64_u32 respectively, with an addend of 0.
v_mul_lo, v_mul_hi and v_mad_i64/u64 are all quarter-rate instructions
so it is better to use one instruction than two.

Further improvements are possible to make better use of the addend
operand, but this is already a strict improvement over what we have
now.

[foad]

The change in the instruction mix in the generated lit tests looks like this:

$ git show | awk '/^-/&&$3~/[sv]_/{a[$3]--}/^+/&&$3~/[sv]_/{a[$3]++}END{for(i in a)if(a[i])printf"%+d %s\n",a[i],i}' | sort -n
-604 v_mul_hi_u32
-567 v_mul_lo_u32
-31 s_mul_i32
-15 v_mul_hi_i32
-10 s_and_b32
-4 v_cndmask_b32_e32
-2 s_mov_b32
-2 v_cmp_ne_u32_e32
-2 v_lshlrev_b32_e32
-2 v_lshrrev_b32_e32
-2 v_mov_b32_e32
+15 v_mad_i64_i32
+1 v_mul_hi_i32_i24_e32
+20 v_addc_u32_e32
+21 v_add_i32_e32
+2 s_waitcnt
+2 v_cmp_ne_u32_e64
+2 v_mul_i32_i24_e64
+2 v_mul_u32_u24_e64
+4 v_cndmask_b32_e64
+4 v_mul_i32_i24_e32
+4 v_mul_u32_u24_e32
+617 v_mad_u64_u32

So it is mostly replacing v_mul_hi_u32+v_mul_lo_u32 with v_mad_u64_u32.

[arsenm]

GlobalISel version?

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1026

Could this try to do better than 0 if the source is an add?

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

812–813

Seems missing a subtarget check, I'm pretty sure gfx6 didn't have mad_u64_u32

[foad]

In D113986#3134553, @arsenm wrote:

GlobalISel version?

Yes that would be nice, but GlobalISel doesn't seem to have a GMIR opcode corresponding to [su]mul_lohi, at least not yet, so I'm not sure what the best approach would be. Anyway this patch is big enough already so I'd prefer to keep it for SelectionDAG and work on GlobalISel later.

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1026

No, not if the source is an add. It would have to be if the result is used in an add, like what SITargetLowering::performAddCombine does for i64 mul. That's the further improvement I alluded to in the commit message.

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

812–813

Good point, will fix.

[foad]

Add missing subtarget check.

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h

private:
 
   SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
 
   void SelectADD_SUB_I64(SDNode *N);
   void SelectAddcSubb(SDNode *N);
   void SelectUADDO_USUBO(SDNode *N);
   void SelectDIV_SCALE(SDNode *N);
   void SelectMAD_64_32(SDNode *N);
+  void SelectMUL_LOHI(SDNode *N);
   void SelectFMA_W_CHAIN(SDNode *N);
   void SelectFMUL_W_CHAIN(SDNode *N);
   SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
                    uint32_t Width);
   void SelectS_BFEFromShifts(SDNode *N);
   void SelectS_BFE(SDNode *N);
   bool isCBranchSCC(const SDNode *N) const;
   void SelectBRCOND(SDNode *N);

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

case AMDGPUISD::DIV_SCALE: {
     SelectDIV_SCALE(N);
     return;
   }
   case AMDGPUISD::MAD_I64_I32:
   case AMDGPUISD::MAD_U64_U32: {
     SelectMAD_64_32(N);
     return;
   }
+  case ISD::SMUL_LOHI:
+  case ISD::UMUL_LOHI:
+    return SelectMUL_LOHI(N);
   case ISD::CopyToReg: {
     const SITargetLowering& Lowering =
       *static_cast<const SITargetLowering*>(getTargetLowering());
     N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
     break;
   }
   case ISD::AND:
   case ISD::SRL:
void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
   unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
 
   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
                     Clamp };
   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
 }
 
+// We need to handle this here because tablegen doesn't support matching
+// instructions with multiple outputs.
+void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) {
+  SDLoc SL(N);
+  bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
+  unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
+
+  SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64);
+  SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
+  SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Zero, Clamp};
+  SDNode *Mad = CurDAG->getMachineNode(Opc, SL, N->getVTList(), Ops);
+  if (!SDValue(N, 0).use_empty()) {
+    SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32);
+    SDNode *Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
+                                        MVT::i32, SDValue(Mad, 0), Sub0);
+    ReplaceUses(SDValue(N, 0), SDValue(Lo, 0));
+  }
+  if (!SDValue(N, 1).use_empty()) {
+    SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32);
+    SDNode *Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
+                                        MVT::i32, SDValue(Mad, 0), Sub1);
+    ReplaceUses(SDValue(N, 1), SDValue(Hi, 0));
+  }
+  CurDAG->RemoveDeadNode(N);
+}
+
 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
   if (!isUInt<16>(Offset))
     return false;
 
   if (!Base || Subtarget->hasUsableDSOffset() ||
       Subtarget->unsafeDSOffsetFoldingEnabled())
     return true;
 

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h

protected:
   SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL,
                                        unsigned Opc, SDValue LHS,
                                        uint32_t ValLo, uint32_t ValHi) const;
   SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performTruncateCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performMulLoHiCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
                              SDValue RHS, DAGCombinerInfo &DCI) const;
   SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 
   bool isConstantCostlierToNegate(SDValue N) const;
   SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const;

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

if (AMDGPUBypassSlowDiv)
     addBypassSlowDiv(64, 32);
 
   setTargetDAGCombine(ISD::BITCAST);
   setTargetDAGCombine(ISD::SHL);
   setTargetDAGCombine(ISD::SRA);
   setTargetDAGCombine(ISD::SRL);
   setTargetDAGCombine(ISD::TRUNCATE);
   setTargetDAGCombine(ISD::MUL);
+  setTargetDAGCombine(ISD::SMUL_LOHI);
+  setTargetDAGCombine(ISD::UMUL_LOHI);
   setTargetDAGCombine(ISD::MULHU);
   setTargetDAGCombine(ISD::MULHS);
   setTargetDAGCombine(ISD::SELECT);
   setTargetDAGCombine(ISD::SELECT_CC);
   setTargetDAGCombine(ISD::STORE);
   setTargetDAGCombine(ISD::FADD);
   setTargetDAGCombine(ISD::FSUB);
   setTargetDAGCombine(ISD::FNEG);
if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
     return SDValue();
   }
 
   // We need to use sext even for MUL_U24, because MUL_U24 is used
   // for signed multiply of 8 and 16-bit types.
   return DAG.getSExtOrTrunc(Mul, DL, VT);
 }
 
+SDValue
+AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
+                                            DAGCombinerInfo &DCI) const {
+  if (N->getValueType(0) != MVT::i32)
+    return SDValue();
+
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc DL(N);
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
+  // in the source into any_extends if the result of the mul is truncated. Since
+  // we can assume the high bits are whatever we want, use the underlying value
+  // to avoid the unknown high bits from interfering.
+  if (N0.getOpcode() == ISD::ANY_EXTEND)
+    N0 = N0.getOperand(0);
+  if (N1.getOpcode() == ISD::ANY_EXTEND)
+    N1 = N1.getOperand(0);
+
+  // Try to use two fast 24-bit multiplies (one for each half of the result)
+  // instead of one slow extending multiply.
+  unsigned LoOpcode, HiOpcode;
+  if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
+    N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
+    N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
+    LoOpcode = AMDGPUISD::MUL_U24;
+    HiOpcode = AMDGPUISD::MULHI_U24;
+  } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
+    N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
+    N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
+    LoOpcode = AMDGPUISD::MUL_I24;
+    HiOpcode = AMDGPUISD::MULHI_I24;
+  } else {
+    return SDValue();
+  }
+
+  SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1);
+  SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1);
+  DCI.CombineTo(N, Lo, Hi);
+  return SDValue(N, 0);
+}
+
 SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
                                                   DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
 
   if (!Subtarget->hasMulI24() || VT.isVector())
     return SDValue();
 
   // Don't generate 24-bit multiplies on values that are in SGPRs, since
if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
       break;
 
     return performSraCombine(N, DCI);
   }
   case ISD::TRUNCATE:
     return performTruncateCombine(N, DCI);
   case ISD::MUL:
     return performMulCombine(N, DCI);
+  case ISD::SMUL_LOHI:
+  case ISD::UMUL_LOHI:
+    return performMulLoHiCombine(N, DCI);
   case ISD::MULHS:
     return performMulhsCombine(N, DCI);
   case ISD::MULHU:
     return performMulhuCombine(N, DCI);
   case AMDGPUISD::MUL_I24:
   case AMDGPUISD::MUL_U24:
   case AMDGPUISD::MULHI_I24:
   case AMDGPUISD::MULHI_U24:

llvm/lib/Target/AMDGPU/SIISelLowering.h

private:
   SDValue convertArgType(
     SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Val,
     bool Signed, const ISD::InputArg *Arg = nullptr) const;
 
   /// Custom lowering for ISD::FP_ROUND for MVT::f16.
   SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerXMULO(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue getSegmentAperture(unsigned AS, const SDLoc &DL,
                              SelectionDAG &DAG) const;
 
   SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

#endif
 
   for (MVT VT : { MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8 }) {
     setOperationAction(ISD::SELECT, VT, Custom);
   }
 
   setOperationAction(ISD::SMULO, MVT::i64, Custom);
   setOperationAction(ISD::UMULO, MVT::i64, Custom);
 
+  if (Subtarget->hasMad64_32()) {
+    setOperationAction(ISD::SMUL_LOHI, MVT::i32, Custom);
+    setOperationAction(ISD::UMUL_LOHI, MVT::i32, Custom);
+  }
+
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f16, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2i16, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2f16, Custom);
 
SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::UADDSAT:
   case ISD::USUBSAT:
   case ISD::SADDSAT:
   case ISD::SSUBSAT:
     return splitBinaryVectorOp(Op, DAG);
   case ISD::SMULO:
   case ISD::UMULO:
     return lowerXMULO(Op, DAG);
+  case ISD::SMUL_LOHI:
+  case ISD::UMUL_LOHI:
+    return lowerXMUL_LOHI(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC:
     return LowerDYNAMIC_STACKALLOC(Op, DAG);
   }
   return SDValue();
 }
 
 // Used for D16: Casts the result of an instruction into the right vector,
 // packs values if loads return unpacked values.
SDValue Sign = isSigned
     ? DAG.getNode(ISD::SRA, SL, VT, Result,
                   DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32))
     : DAG.getConstant(0, SL, VT);
   SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE);
 
   return DAG.getMergeValues({ Result, Overflow }, SL);
 }
 
+SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
+  if (Op->isDivergent()) {
+    // Select to V_MAD_[IU]64_[IU]32.
+    return Op;
+  }
+  if (Subtarget->hasSMulHi()) {
+    // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
+    return SDValue();
+  }
+  // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
+  // calculate the high part, so we might as well do the whole thing with
+  // V_MAD_[IU]64_[IU]32.
+  return Op;
+}
+
 SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
   if (!Subtarget->isTrapHandlerEnabled() ||
       Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
     return lowerTrapEndpgm(Op, DAG);
 
   if (Optional<uint8_t> HsaAbiVer = AMDGPU::getHsaAbiVersion(Subtarget)) {
     switch (*HsaAbiVer) {
     case ELF::ELFABIVERSION_AMDGPU_HSA_V2:

llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll

 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX8-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s12, s6
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
-; GFX8-NEXT:    s_mov_b32 s13, s7
-; GFX8-NEXT:    s_mul_i32 s7, s1, s6
-; GFX8-NEXT:    s_mul_i32 s6, s0, s6
+; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], s0, v0, 0
+; GFX8-NEXT:    s_mul_i32 s6, s1, s6
 ; GFX8-NEXT:    s_mov_b32 s15, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s14, -1
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s7, v0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_mov_b32 s13, s7
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s6, v1
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
 ; GFX8-NEXT:  .LBB4_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mul_lo_u32 v0, s1, v2
-; GFX8-NEXT:    v_mul_hi_u32 v3, s0, v2
+; GFX8-NEXT:    v_mul_lo_u32 v4, s1, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s0, v2, 0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v1, s0, v2
-; GFX8-NEXT:    s_mov_b32 s7, 0xf000
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s6, -1
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: add_i64_uniform:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX9-NEXT:    s_mov_b64 s[8:9], exec

 ; GFX9-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:  .LBB4_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v2
+; GFX9-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v2
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v3, v4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: add_i64_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_clause 0x1
 ; GFX1064-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34

 ; GFX1064-NEXT:    buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  .LBB4_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1064-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1064-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1064-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1064-NEXT:    s_mov_b32 s6, -1
-; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1064-NEXT:    v_add_co_u32 v0, vcc, s0, v2
 ; GFX1064-NEXT:    v_add_co_ci_u32_e32 v1, vcc, s1, v1, vcc
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_clause 0x1

 ; GFX1032-NEXT:    buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:    buffer_gl1_inv
 ; GFX1032-NEXT:  .LBB4_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1032-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1032-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1032-NEXT:    v_mad_u64_u32 v[2:3], s0, s2, v2, 0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s6, -1
-; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1032-NEXT:    v_add_co_u32 v0, vcc_lo, s0, v2
 ; GFX1032-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(1)* %inout, i64 %additive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void

 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX8-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s12, s6
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
-; GFX8-NEXT:    s_mov_b32 s13, s7
-; GFX8-NEXT:    s_mul_i32 s7, s1, s6
-; GFX8-NEXT:    s_mul_i32 s6, s0, s6
+; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[8:9], s0, v0, 0
+; GFX8-NEXT:    s_mul_i32 s6, s1, s6
 ; GFX8-NEXT:    s_mov_b32 s15, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s14, -1
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s7, v0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_mov_b32 s13, s7
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s6, v1
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[12:15], 0 glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
 ; GFX8-NEXT:  .LBB10_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mul_lo_u32 v0, s1, v2
-; GFX8-NEXT:    v_mul_hi_u32 v3, s0, v2
+; GFX8-NEXT:    v_mul_lo_u32 v4, s1, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s0, v2, 0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v1, s0, v2
-; GFX8-NEXT:    s_mov_b32 s7, 0xf000
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v2
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s6, -1
-; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v2, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: sub_i64_uniform:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX9-NEXT:    s_mov_b64 s[8:9], exec

 ; GFX9-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[12:15], 0 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:  .LBB10_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v2
+; GFX9-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v2
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v3, v4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s0, v2
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: sub_i64_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_clause 0x1
 ; GFX1064-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34

 ; GFX1064-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  .LBB10_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1064-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1064-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1064-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1064-NEXT:    s_mov_b32 s6, -1
-; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1064-NEXT:    v_sub_co_u32 v0, vcc, s0, v2
 ; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s1, v1, vcc
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: sub_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_clause 0x1

 ; GFX1032-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:    buffer_gl1_inv
 ; GFX1032-NEXT:  .LBB10_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1032-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1032-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1032-NEXT:    v_mad_u64_u32 v[2:3], s0, s2, v2, 0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s6, -1
-; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1032-NEXT:    v_sub_co_u32 v0, vcc_lo, s0, v2
 ; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(1)* %inout, i64 %subitive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void

llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll

 ; GFX8-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX8-NEXT:    s_cbranch_execz .LBB5_2
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_bcnt1_i32_b64 s8, s[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT:    s_mul_i32 s7, s3, s6
-; GFX8-NEXT:    s_mul_i32 s6, s2, s6
+; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0
+; GFX8-NEXT:    s_mul_i32 s6, s3, s8
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s7, v0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s6, v1
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_add_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:  .LBB5_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s4, s0
-; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX8-NEXT:    v_mul_lo_u32 v0, s3, v2
-; GFX8-NEXT:    v_mul_hi_u32 v3, s2, v2
 ; GFX8-NEXT:    s_mov_b32 s5, s1
+; GFX8-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v1, s2, v2
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
 ; GFX8-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s6, -1
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: add_i64_uniform:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0

 ; GFX9-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_add_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:  .LBB5_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0
 ; GFX9-NEXT:    s_mov_b32 s4, s0
-; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v2
 ; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v3, v4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: add_i64_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0

 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_add_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  .LBB5_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1064-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1064-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1064-NEXT:    v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1064-NEXT:    v_add_co_u32 v0, vcc, s2, v2
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
 ; GFX1064-NEXT:    v_add_co_ci_u32_e32 v1, vcc, s4, v1, vcc
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: add_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry

 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_add_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  .LBB5_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1032-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1032-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1032-NEXT:    v_mad_u64_u32 v[2:3], s2, s2, v2, 0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1032-NEXT:    v_add_co_u32 v0, vcc_lo, s2, v2
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(3)* @local_var64, i64 %additive acq_rel
   store i64 %old, i64 addrspace(1)* %out

 ; GFX8-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v2, s7, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX8-NEXT:    s_cbranch_execz .LBB12_2
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_bcnt1_i32_b64 s8, s[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT:    s_mul_i32 s7, s3, s6
-; GFX8-NEXT:    s_mul_i32 s6, s2, s6
+; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0
+; GFX8-NEXT:    s_mul_i32 s6, s3, s8
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s7, v0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s6, v1
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:  .LBB12_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s4, s0
-; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX8-NEXT:    v_mul_lo_u32 v0, s3, v2
-; GFX8-NEXT:    v_mul_hi_u32 v3, s2, v2
 ; GFX8-NEXT:    s_mov_b32 s5, s1
+; GFX8-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v1, s2, v2
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v2
 ; GFX8-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s6, -1
-; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v2, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: sub_i64_uniform:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0

 ; GFX9-NEXT:    v_mov_b32_e32 v1, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:  .LBB12_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0
 ; GFX9-NEXT:    s_mov_b32 s4, s0
-; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v2
 ; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v3, v4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s0, v2
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: sub_i64_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0

 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
 ; GFX1064-NEXT:  .LBB12_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1064-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1064-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1064-NEXT:    v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1064-NEXT:    v_sub_co_u32 v0, vcc, s2, v2
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
 ; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc
 ; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: sub_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry

 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_sub_rtn_u64 v[0:1], v3, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
 ; GFX1032-NEXT:  .LBB12_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v2
-; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v2
-; GFX1032-NEXT:    v_mul_lo_u32 v2, s2, v2
+; GFX1032-NEXT:    v_mul_lo_u32 v4, s3, v2
+; GFX1032-NEXT:    v_mad_u64_u32 v[2:3], s2, s2, v2, 0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1032-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v3, v4
 ; GFX1032-NEXT:    v_sub_co_u32 v0, vcc_lo, s2, v2
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
 ; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(3)* @local_var64, i64 %subitive acq_rel
   store i64 %old, i64 addrspace(1)* %out

llvm/test/CodeGen/AMDGPU/bypass-div.ll

 ; GFX9-NEXT:    v_or_b32_e32 v5, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB0_2
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v4
-; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v6, v3
-; GFX9-NEXT:    v_sub_co_u32_e32 v7, vcc, 0, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v8, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v6
-; GFX9-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX9-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v9
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v9, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v10, v3, v9
+; GFX9-NEXT:    v_xor_b32_e32 v11, v2, v9
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, v11
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v10
+; GFX9-NEXT:    v_sub_co_u32_e32 v7, vcc, 0, v11
+; GFX9-NEXT:    v_subb_co_u32_e32 v8, vcc, 0, v10, vcc
+; GFX9-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; GFX9-NEXT:    v_rcp_f32_e32 v2, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v14, 0
-; GFX9-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
-; GFX9-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v5
-; GFX9-NEXT:    v_trunc_f32_e32 v6, v6
-; GFX9-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v6
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    v_mul_lo_u32 v11, v7, v6
-; GFX9-NEXT:    v_mul_lo_u32 v9, v8, v5
-; GFX9-NEXT:    v_mul_hi_u32 v10, v7, v5
-; GFX9-NEXT:    v_mul_lo_u32 v12, v7, v5
-; GFX9-NEXT:    v_add3_u32 v9, v10, v11, v9
-; GFX9-NEXT:    v_mul_lo_u32 v10, v5, v9
-; GFX9-NEXT:    v_mul_hi_u32 v11, v5, v12
-; GFX9-NEXT:    v_mul_hi_u32 v13, v5, v9
-; GFX9-NEXT:    v_mul_hi_u32 v15, v6, v9
-; GFX9-NEXT:    v_mul_lo_u32 v9, v6, v9
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v13, v6, v12
-; GFX9-NEXT:    v_mul_hi_u32 v12, v6, v12
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v13
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, v11, v12, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, v15, v14, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v10, v9
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v9
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v10, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v9, v7, v6
-; GFX9-NEXT:    v_mul_lo_u32 v8, v8, v5
-; GFX9-NEXT:    v_mul_hi_u32 v10, v7, v5
-; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v5
-; GFX9-NEXT:    v_add3_u32 v8, v10, v9, v8
-; GFX9-NEXT:    v_mul_lo_u32 v11, v5, v8
-; GFX9-NEXT:    v_mul_hi_u32 v12, v5, v7
-; GFX9-NEXT:    v_mul_hi_u32 v13, v5, v8
-; GFX9-NEXT:    v_mul_hi_u32 v10, v6, v7
-; GFX9-NEXT:    v_mul_lo_u32 v7, v6, v7
-; GFX9-NEXT:    v_mul_hi_u32 v9, v6, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v12, v11
-; GFX9-NEXT:    v_addc_co_u32_e32 v12, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v11, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v12, v10, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v14, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v8, vcc
-; GFX9-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v7
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v7
-; GFX9-NEXT:    v_mul_lo_u32 v8, v0, v6
-; GFX9-NEXT:    v_mul_hi_u32 v9, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v10, v0, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v7, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v9, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v10, v1, v5
-; GFX9-NEXT:    v_mul_hi_u32 v5, v1, v5
-; GFX9-NEXT:    v_mul_hi_u32 v11, v1, v6
-; GFX9-NEXT:    v_mul_lo_u32 v6, v1, v6
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v9, v5, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v14, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v3, v5
-; GFX9-NEXT:    v_mul_lo_u32 v9, v2, v6
-; GFX9-NEXT:    v_mul_hi_u32 v10, v2, v5
-; GFX9-NEXT:    v_mul_lo_u32 v11, v2, v5
-; GFX9-NEXT:    v_add3_u32 v8, v10, v9, v8
-; GFX9-NEXT:    v_sub_u32_e32 v9, v1, v8
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v11
-; GFX9-NEXT:    v_subb_co_u32_e64 v9, s[4:5], v9, v3, vcc
-; GFX9-NEXT:    v_sub_co_u32_e64 v10, s[4:5], v0, v2
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v9, s[4:5], 0, v9, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], v9, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v11, v10, s[4:5]
-; GFX9-NEXT:    v_add_co_u32_e64 v10, s[4:5], 2, v5
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v8, vcc
-; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[4:5], 0, v6, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_add_co_u32_e64 v12, s[4:5], 1, v5
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_addc_co_u32_e64 v13, s[4:5], 0, v6, s[4:5]
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX9-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v12, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v8, v6
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v6, 0
+; GFX9-NEXT:    v_mul_lo_u32 v5, v7, v12
+; GFX9-NEXT:    v_mul_hi_u32 v13, v6, v2
+; GFX9-NEXT:    v_add3_u32 v5, v3, v5, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v13, vcc, v13, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v12, v2, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v13, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v15, v3, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v14, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v13, vcc, v6, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v12, vcc, v12, v3, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v4, v7, v12
+; GFX9-NEXT:    v_mul_lo_u32 v5, v8, v13
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v13, 0
+; GFX9-NEXT:    v_add3_u32 v5, v3, v4, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v12, v5, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v13, v5, 0
+; GFX9-NEXT:    v_mul_hi_u32 v15, v13, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v12, v2, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v15, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v7
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v5, v8, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v4, v14, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v13, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v12, v3, vcc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_xor_b32_e32 v6, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v1, v4, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
+; GFX9-NEXT:    v_mul_hi_u32 v7, v6, v2
+; GFX9-NEXT:    v_xor_b32_e32 v5, v5, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v7, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, v8, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v14, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v7, v10, v2
+; GFX9-NEXT:    v_mul_lo_u32 v8, v11, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v11, v2, 0
+; GFX9-NEXT:    v_add3_u32 v1, v1, v8, v7
+; GFX9-NEXT:    v_sub_u32_e32 v7, v5, v1
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT:    v_subb_co_u32_e64 v6, s[4:5], v7, v10, vcc
+; GFX9-NEXT:    v_sub_co_u32_e64 v7, s[4:5], v0, v11
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v6, s[4:5], 0, v6, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v10
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v11
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], v6, v10
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v7, s[4:5]
+; GFX9-NEXT:    v_add_co_u32_e64 v7, s[4:5], 2, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v5, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e64 v8, s[4:5], 0, v3, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
+; GFX9-NEXT:    v_add_co_u32_e64 v12, s[4:5], 1, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v11
+; GFX9-NEXT:    v_addc_co_u32_e64 v13, s[4:5], 0, v3, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v6
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v12, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v13, v11, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v2, v7, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v12, v7, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v13, v8, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v2, v4, v9
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v6, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v1, v2
 ; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v0, v2, vcc
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:  .LBB0_2: ; %Flow
 ; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]

 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB1_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, v2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, 0, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v10, vcc, 0, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
 ; GFX9-NEXT:    v_rcp_f32_e32 v4, v4
-; GFX9-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-NEXT:    v_mov_b32_e32 v13, 0
 ; GFX9-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
 ; GFX9-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX9-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v9, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v10, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v11, v6, v4
-; GFX9-NEXT:    v_add3_u32 v8, v10, v8, v9
-; GFX9-NEXT:    v_mul_hi_u32 v9, v4, v11
-; GFX9-NEXT:    v_mul_lo_u32 v10, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v13, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v14, v5, v8
-; GFX9-NEXT:    v_mul_lo_u32 v8, v5, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v13, v5, v11
-; GFX9-NEXT:    v_mul_hi_u32 v11, v5, v11
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v13
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v10, v11, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, v14, v12, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v9, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v9, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v6, v6, v4
-; GFX9-NEXT:    v_add3_u32 v7, v9, v8, v7
-; GFX9-NEXT:    v_mul_lo_u32 v10, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v11, v4, v6
-; GFX9-NEXT:    v_mul_hi_u32 v13, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v9, v5, v6
-; GFX9-NEXT:    v_mul_lo_u32 v6, v5, v6
-; GFX9-NEXT:    v_mul_hi_u32 v8, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v7, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v10, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v11, v9, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v8, v12, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v8, vcc
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v5
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v9, v4
+; GFX9-NEXT:    v_mul_lo_u32 v6, v10, v8
+; GFX9-NEXT:    v_mul_lo_u32 v7, v11, v9
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0
+; GFX9-NEXT:    v_add3_u32 v7, v5, v6, v7
+; GFX9-NEXT:    v_mul_hi_u32 v12, v9, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v12, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v14, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v12, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v14, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v13, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v7, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v6, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v7, v0, v4
-; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v9, v1, v5
-; GFX9-NEXT:    v_mul_lo_u32 v5, v1, v5
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v7, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v9, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v14, vcc, v8, v5, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v6, v10, v14
+; GFX9-NEXT:    v_mul_lo_u32 v7, v11, v12
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0
+; GFX9-NEXT:    v_add3_u32 v7, v5, v6, v7
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v14, v7, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v12, v7, 0
+; GFX9-NEXT:    v_mul_hi_u32 v11, v12, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v14, v4, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v11, v7
 ; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v1, v4
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v7, v4, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v9, v12, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v9
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v7, v10, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v13, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v5
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v6, v3, v4
-; GFX9-NEXT:    v_mul_lo_u32 v7, v2, v5
-; GFX9-NEXT:    v_mul_hi_u32 v8, v2, v4
-; GFX9-NEXT:    v_mul_lo_u32 v9, v2, v4
-; GFX9-NEXT:    v_add3_u32 v6, v8, v7, v6
-; GFX9-NEXT:    v_sub_u32_e32 v7, v1, v6
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v9
-; GFX9-NEXT:    v_subb_co_u32_e64 v7, s[4:5], v7, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v12, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v14, v5, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0
+; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v6
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v8, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v9, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v4, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v8, v3, v6
+; GFX9-NEXT:    v_mul_lo_u32 v9, v2, v7
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0
+; GFX9-NEXT:    v_add3_u32 v5, v5, v9, v8
+; GFX9-NEXT:    v_sub_u32_e32 v8, v1, v5
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_subb_co_u32_e64 v4, s[4:5], v8, v3, vcc
 ; GFX9-NEXT:    v_sub_co_u32_e64 v8, s[4:5], v0, v2
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v7, s[4:5], 0, v7, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v3
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v4, s[4:5], 0, v4, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v4, v3
 ; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
 ; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v2
 ; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], v7, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v9, v8, s[4:5]
-; GFX9-NEXT:    v_add_co_u32_e64 v8, s[4:5], 2, v4
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
-; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, v5, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], v4, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v9, v8, s[4:5]
+; GFX9-NEXT:    v_add_co_u32_e64 v8, s[4:5], 2, v6
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, v7, s[4:5]
 ; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_add_co_u32_e64 v10, s[4:5], 1, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GFX9-NEXT:    v_add_co_u32_e64 v10, s[4:5], 1, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[4:5], 0, v5, s[4:5]
+; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[4:5], 0, v7, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v9, s[4:5]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v11, v9, s[4:5]
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, v10, v8, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v7, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v0, vcc
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:  .LBB1_2: ; %Flow
 ; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[6:7]
 ; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB1_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2

 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB2_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v4
-; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, 0, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
-; GFX9-NEXT:    v_rcp_f32_e32 v4, v4
+; GFX9-NEXT:    v_xor_b32_e32 v9, v3, v4
+; GFX9-NEXT:    v_xor_b32_e32 v10, v2, v4
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, v10
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v9
+; GFX9-NEXT:    v_sub_co_u32_e32 v7, vcc, 0, v10
+; GFX9-NEXT:    v_subb_co_u32_e32 v8, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; GFX9-NEXT:    v_rcp_f32_e32 v2, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v13, 0
-; GFX9-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GFX9-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
-; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
-; GFX9-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT:    v_mul_lo_u32 v10, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v8, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v11, v6, v4
-; GFX9-NEXT:    v_add3_u32 v8, v9, v10, v8
-; GFX9-NEXT:    v_mul_lo_u32 v9, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v10, v4, v11
-; GFX9-NEXT:    v_mul_hi_u32 v12, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v14, v5, v8
-; GFX9-NEXT:    v_mul_lo_u32 v8, v5, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v10, v9
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, 0, v12, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v12, v5, v11
-; GFX9-NEXT:    v_mul_hi_u32 v11, v5, v11
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v12
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v10, v11, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, v14, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v9, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v9, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v6, v6, v4
-; GFX9-NEXT:    v_add3_u32 v7, v9, v8, v7
-; GFX9-NEXT:    v_mul_lo_u32 v10, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v11, v4, v6
-; GFX9-NEXT:    v_mul_hi_u32 v12, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v9, v5, v6
-; GFX9-NEXT:    v_mul_lo_u32 v6, v5, v6
-; GFX9-NEXT:    v_mul_hi_u32 v8, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v12, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v7, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v10, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v11, v9, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v8, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v7, vcc
-; GFX9-NEXT:    v_ashrrev_i32_e32 v6, 31, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v6
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v6
-; GFX9-NEXT:    v_mul_lo_u32 v7, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v4
-; GFX9-NEXT:    v_mul_hi_u32 v9, v0, v5
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v6, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v6
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v8, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v9, v1, v4
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GFX9-NEXT:    v_mul_hi_u32 v10, v1, v5
-; GFX9-NEXT:    v_mul_lo_u32 v5, v1, v5
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v9
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v8, v4, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v10, v13, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v5
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v7, v3, v4
-; GFX9-NEXT:    v_mul_lo_u32 v5, v2, v5
-; GFX9-NEXT:    v_mul_hi_u32 v8, v2, v4
-; GFX9-NEXT:    v_mul_lo_u32 v4, v2, v4
-; GFX9-NEXT:    v_add3_u32 v5, v8, v5, v7
-; GFX9-NEXT:    v_sub_u32_e32 v7, v1, v5
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
-; GFX9-NEXT:    v_subb_co_u32_e64 v4, s[4:5], v7, v3, vcc
-; GFX9-NEXT:    v_sub_co_u32_e64 v7, s[4:5], v0, v2
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v8, s[6:7], 0, v4, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v8, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[6:7]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[6:7]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], v8, v3
-; GFX9-NEXT:    v_subb_co_u32_e64 v4, s[4:5], v4, v3, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v9, v10, s[6:7]
-; GFX9-NEXT:    v_sub_co_u32_e64 v10, s[4:5], v7, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v7, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v4, s[4:5]
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX9-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v11, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v8, v6
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v6, 0
+; GFX9-NEXT:    v_mul_lo_u32 v5, v7, v11
+; GFX9-NEXT:    v_mul_hi_u32 v12, v6, v2
+; GFX9-NEXT:    v_add3_u32 v5, v3, v5, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v12, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v11, v2, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v14, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v11, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v12, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v14, v3, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v6, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, v11, v3, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v4, v7, v11
+; GFX9-NEXT:    v_mul_lo_u32 v5, v8, v12
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v12, 0
+; GFX9-NEXT:    v_add3_u32 v5, v3, v4, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v11, v5, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v12, v5, 0
+; GFX9-NEXT:    v_mul_hi_u32 v14, v12, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v11, v2, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v14, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v7
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v5, v8, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v4, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v12, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v11, v3, vcc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v5
+; GFX9-NEXT:    v_xor_b32_e32 v6, v0, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
+; GFX9-NEXT:    v_mul_hi_u32 v7, v6, v2
+; GFX9-NEXT:    v_xor_b32_e32 v4, v4, v5
+; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v7, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, v8, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v2, v9, v0
+; GFX9-NEXT:    v_mul_lo_u32 v3, v10, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v10, v0, 0
+; GFX9-NEXT:    v_add3_u32 v1, v1, v3, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, v4, v1
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT:    v_subb_co_u32_e64 v2, s[4:5], v2, v9, vcc
+; GFX9-NEXT:    v_sub_co_u32_e64 v3, s[4:5], v0, v10
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v6, s[6:7], 0, v2, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v6, v9
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[6:7]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v3, v10
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[6:7]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], v6, v9
+; GFX9-NEXT:    v_subb_co_u32_e64 v2, s[4:5], v2, v9, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[6:7]
+; GFX9-NEXT:    v_sub_co_u32_e64 v8, s[4:5], v3, v10
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v4, v1, vcc
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v9
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v7
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v10
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v9
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v3, v8, s[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v6
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v6
-; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v0, v6
-; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v1, v6, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v5
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v5
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v0, v5
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v1, v5, vcc
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:  .LBB2_2: ; %Flow
 ; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[8:9]
 ; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB2_4
 ; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v2

 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB3_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, v2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, 0, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v10, vcc, 0, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
 ; GFX9-NEXT:    v_rcp_f32_e32 v4, v4
-; GFX9-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-NEXT:    v_mov_b32_e32 v13, 0
 ; GFX9-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
 ; GFX9-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX9-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v9, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v10, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v11, v6, v4
-; GFX9-NEXT:    v_add3_u32 v8, v10, v8, v9
-; GFX9-NEXT:    v_mul_hi_u32 v9, v4, v11
-; GFX9-NEXT:    v_mul_lo_u32 v10, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v13, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v14, v5, v8
-; GFX9-NEXT:    v_mul_lo_u32 v8, v5, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v13, v5, v11
-; GFX9-NEXT:    v_mul_hi_u32 v11, v5, v11
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v13
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v10, v11, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, v14, v12, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v9, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v9, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v6, v6, v4
-; GFX9-NEXT:    v_add3_u32 v7, v9, v8, v7
-; GFX9-NEXT:    v_mul_lo_u32 v10, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v11, v4, v6
-; GFX9-NEXT:    v_mul_hi_u32 v13, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v9, v5, v6
-; GFX9-NEXT:    v_mul_lo_u32 v6, v5, v6
-; GFX9-NEXT:    v_mul_hi_u32 v8, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v7, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v10, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v11, v9, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v8, v12, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v8, vcc
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v5
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v9, v4
+; GFX9-NEXT:    v_mul_lo_u32 v6, v10, v8
+; GFX9-NEXT:    v_mul_lo_u32 v7, v11, v9
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0
+; GFX9-NEXT:    v_add3_u32 v7, v5, v6, v7
+; GFX9-NEXT:    v_mul_hi_u32 v12, v9, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v12, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v14, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v12, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v14, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v13, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v7, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v6, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v7, v0, v4
-; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v9, v1, v5
-; GFX9-NEXT:    v_mul_lo_u32 v5, v1, v5
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v7, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v9, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v14, vcc, v8, v5, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v6, v10, v14
+; GFX9-NEXT:    v_mul_lo_u32 v7, v11, v12
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0
+; GFX9-NEXT:    v_add3_u32 v7, v5, v6, v7
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v14, v7, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v12, v7, 0
+; GFX9-NEXT:    v_mul_hi_u32 v11, v12, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v14, v4, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v11, v7
 ; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v1, v4
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v7, v4, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v9, v12, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v9
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v7, v10, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v13, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v5
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v12, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v14, v5, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0
+; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v6
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v8, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v9, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX9-NEXT:    v_mul_lo_u32 v6, v3, v4
-; GFX9-NEXT:    v_mul_lo_u32 v5, v2, v5
-; GFX9-NEXT:    v_mul_hi_u32 v7, v2, v4
-; GFX9-NEXT:    v_mul_lo_u32 v4, v2, v4
-; GFX9-NEXT:    v_add3_u32 v5, v7, v5, v6
+; GFX9-NEXT:    v_mul_lo_u32 v7, v2, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v2, v4, 0
+; GFX9-NEXT:    v_add3_u32 v5, v5, v7, v6
 ; GFX9-NEXT:    v_sub_u32_e32 v6, v1, v5
 ; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
 ; GFX9-NEXT:    v_subb_co_u32_e64 v4, s[4:5], v6, v3, vcc
 ; GFX9-NEXT:    v_sub_co_u32_e64 v6, s[4:5], v0, v2
 ; GFX9-NEXT:    v_subbrev_co_u32_e64 v7, s[6:7], 0, v4, s[4:5]
 ; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v7, v3
 ; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[6:7]
 ; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v6, v2

 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX9-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB8_2
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v4
-; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v6, v3
-; GFX9-NEXT:    v_sub_co_u32_e32 v7, vcc, 0, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v8, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v6
-; GFX9-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX9-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v9
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v9, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v10, v3, v9
+; GFX9-NEXT:    v_xor_b32_e32 v11, v2, v9
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, v11
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v10
+; GFX9-NEXT:    v_sub_co_u32_e32 v7, vcc, 0, v11
+; GFX9-NEXT:    v_subb_co_u32_e32 v8, vcc, 0, v10, vcc
+; GFX9-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; GFX9-NEXT:    v_rcp_f32_e32 v2, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v14, 0
-; GFX9-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
-; GFX9-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v5
-; GFX9-NEXT:    v_trunc_f32_e32 v6, v6
-; GFX9-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v6
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    v_mul_lo_u32 v11, v7, v6
-; GFX9-NEXT:    v_mul_lo_u32 v9, v8, v5
-; GFX9-NEXT:    v_mul_hi_u32 v10, v7, v5
-; GFX9-NEXT:    v_mul_lo_u32 v12, v7, v5
-; GFX9-NEXT:    v_add3_u32 v9, v10, v11, v9
-; GFX9-NEXT:    v_mul_lo_u32 v10, v5, v9
-; GFX9-NEXT:    v_mul_hi_u32 v11, v5, v12
-; GFX9-NEXT:    v_mul_hi_u32 v13, v5, v9
-; GFX9-NEXT:    v_mul_hi_u32 v15, v6, v9
-; GFX9-NEXT:    v_mul_lo_u32 v9, v6, v9
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v13, v6, v12
-; GFX9-NEXT:    v_mul_hi_u32 v12, v6, v12
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v13
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, v11, v12, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, v15, v14, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v10, v9
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, 0, v11, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v9
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v10, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v9, v7, v6
-; GFX9-NEXT:    v_mul_lo_u32 v8, v8, v5
-; GFX9-NEXT:    v_mul_hi_u32 v10, v7, v5
-; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v5
-; GFX9-NEXT:    v_add3_u32 v8, v10, v9, v8
-; GFX9-NEXT:    v_mul_lo_u32 v11, v5, v8
-; GFX9-NEXT:    v_mul_hi_u32 v12, v5, v7
-; GFX9-NEXT:    v_mul_hi_u32 v13, v5, v8
-; GFX9-NEXT:    v_mul_hi_u32 v10, v6, v7
-; GFX9-NEXT:    v_mul_lo_u32 v7, v6, v7
-; GFX9-NEXT:    v_mul_hi_u32 v9, v6, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v12, v11
-; GFX9-NEXT:    v_addc_co_u32_e32 v12, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v11, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v12, v10, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v14, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v8, vcc
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX9-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v12, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, v8, v6
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v6, 0
+; GFX9-NEXT:    v_mul_lo_u32 v5, v7, v12
+; GFX9-NEXT:    v_mul_hi_u32 v13, v6, v2
+; GFX9-NEXT:    v_add3_u32 v5, v3, v5, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v13, vcc, v13, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v12, v2, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v15, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v13, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v15, v3, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v14, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v13, vcc, v6, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v12, vcc, v12, v3, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v4, v7, v12
+; GFX9-NEXT:    v_mul_lo_u32 v5, v8, v13
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v13, 0
+; GFX9-NEXT:    v_add3_u32 v5, v3, v4, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v12, v5, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v13, v5, 0
+; GFX9-NEXT:    v_mul_hi_u32 v15, v13, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v12, v2, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v15, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v7
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v5, v8, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v4, v14, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v13, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v12, v3, vcc
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v7
-; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v7
-; GFX9-NEXT:    v_mul_lo_u32 v8, v0, v6
-; GFX9-NEXT:    v_mul_hi_u32 v9, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v10, v0, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v7, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v9, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v10, v1, v5
-; GFX9-NEXT:    v_mul_hi_u32 v5, v1, v5
-; GFX9-NEXT:    v_mul_hi_u32 v11, v1, v6
-; GFX9-NEXT:    v_mul_lo_u32 v6, v1, v6
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v9, v5, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v14, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v5, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v3, v5
-; GFX9-NEXT:    v_mul_lo_u32 v9, v2, v6
-; GFX9-NEXT:    v_mul_hi_u32 v10, v2, v5
-; GFX9-NEXT:    v_mul_lo_u32 v11, v2, v5
-; GFX9-NEXT:    v_add3_u32 v8, v10, v9, v8
-; GFX9-NEXT:    v_sub_u32_e32 v9, v1, v8
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v11
-; GFX9-NEXT:    v_subb_co_u32_e64 v9, s[4:5], v9, v3, vcc
-; GFX9-NEXT:    v_sub_co_u32_e64 v10, s[4:5], v0, v2
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v11, s[6:7], 0, v9, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v11, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[6:7]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v10, v2
+; GFX9-NEXT:    v_xor_b32_e32 v5, v0, v7
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v1, v7, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v3, 0
+; GFX9-NEXT:    v_mul_hi_u32 v6, v5, v2
+; GFX9-NEXT:    v_xor_b32_e32 v4, v4, v7
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, v8, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v14, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v6, v10, v2
+; GFX9-NEXT:    v_mul_lo_u32 v8, v11, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v11, v2, 0
+; GFX9-NEXT:    v_add3_u32 v1, v1, v8, v6
+; GFX9-NEXT:    v_sub_u32_e32 v6, v4, v1
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v5, v0
+; GFX9-NEXT:    v_subb_co_u32_e64 v6, s[4:5], v6, v10, vcc
+; GFX9-NEXT:    v_sub_co_u32_e64 v8, s[4:5], v0, v11
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v12, s[6:7], 0, v6, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v10
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[6:7]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v8, v11
 ; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[6:7]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], v11, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v12, v13, s[6:7]
-; GFX9-NEXT:    v_add_co_u32_e64 v13, s[6:7], 2, v5
-; GFX9-NEXT:    v_addc_co_u32_e64 v14, s[6:7], 0, v6, s[6:7]
-; GFX9-NEXT:    v_add_co_u32_e64 v15, s[6:7], 1, v5
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v8, vcc
-; GFX9-NEXT:    v_addc_co_u32_e64 v16, s[6:7], 0, v6, s[6:7]
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v12
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v12, v16, v14, s[6:7]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], v12, v10
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[6:7]
+; GFX9-NEXT:    v_add_co_u32_e64 v13, s[6:7], 2, v2
+; GFX9-NEXT:    v_addc_co_u32_e64 v14, s[6:7], 0, v3, s[6:7]
+; GFX9-NEXT:    v_add_co_u32_e64 v15, s[6:7], 1, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v4, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e64 v16, s[6:7], 0, v3, s[6:7]
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v5
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v11
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v16, v14, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v14, 0, -1, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v8, v14, vcc
-; GFX9-NEXT:    v_subb_co_u32_e64 v3, s[4:5], v9, v3, s[4:5]
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, v15, v13, s[6:7]
-; GFX9-NEXT:    v_sub_co_u32_e64 v2, s[4:5], v10, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v8, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v8, v7, v4
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v3, s[4:5], 0, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v12, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v4, v5, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s[6:7]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v10
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v14, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v15, v13, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v9
+; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v5
+; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v5
+; GFX9-NEXT:    v_sub_co_u32_e64 v4, s[8:9], v2, v5
+; GFX9-NEXT:    v_subb_co_u32_e64 v2, s[4:5], v6, v10, s[4:5]
+; GFX9-NEXT:    v_subb_co_u32_e64 v5, s[8:9], v3, v5, s[8:9]
+; GFX9-NEXT:    v_sub_co_u32_e64 v3, s[4:5], v8, v11
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v12, v2, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v8, v3, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v6, v6, v8
-; GFX9-NEXT:    v_sub_co_u32_e64 v4, s[8:9], v4, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v7
-; GFX9-NEXT:    v_subb_co_u32_e64 v5, s[8:9], v6, v8, s[8:9]
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v7
 ; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, v0, v7
 ; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v1, v7, vcc
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:  .LBB8_2: ; %Flow
 ; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[10:11]
 ; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]

 ; GFX9-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, v2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT:    v_sub_co_u32_e32 v6, vcc, 0, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v10, vcc, 0, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
 ; GFX9-NEXT:    v_rcp_f32_e32 v4, v4
-; GFX9-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-NEXT:    v_mov_b32_e32 v13, 0
 ; GFX9-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
 ; GFX9-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX9-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v9, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v10, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v11, v6, v4
-; GFX9-NEXT:    v_add3_u32 v8, v10, v8, v9
-; GFX9-NEXT:    v_mul_hi_u32 v9, v4, v11
-; GFX9-NEXT:    v_mul_lo_u32 v10, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v13, v4, v8
-; GFX9-NEXT:    v_mul_hi_u32 v14, v5, v8
-; GFX9-NEXT:    v_mul_lo_u32 v8, v5, v8
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v13, v5, v11
-; GFX9-NEXT:    v_mul_hi_u32 v11, v5, v11
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v13
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v10, v11, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, v14, v12, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v9, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v9, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v4
-; GFX9-NEXT:    v_mul_hi_u32 v9, v6, v4
-; GFX9-NEXT:    v_mul_lo_u32 v6, v6, v4
-; GFX9-NEXT:    v_add3_u32 v7, v9, v8, v7
-; GFX9-NEXT:    v_mul_lo_u32 v10, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v11, v4, v6
-; GFX9-NEXT:    v_mul_hi_u32 v13, v4, v7
-; GFX9-NEXT:    v_mul_hi_u32 v9, v5, v6
-; GFX9-NEXT:    v_mul_lo_u32 v6, v5, v6
-; GFX9-NEXT:    v_mul_hi_u32 v8, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
-; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v13, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v7, v5, v7
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v10, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v11, v9, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v8, v12, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v8, vcc
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v8, v5
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v9, v4
+; GFX9-NEXT:    v_mul_lo_u32 v6, v10, v8
+; GFX9-NEXT:    v_mul_lo_u32 v7, v11, v9
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0
+; GFX9-NEXT:    v_add3_u32 v7, v5, v6, v7
+; GFX9-NEXT:    v_mul_hi_u32 v12, v9, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v12, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v14, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v12, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v14, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v13, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v7, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v6, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v7, v0, v4
-; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v5
-; GFX9-NEXT:    v_mul_hi_u32 v9, v1, v5
-; GFX9-NEXT:    v_mul_lo_u32 v5, v1, v5
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v7, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v9, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v14, vcc, v8, v5, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v6, v10, v14
+; GFX9-NEXT:    v_mul_lo_u32 v7, v11, v12
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0
+; GFX9-NEXT:    v_add3_u32 v7, v5, v6, v7
+; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v14, v7, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v12, v7, 0
+; GFX9-NEXT:    v_mul_hi_u32 v11, v12, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v14, v4, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v11, v7
 ; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v1, v4
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v7, v4, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v9, v12, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v9
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v7, v10, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v13, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v5
 ; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v6, v3, v4
-; GFX9-NEXT:    v_mul_lo_u32 v7, v2, v5
-; GFX9-NEXT:    v_mul_hi_u32 v8, v2, v4
-; GFX9-NEXT:    v_mul_lo_u32 v9, v2, v4
-; GFX9-NEXT:    v_add3_u32 v6, v8, v7, v6
-; GFX9-NEXT:    v_sub_u32_e32 v7, v1, v6
-; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v9
-; GFX9-NEXT:    v_subb_co_u32_e64 v7, s[4:5], v7, v3, vcc
-; GFX9-NEXT:    v_sub_co_u32_e64 v8, s[4:5], v0, v2
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v9, s[6:7], 0, v7, s[4:5]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v9, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[6:7]
-; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v8, v2
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v12, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v14, v5, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0
+; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v6
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v8, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v9, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v4, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v8, v3, v6
+; GFX9-NEXT:    v_mul_lo_u32 v9, v2, v7
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0
+; GFX9-NEXT:    v_add3_u32 v5, v5, v9, v8
+; GFX9-NEXT:    v_sub_u32_e32 v8, v1, v5
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_subb_co_u32_e64 v8, s[4:5], v8, v3, vcc
+; GFX9-NEXT:    v_sub_co_u32_e64 v9, s[4:5], v0, v2
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v10, s[6:7], 0, v8, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v10, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[6:7]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[6:7], v9, v2
 ; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[6:7]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], v9, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v10, v11, s[6:7]
-; GFX9-NEXT:    v_add_co_u32_e64 v11, s[6:7], 2, v4
-; GFX9-NEXT:    v_addc_co_u32_e64 v12, s[6:7], 0, v5, s[6:7]
-; GFX9-NEXT:    v_add_co_u32_e64 v13, s[6:7], 1, v4
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
-; GFX9-NEXT:    v_addc_co_u32_e64 v14, s[6:7], 0, v5, s[6:7]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], v10, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v11, s[6:7]
+; GFX9-NEXT:    v_add_co_u32_e64 v11, s[6:7], 2, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v12, s[6:7], 0, v7, s[6:7]
+; GFX9-NEXT:    v_add_co_u32_e64 v13, s[6:7], 1, v6
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_addc_co_u32_e64 v14, s[6:7], 0, v7, s[6:7]
 ; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v10
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, v14, v12, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v14, v12, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_subb_co_u32_e64 v3, s[4:5], v7, v3, s[4:5]
-; GFX9-NEXT:    v_sub_co_u32_e64 v2, s[4:5], v8, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v12, vcc
+; GFX9-NEXT:    v_subb_co_u32_e64 v3, s[4:5], v8, v3, s[4:5]
+; GFX9-NEXT:    v_sub_co_u32_e64 v2, s[4:5], v9, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v12, vcc
 ; GFX9-NEXT:    v_subbrev_co_u32_e64 v3, s[4:5], 0, v3, s[4:5]
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v13, v11, s[6:7]
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v10, v3, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v7, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v13, v11, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v8, v2, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, v2, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v0, v1, vcc
 ; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:  .LBB9_2: ; %Flow
 ; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], s[8:9]
 ; GFX9-NEXT:    s_xor_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_4
 ; GFX9-NEXT:  ; %bb.3:

llvm/test/CodeGen/AMDGPU/llvm.mulo.ll

 ; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
 ; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
 ; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: umulo_i64_v_v:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mul_lo_u32 v5, v0, v3
-; GFX9-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v3
-; GFX9-NEXT:    v_mul_lo_u32 v7, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v6, v5
-; GFX9-NEXT:    v_mul_hi_u32 v10, v1, v3
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v8, v4, vcc
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v4, v1
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[3:4]
-; GFX9-NEXT:    v_add3_u32 v1, v6, v5, v7
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v4, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v10, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v4, v3, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v10, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v9, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v2
+; GFX9-NEXT:    v_mul_lo_u32 v5, v5, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v8, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v4
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: umulo_i64_v_v:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_mul_lo_u32 v5, v0, v3
-; GFX10-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX10-NEXT:    v_mul_hi_u32 v4, v0, v3
-; GFX10-NEXT:    v_mul_lo_u32 v8, v1, v2
-; GFX10-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX10-NEXT:    v_mul_hi_u32 v9, v1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX10-NEXT:    v_add_co_u32 v10, vcc_lo, v6, v5
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v4, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v3, vcc_lo, v10, v8
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v4, v7, vcc_lo
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v9, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v3, vcc_lo, v3, v1
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v4, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v1, v6, v5, v8
-; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
+; GFX10-NEXT:    v_mov_b32_e32 v5, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s4, v5, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s4, v5, v3, 0
+; GFX10-NEXT:    v_mov_b32_e32 v8, v1
+; GFX10-NEXT:    v_mul_lo_u32 v5, v5, v2
+; GFX10-NEXT:    v_mul_lo_u32 v4, v4, v3
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
+; GFX10-NEXT:    v_add3_u32 v1, v1, v4, v5
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v6, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v7, v10, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v12, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v11
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %x, i64 %y)
   ret { i64, i1 } %umulo
 }
 
 define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {

 ; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; SI-NEXT:    v_mov_b32_e32 v0, v4
 ; SI-NEXT:    v_mov_b32_e32 v1, v5
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: smulo_i64_v_v:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mul_lo_u32 v5, v0, v3
-; GFX9-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX9-NEXT:    v_mul_hi_u32 v8, v0, v3
-; GFX9-NEXT:    v_mul_lo_u32 v7, v1, v2
-; GFX9-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v6, v5
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_mul_hi_i32 v10, v1, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v7
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v8, v4, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v8, v1, v3
-; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v10, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v8
-; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v9, vcc
-; GFX9-NEXT:    v_sub_co_u32_e32 v9, vcc, v4, v2
-; GFX9-NEXT:    v_subbrev_co_u32_e32 v10, vcc, 0, v8, vcc
-; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v4, v9, vcc
-; GFX9-NEXT:    v_sub_co_u32_e32 v9, vcc, v8, v0
-; GFX9-NEXT:    v_subbrev_co_u32_e32 v4, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v4, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v10, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_mad_i64_i32 v[6:7], s[4:5], v4, v3, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v10, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v9, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v8, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v6, v2
+; GFX9-NEXT:    v_subbrev_co_u32_e32 v9, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v6, v5
+; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v2
+; GFX9-NEXT:    v_mul_lo_u32 v5, v5, v3
+; GFX9-NEXT:    v_subbrev_co_u32_e32 v9, vcc, 0, v7, vcc
 ; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
-; GFX9-NEXT:    v_add3_u32 v1, v6, v5, v7
-; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX9-NEXT:    v_mov_b32_e32 v6, v5
-; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[3:4], v[5:6]
+; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v4
+; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v8, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v5, v4
+; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[2:3], v[4:5]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: smulo_i64_v_v:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_mul_lo_u32 v4, v0, v3
-; GFX10-NEXT:    v_mul_hi_u32 v5, v0, v2
-; GFX10-NEXT:    v_mul_hi_u32 v6, v0, v3
-; GFX10-NEXT:    v_mul_lo_u32 v8, v1, v2
-; GFX10-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX10-NEXT:    v_mul_hi_i32 v9, v1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v11, v1, v3
-; GFX10-NEXT:    v_add_co_u32 v10, vcc_lo, v5, v4
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v6, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v10, vcc_lo, v10, v8
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v6, v7, vcc_lo
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v9, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
+; GFX10-NEXT:    v_mov_b32_e32 v5, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
+; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s4, v5, v2, 0
+; GFX10-NEXT:    v_mad_i64_i32 v[11:12], s4, v5, v3, 0
+; GFX10-NEXT:    v_mov_b32_e32 v8, v1
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
+; GFX10-NEXT:    v_mul_lo_u32 v8, v5, v2
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v7, v10, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v12, vcc_lo
+; GFX10-NEXT:    v_mul_lo_u32 v9, v4, v3
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v11
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
-; GFX10-NEXT:    v_sub_co_u32 v9, vcc_lo, v6, v2
+; GFX10-NEXT:    v_sub_co_u32 v2, vcc_lo, v6, v2
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    v_add3_u32 v1, v5, v4, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GFX10-NEXT:    v_sub_co_u32 v8, vcc_lo, v6, v0
-; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v9, vcc_lo, 0, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
+; GFX10-NEXT:    v_add3_u32 v1, v1, v9, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v7, v10, vcc_lo
+; GFX10-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; GFX10-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_mov_b32_e32 v5, v4
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[4:5]
+; GFX10-NEXT:    v_mov_b32_e32 v3, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %smulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %x, i64 %y)
   ret { i64, i1 } %smulo
 }
 
 define amdgpu_kernel void @umulo_i64_s(i64 %x, i64 %y) {

llvm/test/CodeGen/AMDGPU/mad_64_32.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=CI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s
 
 define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; CI-LABEL: mad_i64_i32_sextops:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
 ; CI-NEXT:    s_setpc_b64 s[30:31]
 ;
; SI-NEXT:    s_setpc_b64 s[30:31]
   %mad = add i64 %arg2, %mul
   ret i64 %mad
 }
 
 define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; CI-LABEL: mad_i64_i32_sextops_i32_i128:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT:    v_mul_hi_u32 v6, v0, v1
-; CI-NEXT:    v_ashrrev_i32_e32 v12, 31, v0
-; CI-NEXT:    v_mov_b32_e32 v7, 0
-; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v1
-; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v12, v1, v[6:7]
-; CI-NEXT:    v_mul_hi_i32 v11, v1, v12
-; CI-NEXT:    v_mul_lo_u32 v10, v1, v12
-; CI-NEXT:    v_mov_b32_e32 v6, v9
-; CI-NEXT:    v_mov_b32_e32 v9, v7
-; CI-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v0, v13, v[8:9]
-; CI-NEXT:    v_mad_i64_i32 v[10:11], s[4:5], v13, v0, v[10:11]
-; CI-NEXT:    v_add_i32_e32 v8, vcc, v6, v8
-; CI-NEXT:    v_addc_u32_e64 v9, s[4:5], 0, 0, vcc
-; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v12, v13, v[8:9]
-; CI-NEXT:    v_mul_lo_u32 v0, v0, v1
-; CI-NEXT:    v_mov_b32_e32 v1, v7
-; CI-NEXT:    v_add_i32_e32 v6, vcc, v8, v10
-; CI-NEXT:    v_addc_u32_e32 v8, vcc, v9, v11, vcc
-; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; CI-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
+; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
+; CI-NEXT:    v_mov_b32_e32 v8, 0
+; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v1, v[7:8]
+; CI-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
+; CI-NEXT:    v_mad_i64_i32 v[11:12], s[4:5], v1, v13, 0
+; CI-NEXT:    v_mov_b32_e32 v7, v10
+; CI-NEXT:    v_mov_b32_e32 v10, v8
+; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[9:10]
+; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[11:12]
+; CI-NEXT:    v_add_i32_e32 v9, vcc, v7, v9
+; CI-NEXT:    v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
+; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v14, v[9:10]
+; CI-NEXT:    v_add_i32_e32 v7, vcc, v9, v0
+; CI-NEXT:    v_addc_u32_e32 v9, vcc, v10, v1, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v8
+; CI-NEXT:    v_add_i32_e32 v0, vcc, v6, v2
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v6, v4, vcc
-; CI-NEXT:    v_addc_u32_e32 v3, vcc, v8, v5, vcc
+; CI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; CI-NEXT:    v_addc_u32_e32 v3, vcc, v9, v5, vcc
 ; CI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; SI-LABEL: mad_i64_i32_sextops_i32_i128:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    v_ashrrev_i32_e32 v6, 31, v0
 ; SI-NEXT:    v_mul_lo_u32 v11, v6, v1
 ; SI-NEXT:    v_mul_hi_u32 v12, v0, v1
; SI-NEXT:    s_setpc_b64 s[30:31]
   %trunc.lhs = and i64 %arg0, 4294967295
   %trunc.rhs = and i64 %arg1, 4294967295
   %mul = mul i64 %trunc.lhs, %trunc.rhs
   %add = add i64 %mul, %arg2
   ret i64 %add
 }
 
 define i64 @mad_u64_u32_bitops_lhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
-; GCN-LABEL: mad_u64_u32_bitops_lhs_mask_small:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_and_b32_e32 v1, 1, v1
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v1, v1, v2
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; CI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v1, 1, v1
+; CI-NEXT:    v_mul_lo_u32 v3, v1, v2
+; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
+; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 1, v1
+; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
+; SI-NEXT:    v_mul_lo_u32 v1, v1, v2
+; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
+; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; SI-NEXT:    s_setpc_b64 s[30:31]
   %trunc.lhs = and i64 %arg0, 8589934591
   %trunc.rhs = and i64 %arg1, 4294967295
   %mul = mul i64 %trunc.lhs, %trunc.rhs
   %add = add i64 %mul, %arg2
   ret i64 %add
 }
 
 define i64 @mad_u64_u32_bitops_rhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
-; GCN-LABEL: mad_u64_u32_bitops_rhs_mask_small:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_and_b32_e32 v1, 1, v3
-; GCN-NEXT:    v_mul_hi_u32 v3, v0, v2
-; GCN-NEXT:    v_mul_lo_u32 v1, v0, v1
-; GCN-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; CI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT:    v_and_b32_e32 v1, 1, v3
+; CI-NEXT:    v_mul_lo_u32 v3, v0, v1
+; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
+; CI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; CI-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 1, v3
+; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
+; SI-NEXT:    v_mul_lo_u32 v1, v0, v1
+; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
+; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; SI-NEXT:    s_setpc_b64 s[30:31]
   %trunc.lhs = and i64 %arg0, 4294967295
   %trunc.rhs = and i64 %arg1, 8589934591
   %mul = mul i64 %trunc.lhs, %trunc.rhs
   %add = add i64 %mul, %arg2
   ret i64 %add
 }
 
 define i64 @mad_i64_i32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {

llvm/test/CodeGen/AMDGPU/mul.ll

define amdgpu_kernel void @v_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   ret void
 }
 
 ; This 64-bit multiply should just use MUL_HI and MUL_LO, since the top
 ; 32-bits of both arguments are sign bits.
 ; FUNC-LABEL: {{^}}mul64_sext_c:
 ; EG-DAG: MULLO_INT
 ; EG-DAG: MULHI_INT
-; GCN-DAG: s_mul_i32
-; GCN-DAG: v_mul_hi_i32
+; SI-DAG: s_mul_i32
+; SI-DAG: v_mul_hi_i32
+; VI: v_mad_i64_i32
 define amdgpu_kernel void @mul64_sext_c(i64 addrspace(1)* %out, i32 %in) {
 entry:
   %0 = sext i32 %in to i64
   %1 = mul i64 %0, 80
   store i64 %1, i64 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul64_sext_c:
 ; EG-DAG: MULLO_INT
 ; EG-DAG: MULHI_INT
-; GCN-DAG: v_mul_lo_u32
-; GCN-DAG: v_mul_hi_i32
+; SI-DAG: v_mul_lo_u32
+; SI-DAG: v_mul_hi_i32
+; VI: v_mad_i64_i32
 ; GCN: s_endpgm
 define amdgpu_kernel void @v_mul64_sext_c(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %val = load i32, i32 addrspace(1)* %in, align 4
   %ext = sext i32 %val to i64
   %mul = mul i64 %ext, 80
   store i64 %mul, i64 addrspace(1)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul64_sext_inline_imm:
-; GCN-DAG: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, 9
-; GCN-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+; SI-DAG: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+; SI-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+; VI: v_mad_i64_i32 v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, 9, 0
 ; GCN: s_endpgm
 define amdgpu_kernel void @v_mul64_sext_inline_imm(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %val = load i32, i32 addrspace(1)* %in, align 4
   %ext = sext i32 %val to i64
   %mul = mul i64 %ext, 9
   store i64 %mul, i64 addrspace(1)* %out, align 8
   ret void
 }

 
 endif:
   %3 = phi i32 [%1, %if], [%2, %else]
   store i32 %3, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}mul64_in_branch:
-; GCN-DAG: s_mul_i32
-; GCN-DAG: v_mul_hi_u32
+; SI-DAG: s_mul_i32
+; SI-DAG: v_mul_hi_u32
+; VI: v_mad_u64_u32
 ; GCN: s_endpgm
 define amdgpu_kernel void @mul64_in_branch(i64 addrspace(1)* %out, i64 addrspace(1)* %in, i64 %a, i64 %b, i64 %c) {
 entry:
   %0 = icmp eq i64 %a, 0
   br i1 %0, label %if, label %else
 
 if:
   %1 = load i64, i64 addrspace(1)* %in

 
 ; SI-DAG: s_mul_i32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: s_mul_i32
 ; SI-DAG: s_mul_i32
 ; SI-DAG: v_mul_hi_u32
 
-; VI: v_mul_hi_u32
-; VI: s_mul_i32
-; VI: s_mul_i32
-; VI: v_mul_hi_u32
-; VI: v_mul_hi_u32
-; VI: s_mul_i32
-; VI: v_mad_u64_u32
-; VI: s_mul_i32
-; VI: v_mad_u64_u32
-; VI: s_mul_i32
-; VI: s_mul_i32
-; VI: v_mad_u64_u32
-; VI: s_mul_i32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: s_mul_i32
+; VI-DAG: s_mul_i32
+; VI-DAG: s_mul_i32
+; VI-DAG: s_mul_i32
 
 
 ; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, [8 x i32], i128 %a, [8 x i32], i128 %b) nounwind #0 {
   %mul = mul i128 %a, %b
   store i128 %mul, i128 addrspace(1)* %out
   ret void
 }

 ; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_lo_u32
 
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mad_u64_u32
+; VI-DAG: v_mul_lo_u32
+; VI-DAG: v_mul_lo_u32
 ; VI-DAG: v_mul_lo_u32
-; VI-DAG: v_mul_hi_u32
-; VI: v_mad_u64_u32
-; VI: v_mad_u64_u32
-; VI: v_mad_u64_u32
 
 ; GCN: {{buffer|flat}}_store_dwordx4
 define amdgpu_kernel void @v_mul_i128(i128 addrspace(1)* %out, i128 addrspace(1)* %aptr, i128 addrspace(1)* %bptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.a = getelementptr inbounds i128, i128 addrspace(1)* %aptr, i32 %tid
   %gep.b = getelementptr inbounds i128, i128 addrspace(1)* %bptr, i32 %tid
   %gep.out = getelementptr inbounds i128, i128 addrspace(1)* %bptr, i32 %tid
   %a = load i128, i128 addrspace(1)* %gep.a

llvm/test/CodeGen/AMDGPU/mul_int24.ll

 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x70
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_bfe_i32 s1, s2, 0x180000
 ; VI-NEXT:    s_bfe_i32 s0, s0, 0x180000
 ; VI-NEXT:    v_mov_b32_e32 v0, s1
 ; VI-NEXT:    v_mul_hi_i32_i24_e32 v1, s0, v0
-; VI-NEXT:    s_mul_i32 s0, s0, s1
-; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mul_i32_i24_e32 v0, s0, v0
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: test_smul24_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x4c
 ; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x70

 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_bfe_i32 s0, s0, 0x180000
 ; VI-NEXT:    v_mul_hi_i32_i24_e64 v1, s0, s0
-; VI-NEXT:    s_mul_i32 s0, s0, s0
-; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mul_i32_i24_e64 v0, s0, s0
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: test_smul24_i64_square:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000

 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshl_b32 s1, s2, 8
 ; VI-NEXT:    s_lshl_b32 s3, s0, 8
 ; VI-NEXT:    s_ashr_i64 s[2:3], s[2:3], 40
 ; VI-NEXT:    s_ashr_i64 s[0:1], s[0:1], 40
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mul_hi_i32_i24_e32 v1, s0, v0
-; VI-NEXT:    s_mul_i32 s0, s0, s2
-; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mul_i32_i24_e32 v0, s0, v0
 ; VI-NEXT:    v_lshlrev_b64 v[0:1], 31, v[0:1]
 ; VI-NEXT:    v_ashrrev_i64 v[0:1], 31, v[0:1]
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: test_smul24_i33:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24

llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll

 ; VI-LABEL: test_umul24_i64:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_load_dword s7, s[0:1], 0x34
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_mov_b32 s0, s4
-; VI-NEXT:    s_mov_b32 s4, 0xffffff
-; VI-NEXT:    s_mov_b32 s1, s5
-; VI-NEXT:    s_and_b32 s5, s6, s4
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_and_b32 s4, s7, s4
-; VI-NEXT:    s_mul_i32 s5, s5, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s7
+; VI-NEXT:    s_mov_b32 s1, s5
 ; VI-NEXT:    v_mul_hi_u32_u24_e32 v1, s6, v0
-; VI-NEXT:    v_mov_b32_e32 v0, s5
+; VI-NEXT:    v_mul_u32_u24_e32 v0, s6, v0
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: test_umul24_i64:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x34

 ;
 ; VI-LABEL: test_umul24_i64_square:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x4c
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_and_b32 s1, s0, 0xffffff
-; VI-NEXT:    s_mul_i32 s1, s1, s1
 ; VI-NEXT:    v_mul_hi_u32_u24_e64 v1, s0, s0
-; VI-NEXT:    v_mov_b32_e32 v0, s1
+; VI-NEXT:    v_mul_u32_u24_e64 v0, s0, s0
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: test_umul24_i64_square:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x4c
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000

 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: test_umul24_i33:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
 ; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
-; VI-NEXT:    s_mov_b32 s1, 0xffffff
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    s_and_b32 s3, s2, s1
-; VI-NEXT:    s_and_b32 s1, s0, s1
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    s_mul_i32 s3, s3, s1
-; VI-NEXT:    v_mul_hi_u32_u24_e32 v0, s2, v0
-; VI-NEXT:    v_and_b32_e32 v1, 1, v0
-; VI-NEXT:    v_mov_b32_e32 v0, s3
+; VI-NEXT:    v_mov_b32_e32 v1, s0
+; VI-NEXT:    v_mul_u32_u24_e32 v0, s2, v1
+; VI-NEXT:    v_mul_hi_u32_u24_e32 v1, s2, v1
+; VI-NEXT:    v_and_b32_e32 v1, 1, v1
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: test_umul24_i33:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
 ; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x34

llvm/test/CodeGen/AMDGPU/udiv.ll

 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_test_udiv64_mulhi_fold:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; VI-NEXT:    v_madak_f32 v2, 0, v2, 0x47c35000
 ; VI-NEXT:    v_rcp_f32_e32 v2, v2
-; VI-NEXT:    s_mov_b32 s4, 0xfffe7960
+; VI-NEXT:    s_mov_b32 s6, 0xfffe7960
 ; VI-NEXT:    v_mov_b32_e32 v9, 0
 ; VI-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
 ; VI-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
 ; VI-NEXT:    v_trunc_f32_e32 v3, v3
 ; VI-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
-; VI-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; VI-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; VI-NEXT:    v_mul_hi_u32 v4, v2, s4
-; VI-NEXT:    v_mul_lo_u32 v5, v3, s4
-; VI-NEXT:    v_mul_lo_u32 v6, v2, s4
-; VI-NEXT:    v_subrev_u32_e32 v4, vcc, v2, v4
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; VI-NEXT:    v_mul_lo_u32 v5, v2, v4
-; VI-NEXT:    v_mul_hi_u32 v7, v2, v6
-; VI-NEXT:    v_mul_hi_u32 v8, v2, v4
-; VI-NEXT:    v_mul_hi_u32 v10, v3, v4
-; VI-NEXT:    v_mul_lo_u32 v4, v3, v4
-; VI-NEXT:    v_add_u32_e32 v5, vcc, v7, v5
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; VI-NEXT:    v_mul_lo_u32 v8, v3, v6
-; VI-NEXT:    v_mul_hi_u32 v6, v3, v6
-; VI-NEXT:    v_add_u32_e32 v5, vcc, v5, v8
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; VI-NEXT:    v_addc_u32_e32 v6, vcc, v10, v9, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; VI-NEXT:    v_cvt_u32_f32_e32 v6, v2
+; VI-NEXT:    v_cvt_u32_f32_e32 v7, v3
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
+; VI-NEXT:    v_mul_lo_u32 v4, v7, s6
+; VI-NEXT:    v_subrev_u32_e32 v3, vcc, v6, v3
+; VI-NEXT:    v_add_u32_e32 v5, vcc, v4, v3
+; VI-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
+; VI-NEXT:    v_mul_hi_u32 v8, v6, v2
+; VI-NEXT:    v_add_u32_e32 v8, vcc, v8, v3
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
+; VI-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
+; VI-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, v10, v3, vcc
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v9, vcc
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
-; VI-NEXT:    v_mul_hi_u32 v4, v2, s4
-; VI-NEXT:    v_mul_lo_u32 v5, v3, s4
-; VI-NEXT:    v_mul_lo_u32 v6, v2, s4
-; VI-NEXT:    s_mov_b32 s4, 0x186a0
-; VI-NEXT:    v_subrev_u32_e32 v4, vcc, v2, v4
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v4, v5
-; VI-NEXT:    v_mul_lo_u32 v5, v2, v4
-; VI-NEXT:    v_mul_hi_u32 v7, v2, v6
-; VI-NEXT:    v_mul_hi_u32 v8, v2, v4
-; VI-NEXT:    v_mul_hi_u32 v10, v3, v4
-; VI-NEXT:    v_mul_lo_u32 v4, v3, v4
-; VI-NEXT:    v_add_u32_e32 v5, vcc, v7, v5
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; VI-NEXT:    v_mul_lo_u32 v8, v3, v6
-; VI-NEXT:    v_mul_hi_u32 v6, v3, v6
-; VI-NEXT:    v_add_u32_e32 v5, vcc, v5, v8
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; VI-NEXT:    v_addc_u32_e32 v6, vcc, v10, v9, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
+; VI-NEXT:    v_mul_lo_u32 v4, v7, s6
+; VI-NEXT:    s_mov_b32 s6, 0x186a0
+; VI-NEXT:    v_subrev_u32_e32 v3, vcc, v6, v3
+; VI-NEXT:    v_add_u32_e32 v5, vcc, v3, v4
+; VI-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
+; VI-NEXT:    v_mul_hi_u32 v8, v6, v2
+; VI-NEXT:    v_add_u32_e32 v8, vcc, v8, v3
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
+; VI-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
+; VI-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, v10, v3, vcc
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v9, vcc
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
-; VI-NEXT:    v_mul_lo_u32 v4, v0, v3
-; VI-NEXT:    v_mul_hi_u32 v5, v0, v2
-; VI-NEXT:    v_mul_hi_u32 v6, v0, v3
-; VI-NEXT:    v_mul_hi_u32 v7, v1, v3
-; VI-NEXT:    v_mul_lo_u32 v3, v1, v3
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; VI-NEXT:    v_mul_lo_u32 v6, v1, v2
-; VI-NEXT:    v_mul_hi_u32 v2, v1, v2
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v4, v6
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v5, v2, vcc
-; VI-NEXT:    v_addc_u32_e32 v4, vcc, v7, v9, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; VI-NEXT:    v_mul_lo_u32 v4, v3, s4
-; VI-NEXT:    v_mul_hi_u32 v5, v2, s4
-; VI-NEXT:    v_mul_lo_u32 v6, v2, s4
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; VI-NEXT:    v_sub_u32_e32 v0, vcc, v0, v6
-; VI-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
-; VI-NEXT:    v_subrev_u32_e32 v4, vcc, s4, v0
-; VI-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, v6, v2
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v3, vcc
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, v5, 0
+; VI-NEXT:    v_mul_hi_u32 v6, v0, v4
+; VI-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, v4, 0
+; VI-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, v5, 0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, v6, v2
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v3, vcc
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, v2, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; VI-NEXT:    v_mul_lo_u32 v6, v5, s6
+; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, s6, 0
 ; VI-NEXT:    s_mov_b32 s4, 0x1869f
-; VI-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v4
-; VI-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; VI-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; VI-NEXT:    v_add_u32_e32 v5, vcc, 2, v2
-; VI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v7, vcc, 1, v2
+; VI-NEXT:    v_add_u32_e32 v3, vcc, v3, v6
+; VI-NEXT:    v_sub_u32_e32 v0, vcc, v0, v2
+; VI-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; VI-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v0
+; VI-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v1, vcc
+; VI-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; VI-NEXT:    v_cndmask_b32_e32 v2, -1, v2, vcc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 2, v4
+; VI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v7, vcc, 1, v4
 ; VI-NEXT:    v_cmp_lt_u32_e64 s[4:5], s4, v0
-; VI-NEXT:    v_addc_u32_e32 v8, vcc, 0, v3, vcc
+; VI-NEXT:    v_addc_u32_e32 v8, vcc, 0, v5, vcc
 ; VI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[4:5]
 ; VI-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; VI-NEXT:    v_cndmask_b32_e64 v0, -1, v0, s[4:5]
-; VI-NEXT:    v_cndmask_b32_e32 v4, v7, v5, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; VI-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
 ; VI-NEXT:    v_cndmask_b32_e32 v1, v8, v6, vcc
-; VI-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; VI-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e64 v0, v4, v2, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN-LABEL: v_test_udiv64_mulhi_fold:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GCN-NEXT:    v_madak_f32 v2, 0, v2, 0x47c35000
 ; GCN-NEXT:    v_rcp_f32_e32 v2, v2
-; GCN-NEXT:    s_mov_b32 s4, 0xfffe7960
+; GCN-NEXT:    s_mov_b32 s6, 0xfffe7960
 ; GCN-NEXT:    v_mov_b32_e32 v9, 0
 ; GCN-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
 ; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
 ; GCN-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
-; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v4, v2, s4
-; GCN-NEXT:    v_mul_lo_u32 v5, v3, s4
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s4
-; GCN-NEXT:    v_subrev_u32_e32 v4, vcc, v2, v4
-; GCN-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, v2, v6
-; GCN-NEXT:    v_mul_hi_u32 v8, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v10, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_u32_e32 v5, vcc, v7, v5
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_u32_e32 v5, vcc, v5, v8
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v10, v9, vcc
-; GCN-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GCN-NEXT:    v_cvt_u32_f32_e32 v6, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v7, v3
+; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
+; GCN-NEXT:    v_mul_lo_u32 v4, v7, s6
+; GCN-NEXT:    v_subrev_u32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_add_u32_e32 v5, vcc, v4, v3
+; GCN-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
+; GCN-NEXT:    v_mul_hi_u32 v8, v6, v2
+; GCN-NEXT:    v_add_u32_e32 v8, vcc, v8, v3
+; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
+; GCN-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0
+; GCN-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v10, v3, vcc
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v9, vcc
 ; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
-; GCN-NEXT:    v_mul_hi_u32 v4, v2, s4
-; GCN-NEXT:    v_mul_lo_u32 v5, v3, s4
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s4
-; GCN-NEXT:    s_mov_b32 s4, 0x186a0
-; GCN-NEXT:    v_subrev_u32_e32 v4, vcc, v2, v4
-; GCN-NEXT:    v_add_u32_e32 v4, vcc, v4, v5
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, v2, v6
-; GCN-NEXT:    v_mul_hi_u32 v8, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v10, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_u32_e32 v5, vcc, v7, v5
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_u32_e32 v5, vcc, v5, v8
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v10, v9, vcc
-; GCN-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
+; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
+; GCN-NEXT:    v_mul_lo_u32 v4, v7, s6
+; GCN-NEXT:    s_mov_b32 s6, 0x186a0
+; GCN-NEXT:    v_subrev_u32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_add_u32_e32 v5, vcc, v3, v4
+; GCN-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
+; GCN-NEXT:    v_mul_hi_u32 v8, v6, v2
+; GCN-NEXT:    v_add_u32_e32 v8, vcc, v8, v3
+; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
+; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
+; GCN-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0
+; GCN-NEXT:    v_add_u32_e32 v2, vcc, v8, v2
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v10, v3, vcc
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v9, vcc
 ; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
-; GCN-NEXT:    v_mul_hi_u32 v7, v1, v3
-; GCN-NEXT:    v_mul_lo_u32 v3, v1, v3
-; GCN-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v2
-; GCN-NEXT:    v_mul_hi_u32 v2, v1, v2
-; GCN-NEXT:    v_add_u32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v2, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v9, vcc
-; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, s4
-; GCN-NEXT:    v_mul_hi_u32 v5, v2, s4
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s4
-; GCN-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v0, v6
-; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
-; GCN-NEXT:    v_subrev_u32_e32 v4, vcc, s4, v0
-; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v1, vcc
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GCN-NEXT:    v_add_u32_e32 v4, vcc, v6, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v3, vcc
+; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v0, v5, 0
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v4
+; GCN-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v1, v4, 0
+; GCN-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, v5, 0
+; GCN-NEXT:    v_add_u32_e32 v2, vcc, v6, v2
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v3, vcc
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v9, vcc
+; GCN-NEXT:    v_add_u32_e32 v4, vcc, v2, v4
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v3, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v5, s6
+; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, s6, 0
 ; GCN-NEXT:    s_mov_b32 s4, 0x1869f
-; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v4
-; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
-; GCN-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN-NEXT:    v_add_u32_e32 v5, vcc, 2, v2
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_u32_e32 v7, vcc, 1, v2
+; GCN-NEXT:    v_add_u32_e32 v3, vcc, v3, v6
+; GCN-NEXT:    v_sub_u32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; GCN-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v0
+; GCN-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v2, -1, v2, vcc
+; GCN-NEXT:    v_add_u32_e32 v3, vcc, 2, v4
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v5, vcc
+; GCN-NEXT:    v_add_u32_e32 v7, vcc, 1, v4
 ; GCN-NEXT:    v_cmp_lt_u32_e64 s[4:5], s4, v0
-; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v3, vcc
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[4:5]
 ; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, -1, v0, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e32 v4, v7, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v8, v6, vcc
-; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
-; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v2, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1030-LABEL: v_test_udiv64_mulhi_fold:
 ; GFX1030:       ; %bb.0:
 ; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1030-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1030-NEXT:    s_mov_b32 s4, 0x346d900
 ; GFX1030-NEXT:    s_mov_b32 s5, 0xfffe7960

 ; GFX1030-NEXT:    v_mul_lo_u32 v3, s4, v3
 ; GFX1030-NEXT:    v_add_co_u32 v5, vcc_lo, v5, v6
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v7, vcc_lo
 ; GFX1030-NEXT:    v_add_co_u32 v4, vcc_lo, v5, v4
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v6, v8, vcc_lo
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v9, vcc_lo
 ; GFX1030-NEXT:    v_add_co_u32 v3, vcc_lo, v4, v3
 ; GFX1030-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v5, vcc_lo
-; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v3
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s4, v4, vcc_lo
+; GFX1030-NEXT:    v_add_co_u32 v5, vcc_lo, v2, v3
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, s4, v4, vcc_lo
+; GFX1030-NEXT:    v_mul_hi_u32 v8, v0, v5
+; GFX1030-NEXT:    v_mad_u64_u32 v[4:5], s4, v1, v5, 0
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], s4, v0, v6, 0
+; GFX1030-NEXT:    v_mad_u64_u32 v[6:7], s4, v1, v6, 0
 ; GFX1030-NEXT:    s_mov_b32 s4, 0x186a0
-; GFX1030-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GFX1030-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX1030-NEXT:    v_mul_lo_u32 v5, v0, v3
-; GFX1030-NEXT:    v_mul_hi_u32 v6, v0, v3
-; GFX1030-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX1030-NEXT:    v_mul_hi_u32 v8, v1, v3
-; GFX1030-NEXT:    v_mul_lo_u32 v3, v1, v3
-; GFX1030-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v5
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v6, vcc_lo
-; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v2
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v5, v7, vcc_lo
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v8, vcc_lo
-; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v3
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v4, vcc_lo
-; GFX1030-NEXT:    v_mul_hi_u32 v4, v2, s4
-; GFX1030-NEXT:    v_mul_lo_u32 v6, v2, s4
-; GFX1030-NEXT:    v_mul_lo_u32 v5, v3, s4
-; GFX1030-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v6
-; GFX1030-NEXT:    v_add_nc_u32_e32 v4, v4, v5
-; GFX1030-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
-; GFX1030-NEXT:    v_sub_co_u32 v4, vcc_lo, v0, s4
+; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v8, v2
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX1030-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v3, v5, vcc_lo
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v7, vcc_lo
+; GFX1030-NEXT:    v_add_co_u32 v4, vcc_lo, v2, v6
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v3, vcc_lo
+; GFX1030-NEXT:    v_mad_u64_u32 v[2:3], s5, v4, s4, 0
+; GFX1030-NEXT:    v_mul_lo_u32 v6, v5, s4
+; GFX1030-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v2
+; GFX1030-NEXT:    v_add_nc_u32_e32 v3, v3, v6
+; GFX1030-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX1030-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
 ; GFX1030-NEXT:    s_mov_b32 s4, 0x1869f
-; GFX1030-NEXT:    v_subrev_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
-; GFX1030-NEXT:    v_cmp_lt_u32_e32 vcc_lo, s4, v4
-; GFX1030-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc_lo
-; GFX1030-NEXT:    v_add_co_u32 v6, vcc_lo, v2, 2
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v3, vcc_lo
+; GFX1030-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
+; GFX1030-NEXT:    v_cmp_lt_u32_e32 vcc_lo, s4, v2
+; GFX1030-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc_lo
+; GFX1030-NEXT:    v_add_co_u32 v6, vcc_lo, v4, 2
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
 ; GFX1030-NEXT:    v_cmp_lt_u32_e32 vcc_lo, s4, v0
 ; GFX1030-NEXT:    v_cmp_eq_u32_e64 s4, 0, v1
 ; GFX1030-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
-; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v5
+; GFX1030-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
 ; GFX1030-NEXT:    v_cndmask_b32_e64 v0, -1, v0, s4
-; GFX1030-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX1030-NEXT:    v_add_co_u32 v5, vcc_lo, v2, 1
-; GFX1030-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v3, vcc_lo
-; GFX1030-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, v5, v6, vcc_lo
-; GFX1030-NEXT:    v_cndmask_b32_e32 v4, v8, v7, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e32 v2, -1, v2, vcc_lo
+; GFX1030-NEXT:    v_add_co_u32 v3, vcc_lo, v4, 1
+; GFX1030-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v5, vcc_lo
+; GFX1030-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX1030-NEXT:    v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e32 v2, v8, v7, vcc_lo
 ; GFX1030-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX1030-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-; GFX1030-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc_lo
+; GFX1030-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc_lo
 ; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; EG-LABEL: v_test_udiv64_mulhi_fold:
 ; EG:       ; %bb.0:
 ; EG-NEXT:    CF_END
 ; EG-NEXT:    PAD
   %d = udiv i64 %arg, 100000
   ret i64 %d
 }

llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll

; GFX9-O3-NEXT:    s_setpc_b64 s[36:37]
   call void @llvm.amdgcn.raw.buffer.store.i32(i32 %tmp137, <4 x i32> %tmp14, i32 4, i32 0, i32 0)
   ret void
 }
 
 define amdgpu_gfx i64 @strict_wwm_called_i64(i64 %a) noinline {
 ; GFX9-O0-LABEL: strict_wwm_called_i64:
 ; GFX9-O0:       ; %bb.0:
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v0
-; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v1
-; GFX9-O0-NEXT:    ; kill: def $vgpr0_vgpr1 killed $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v7
-; GFX9-O0-NEXT:    v_add_co_u32_e64 v4, s[34:35], v2, v3
+; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-O0-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-O0-NEXT:    ; kill: def $vgpr0_vgpr1 killed $vgpr2_vgpr3 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v3
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v3
+; GFX9-O0-NEXT:    v_add_co_u32_e64 v4, s[34:35], v4, v5
 ; GFX9-O0-NEXT:    v_addc_co_u32_e64 v0, s[34:35], v0, v1, s[34:35]
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX9-O0-NEXT:    s_mov_b32 s34, 32
-; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v6
-; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v2
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX9-O0-NEXT:    v_lshrrev_b64 v[0:1], s34, v[0:1]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-O0-NEXT:    v_mul_lo_u32 v2, v0, v1
-; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT:    v_mul_hi_u32 v1, v0, v6
-; GFX9-O0-NEXT:    v_lshrrev_b64 v[7:8], s34, v[4:5]
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v7
-; GFX9-O0-NEXT:    v_mul_lo_u32 v3, v3, v6
-; GFX9-O0-NEXT:    v_add3_u32 v1, v1, v2, v3
+; GFX9-O0-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT:    v_lshrrev_b64 v[6:7], s34, v[4:5]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v6
+; GFX9-O0-NEXT:    v_mul_lo_u32 v2, v2, v3
+; GFX9-O0-NEXT:    v_mad_u64_u32 v[6:7], s[36:37], v0, v3, 0
+; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v7
+; GFX9-O0-NEXT:    v_add3_u32 v0, v0, v1, v2
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr35
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr36
-; GFX9-O0-NEXT:    v_mov_b32_e32 v3, s35
-; GFX9-O0-NEXT:    ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v3
-; GFX9-O0-NEXT:    v_lshlrev_b64 v[1:2], s34, v[1:2]
+; GFX9-O0-NEXT:    v_mov_b32_e32 v2, s35
+; GFX9-O0-NEXT:    ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v2
+; GFX9-O0-NEXT:    v_lshlrev_b64 v[1:2], s34, v[0:1]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v2
-; GFX9-O0-NEXT:    v_mul_lo_u32 v6, v0, v6
+; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
 ; GFX9-O0-NEXT:    s_mov_b32 s35, 0
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-O0-NEXT:    ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v7, v0
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v7
 ; GFX9-O0-NEXT:    v_or_b32_e32 v0, v0, v3
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v6

 ; GFX9-O0-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-O3-LABEL: strict_wwm_called_i64:
 ; GFX9-O3:       ; %bb.0:
 ; GFX9-O3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-O3-NEXT:    v_add_co_u32_e32 v2, vcc, v0, v0
 ; GFX9-O3-NEXT:    v_addc_co_u32_e32 v3, vcc, v1, v1, vcc
 ; GFX9-O3-NEXT:    v_mul_lo_u32 v4, v3, v0
-; GFX9-O3-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX9-O3-NEXT:    v_mul_hi_u32 v5, v2, v0
-; GFX9-O3-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX9-O3-NEXT:    v_add3_u32 v1, v5, v1, v4
+; GFX9-O3-NEXT:    v_mul_lo_u32 v5, v2, v1
+; GFX9-O3-NEXT:    v_mad_u64_u32 v[0:1], s[34:35], v2, v0, 0
+; GFX9-O3-NEXT:    v_add3_u32 v1, v1, v5, v4
 ; GFX9-O3-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
 ; GFX9-O3-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX9-O3-NEXT:    s_setpc_b64 s[30:31]
   %add = add i64 %a, %a
   %mul = mul i64 %add, %a
   %sub = sub i64 %mul, %add
   ret i64 %sub
 }
 
 define amdgpu_gfx void @strict_wwm_call_i64(<4 x i32> inreg %tmp14, i64 inreg %arg) {
 ; GFX9-O0-LABEL: strict_wwm_call_i64:
 ; GFX9-O0:       ; %bb.0:
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-O0-NEXT:    s_or_saveexec_b64 s[34:35], -1
-; GFX9-O0-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O0-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[34:35]
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s33, 8
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s33, 8
 ; GFX9-O0-NEXT:    s_mov_b32 s33, s32
 ; GFX9-O0-NEXT:    s_add_i32 s32, s32, 0xc00
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s30, 0
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s31, 1
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s30, 0
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s31, 1
 ; GFX9-O0-NEXT:    s_mov_b32 s34, s8
 ; GFX9-O0-NEXT:    s_mov_b32 s36, s4
 ; GFX9-O0-NEXT:    ; kill: def $sgpr36 killed $sgpr36 def $sgpr36_sgpr37_sgpr38_sgpr39
 ; GFX9-O0-NEXT:    s_mov_b32 s37, s5
 ; GFX9-O0-NEXT:    s_mov_b32 s38, s6
 ; GFX9-O0-NEXT:    s_mov_b32 s39, s7
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s36, 2
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s37, 3
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s38, 4
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s39, 5
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s36, 2
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s37, 3
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s38, 4
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s39, 5
 ; GFX9-O0-NEXT:    ; kill: def $sgpr34 killed $sgpr34 def $sgpr34_sgpr35
 ; GFX9-O0-NEXT:    s_mov_b32 s35, s9
 ; GFX9-O0-NEXT:    ; kill: def $sgpr30_sgpr31 killed $sgpr34_sgpr35
 ; GFX9-O0-NEXT:    s_mov_b64 s[30:31], 0
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v0, s34
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, s35
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v0
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v1
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v0
 ; GFX9-O0-NEXT:    s_not_b64 exec, exec
-; GFX9-O0-NEXT:    v_mov_b32_e32 v9, s30
-; GFX9-O0-NEXT:    v_mov_b32_e32 v10, s31
+; GFX9-O0-NEXT:    v_mov_b32_e32 v8, s30
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, s31
 ; GFX9-O0-NEXT:    s_not_b64 exec, exec
 ; GFX9-O0-NEXT:    s_or_saveexec_b64 s[30:31], -1
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s30, 6
-; GFX9-O0-NEXT:    v_writelane_b32 v11, s31, 7
-; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v9
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s30, 6
+; GFX9-O0-NEXT:    v_writelane_b32 v10, s31, 7
+; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v8
 ; GFX9-O0-NEXT:    s_mov_b32 s30, 32
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr34_sgpr35
-; GFX9-O0-NEXT:    v_lshrrev_b64 v[3:4], s30, v[9:10]
+; GFX9-O0-NEXT:    v_lshrrev_b64 v[3:4], s30, v[8:9]
 ; GFX9-O0-NEXT:    s_getpc_b64 s[30:31]
 ; GFX9-O0-NEXT:    s_add_u32 s30, s30, strict_wwm_called_i64@gotpcrel32@lo+4
 ; GFX9-O0-NEXT:    s_addc_u32 s31, s31, strict_wwm_called_i64@gotpcrel32@hi+12
 ; GFX9-O0-NEXT:    s_load_dwordx2 s[30:31], s[30:31], 0x0
 ; GFX9-O0-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-O0-NEXT:    s_mov_b64 s[36:37], s[0:1]
 ; GFX9-O0-NEXT:    s_mov_b64 s[0:1], s[36:37]
 ; GFX9-O0-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX9-O0-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-O0-NEXT:    s_swappc_b64 s[30:31], s[30:31]
-; GFX9-O0-NEXT:    v_readlane_b32 s34, v11, 6
-; GFX9-O0-NEXT:    v_readlane_b32 s35, v11, 7
-; GFX9-O0-NEXT:    v_readlane_b32 s36, v11, 2
-; GFX9-O0-NEXT:    v_readlane_b32 s37, v11, 3
-; GFX9-O0-NEXT:    v_readlane_b32 s38, v11, 4
-; GFX9-O0-NEXT:    v_readlane_b32 s39, v11, 5
-; GFX9-O0-NEXT:    v_readlane_b32 s30, v11, 0
-; GFX9-O0-NEXT:    v_readlane_b32 s31, v11, 1
+; GFX9-O0-NEXT:    v_readlane_b32 s34, v10, 6
+; GFX9-O0-NEXT:    v_readlane_b32 s35, v10, 7
+; GFX9-O0-NEXT:    v_readlane_b32 s36, v10, 2
+; GFX9-O0-NEXT:    v_readlane_b32 s37, v10, 3
+; GFX9-O0-NEXT:    v_readlane_b32 s38, v10, 4
+; GFX9-O0-NEXT:    v_readlane_b32 s39, v10, 5
+; GFX9-O0-NEXT:    v_readlane_b32 s30, v10, 0
+; GFX9-O0-NEXT:    v_readlane_b32 s31, v10, 1
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v9
-; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v10
+; GFX9-O0-NEXT:    v_mov_b32_e32 v4, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v9
 ; GFX9-O0-NEXT:    v_add_co_u32_e64 v2, s[40:41], v2, v4
 ; GFX9-O0-NEXT:    v_addc_co_u32_e64 v3, s[40:41], v3, v5, s[40:41]
 ; GFX9-O0-NEXT:    s_mov_b64 exec, s[34:35]
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX9-O0-NEXT:    s_mov_b32 s34, 0
 ; GFX9-O0-NEXT:    buffer_store_dwordx2 v[0:1], off, s[36:39], s34 offset:4
 ; GFX9-O0-NEXT:    s_add_i32 s32, s32, 0xfffff400
-; GFX9-O0-NEXT:    v_readlane_b32 s33, v11, 8
+; GFX9-O0-NEXT:    v_readlane_b32 s33, v10, 8
 ; GFX9-O0-NEXT:    s_or_saveexec_b64 s[34:35], -1
-; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
 ; GFX9-O0-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
 ; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
 ; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_nop 0
 ; GFX9-O0-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload

 ; GFX9-O3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O3-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX9-O3-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GFX9-O3-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; GFX9-O3-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
 ; GFX9-O3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-O3-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O3-NEXT:    s_mov_b64 exec, s[34:35]
-; GFX9-O3-NEXT:    s_mov_b32 s38, s33
+; GFX9-O3-NEXT:    s_mov_b32 s40, s33
 ; GFX9-O3-NEXT:    s_mov_b32 s33, s32
 ; GFX9-O3-NEXT:    s_addk_i32 s32, 0x800
 ; GFX9-O3-NEXT:    s_mov_b64 s[36:37], s[30:31]
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v6, s8
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v7, s9
 ; GFX9-O3-NEXT:    s_not_b64 exec, exec
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-O3-NEXT:    s_not_b64 exec, exec
-; GFX9-O3-NEXT:    s_or_saveexec_b64 s[34:35], -1
+; GFX9-O3-NEXT:    s_or_saveexec_b64 s[38:39], -1
 ; GFX9-O3-NEXT:    s_getpc_b64 s[30:31]
 ; GFX9-O3-NEXT:    s_add_u32 s30, s30, strict_wwm_called_i64@gotpcrel32@lo+4
 ; GFX9-O3-NEXT:    s_addc_u32 s31, s31, strict_wwm_called_i64@gotpcrel32@hi+12
 ; GFX9-O3-NEXT:    s_load_dwordx2 s[30:31], s[30:31], 0x0
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v1, v7
 ; GFX9-O3-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-O3-NEXT:    s_swappc_b64 s[30:31], s[30:31]
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX9-O3-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v6
 ; GFX9-O3-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v7, vcc
-; GFX9-O3-NEXT:    s_mov_b64 exec, s[34:35]
+; GFX9-O3-NEXT:    s_mov_b64 exec, s[38:39]
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-O3-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX9-O3-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0 offset:4
 ; GFX9-O3-NEXT:    s_addk_i32 s32, 0xf800
-; GFX9-O3-NEXT:    s_mov_b32 s33, s38
+; GFX9-O3-NEXT:    s_mov_b32 s33, s40
 ; GFX9-O3-NEXT:    s_or_saveexec_b64 s[30:31], -1
 ; GFX9-O3-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; GFX9-O3-NEXT:    s_nop 0
 ; GFX9-O3-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; GFX9-O3-NEXT:    s_nop 0
 ; GFX9-O3-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; GFX9-O3-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX9-O3-NEXT:    s_nop 0

llvm/test/CodeGen/AMDGPU/wwm-reserved.ll

 
 ; GFX9-LABEL: {{^}}call_i64:
 define amdgpu_kernel void @call_i64(<4 x i32> inreg %tmp14, i64 inreg %arg) {
 ; GFX9: s_load_dwordx2 s{{\[}}[[ARG_LO:[0-9]+]]:[[ARG_HI:[0-9]+]]{{\]}}
 
 ; GFX9-O0: s_mov_b64 s{{\[}}[[ZERO_LO:[0-9]+]]:[[ZERO_HI:[0-9]+]]{{\]}}, 0{{$}}
 ; GFX9-O0: v_mov_b32_e32 v0, s[[ARG_LO]]
 ; GFX9-O0: v_mov_b32_e32 v1, s[[ARG_HI]]
-; GFX9-O0-DAG: v_mov_b32_e32 v10, v1
-; GFX9-O0-DAG: v_mov_b32_e32 v9, v0
+; GFX9-O0-DAG: v_mov_b32_e32 v9, v1
+; GFX9-O0-DAG: v_mov_b32_e32 v8, v0
 
 ; GFX9-O3-DAG: v_mov_b32_e32 v7, s[[ARG_HI]]
 ; GFX9-O3-DAG: v_mov_b32_e32 v6, s[[ARG_LO]]
 
 ; GFX9: s_not_b64 exec, exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_LO]]
-; GFX9-O0-NEXT: v_mov_b32_e32 v10, s[[ZERO_HI]]
+; GFX9-O0-NEXT: v_mov_b32_e32 v8, s[[ZERO_LO]]
+; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_HI]]
 ; GFX9-O3-NEXT: v_mov_b32_e32 v6, 0
 ; GFX9-O3-NEXT: v_mov_b32_e32 v7, 0
 ; GFX9-NEXT: s_not_b64 exec, exec
   %tmp107 = tail call i64 @llvm.amdgcn.set.inactive.i64(i64 %arg, i64 0)
 ; GFX9: s_swappc_b64
   %tmp134 = call i64 @called_i64(i64 %tmp107)
   %tmp136 = add i64 %tmp134, %tmp107
   %tmp137 = tail call i64 @llvm.amdgcn.wwm.i64(i64 %tmp136)

 
 ; GFX9-LABEL: {{^}}strict_wwm_call_i64:
 define amdgpu_kernel void @strict_wwm_call_i64(<4 x i32> inreg %tmp14, i64 inreg %arg) {
 ; GFX9: s_load_dwordx2 s{{\[}}[[ARG_LO:[0-9]+]]:[[ARG_HI:[0-9]+]]{{\]}}
 
 ; GFX9-O0: s_mov_b64 s{{\[}}[[ZERO_LO:[0-9]+]]:[[ZERO_HI:[0-9]+]]{{\]}}, 0{{$}}
 ; GFX9-O0: v_mov_b32_e32 v0, s[[ARG_LO]]
 ; GFX9-O0: v_mov_b32_e32 v1, s[[ARG_HI]]
-; GFX9-O0-DAG: v_mov_b32_e32 v10, v1
-; GFX9-O0-DAG: v_mov_b32_e32 v9, v0
+; GFX9-O0-DAG: v_mov_b32_e32 v9, v1
+; GFX9-O0-DAG: v_mov_b32_e32 v8, v0
 
 ; GFX9-O3-DAG: v_mov_b32_e32 v7, s[[ARG_HI]]
 ; GFX9-O3-DAG: v_mov_b32_e32 v6, s[[ARG_LO]]
 
 ; GFX9: s_not_b64 exec, exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_LO]]
-; GFX9-O0-NEXT: v_mov_b32_e32 v10, s[[ZERO_HI]]
+; GFX9-O0-NEXT: v_mov_b32_e32 v8, s[[ZERO_LO]]
+; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_HI]]
 ; GFX9-O3-NEXT: v_mov_b32_e32 v6, 0
 ; GFX9-O3-NEXT: v_mov_b32_e32 v7, 0
 ; GFX9-NEXT: s_not_b64 exec, exec
   %tmp107 = tail call i64 @llvm.amdgcn.set.inactive.i64(i64 %arg, i64 0)
 ; GFX9: s_swappc_b64
   %tmp134 = call i64 @strict_wwm_called_i64(i64 %tmp107)
   %tmp136 = add i64 %tmp134, %tmp107
   %tmp137 = tail call i64 @llvm.amdgcn.strict.wwm.i64(i64 %tmp136)