diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h @@ -231,6 +231,7 @@ void SelectUADDO_USUBO(SDNode *N); void SelectDIV_SCALE(SDNode *N); void SelectMAD_64_32(SDNode *N); + void SelectMUL_LOHI(SDNode *N); void SelectFMA_W_CHAIN(SDNode *N); void SelectFMUL_W_CHAIN(SDNode *N); SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -654,6 +654,9 @@ SelectMAD_64_32(N); return; } + case ISD::SMUL_LOHI: + case ISD::UMUL_LOHI: + return SelectMUL_LOHI(N); case ISD::CopyToReg: { const SITargetLowering& Lowering = *static_cast(getTargetLowering()); @@ -1001,6 +1004,32 @@ CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); } +// We need to handle this here because tablegen doesn't support matching +// instructions with multiple outputs. +void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) { + SDLoc SL(N); + bool Signed = N->getOpcode() == ISD::SMUL_LOHI; + unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64; + + SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64); + SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); + SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Zero, Clamp}; + SDNode *Mad = CurDAG->getMachineNode(Opc, SL, N->getVTList(), Ops); + if (!SDValue(N, 0).use_empty()) { + SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32); + SDNode *Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL, + MVT::i32, SDValue(Mad, 0), Sub0); + ReplaceUses(SDValue(N, 0), SDValue(Lo, 0)); + } + if (!SDValue(N, 1).use_empty()) { + SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32); + SDNode *Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL, + MVT::i32, SDValue(Mad, 0), Sub1); + ReplaceUses(SDValue(N, 1), SDValue(Hi, 0)); + } + CurDAG->RemoveDeadNode(N); +} + bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const { if (!isUInt<16>(Offset)) return false; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -91,6 +91,7 @@ SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performTruncateCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performMulLoHiCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -594,6 +594,8 @@ setTargetDAGCombine(ISD::SRL); setTargetDAGCombine(ISD::TRUNCATE); setTargetDAGCombine(ISD::MUL); + setTargetDAGCombine(ISD::SMUL_LOHI); + setTargetDAGCombine(ISD::UMUL_LOHI); setTargetDAGCombine(ISD::MULHU); setTargetDAGCombine(ISD::MULHS); setTargetDAGCombine(ISD::SELECT); @@ -3462,6 +3464,50 @@ return DAG.getSExtOrTrunc(Mul, DL, VT); } +SDValue +AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + if (N->getValueType(0) != MVT::i32) + return SDValue(); + + SelectionDAG &DAG = DCI.DAG; + SDLoc DL(N); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + // SimplifyDemandedBits has the annoying habit of turning useful zero_extends + // in the source into any_extends if the result of the mul is truncated. Since + // we can assume the high bits are whatever we want, use the underlying value + // to avoid the unknown high bits from interfering. + if (N0.getOpcode() == ISD::ANY_EXTEND) + N0 = N0.getOperand(0); + if (N1.getOpcode() == ISD::ANY_EXTEND) + N1 = N1.getOperand(0); + + // Try to use two fast 24-bit multiplies (one for each half of the result) + // instead of one slow extending multiply. + unsigned LoOpcode, HiOpcode; + if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { + N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); + N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); + LoOpcode = AMDGPUISD::MUL_U24; + HiOpcode = AMDGPUISD::MULHI_U24; + } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { + N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); + N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); + LoOpcode = AMDGPUISD::MUL_I24; + HiOpcode = AMDGPUISD::MULHI_I24; + } else { + return SDValue(); + } + + SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1); + SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1); + DCI.CombineTo(N, Lo, Hi); + return SDValue(N, 0); +} + SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const { EVT VT = N->getValueType(0); @@ -4103,6 +4149,9 @@ return performTruncateCombine(N, DCI); case ISD::MUL: return performMulCombine(N, DCI); + case ISD::SMUL_LOHI: + case ISD::UMUL_LOHI: + return performMulLoHiCombine(N, DCI); case ISD::MULHS: return performMulhsCombine(N, DCI); case ISD::MULHU: diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -135,6 +135,7 @@ SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const; SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const; SDValue lowerXMULO(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const; SDValue getSegmentAperture(unsigned AS, const SDLoc &DL, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -809,6 +809,9 @@ setOperationAction(ISD::SMULO, MVT::i64, Custom); setOperationAction(ISD::UMULO, MVT::i64, Custom); + setOperationAction(ISD::SMUL_LOHI, MVT::i32, Custom); + setOperationAction(ISD::UMUL_LOHI, MVT::i32, Custom); + setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom); setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom); @@ -4691,6 +4694,9 @@ case ISD::SMULO: case ISD::UMULO: return lowerXMULO(Op, DAG); + case ISD::SMUL_LOHI: + case ISD::UMUL_LOHI: + return lowerXMUL_LOHI(Op, DAG); case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); } @@ -5304,6 +5310,21 @@ return DAG.getMergeValues({ Result, Overflow }, SL); } +SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const { + if (Op->isDivergent()) { + // Select to V_MAD_[IU]64_[IU]32. + return Op; + } + if (Subtarget->hasSMulHi()) { + // Expand to S_MUL_I32 + S_MUL_HI_[IU]32. + return SDValue(); + } + // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to + // calculate the high part, so we might as well do the whole thing with + // V_MAD_[IU]64_[IU]32. + return Op; +} + SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const { if (!Subtarget->isTrapHandlerEnabled() || Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll @@ -9256,113 +9256,101 @@ ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v1 ; GFX6-NEXT: v_mov_b32_e32 v7, 0 +; GFX6-NEXT: s_movk_i32 s10, 0x11f +; GFX6-NEXT: v_mul_lo_u32 v2, v4, s4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, v5, s5 +; GFX6-NEXT: s_mov_b32 s11, 0x976a7377 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v1 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v5, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v5, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v2, v4, s4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, v5, s5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v4, v0 ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s5 -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s5 -; GFX6-NEXT: s_movk_i32 s8, 0x11f -; GFX6-NEXT: s_mov_b32 s9, 0x976a7377 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s5 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v4, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v4, v3, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s5 -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s5 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc ; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s2, v2 ; GFX6-NEXT: s_mov_b32 s4, s0 -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s5 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_mul_lo_u32 v4, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX6-NEXT: s_mov_b32 s5, s1 -; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v3, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s2, v0 -; GFX6-NEXT: v_mul_hi_u32 v4, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v5, s3, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s3, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s3, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s8 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s9 -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s9 -; GFX6-NEXT: v_mov_b32_e32 v5, s8 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s9 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s3, v2 -; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s2, v3 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s3, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[8:9], s3, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v4, v2, s10 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v2, s11, 0 +; GFX6-NEXT: v_mul_lo_u32 v5, v3, s11 +; GFX6-NEXT: s_mov_b32 s8, 0x976a7376 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s3, v1 +; GFX6-NEXT: v_mov_b32_e32 v5, s10 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 ; GFX6-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s9, v3 +; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s11, v0 ; GFX6-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] ; GFX6-NEXT: s_movk_i32 s2, 0x11e ; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s2, v4 -; GFX6-NEXT: s_mov_b32 s9, 0x976a7376 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s9, v5 +; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s8, v5 ; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s8, v4 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s10, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] -; GFX6-NEXT: v_add_i32_e64 v5, s[0:1], 2, v0 -; GFX6-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1] -; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 1, v0 -; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1] +; GFX6-NEXT: v_add_i32_e64 v5, s[0:1], 2, v2 +; GFX6-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v3, s[0:1] +; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 1, v2 +; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1] ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v4, v8, v6, s[0:1] ; GFX6-NEXT: v_mov_b32_e32 v6, s3 -; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v6, v2, vcc -; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s2, v2 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s2, v1 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc -; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s9, v3 -; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s8, v2 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc -; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; GFX6-NEXT: v_cndmask_b32_e64 v2, v7, v5, s[0:1] -; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s8, v0 +; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s10, v1 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc +; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GFX6-NEXT: v_cndmask_b32_e64 v0, v7, v5, s[0:1] +; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc +; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; @@ -9777,106 +9765,94 @@ ; GFX6-NEXT: v_mov_b32_e32 v0, 0x4f800000 ; GFX6-NEXT: v_madak_f32 v0, 0, v0, 0x457ff000 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_movk_i32 s6, 0xf001 +; GFX6-NEXT: s_movk_i32 s4, 0xf001 ; GFX6-NEXT: v_mov_b32_e32 v8, 0 ; GFX6-NEXT: v_mov_b32_e32 v7, 0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GFX6-NEXT: s_movk_i32 s10, 0xfff +; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s4, 0 +; GFX6-NEXT: v_mul_lo_u32 v2, v5, s4 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, v4, v1 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v2, v1 +; GFX6-NEXT: v_mul_hi_u32 v3, v4, v0 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v6, 0 +; GFX6-NEXT: v_add_i32_e32 v9, vcc, v3, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v5, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v10, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v5, v6, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v9, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s4, 0 +; GFX6-NEXT: v_mul_lo_u32 v2, v5, s4 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, v4, v1 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v1, v2 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v4, v0 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: v_mul_hi_u32 v2, v0, s6 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, s6 -; GFX6-NEXT: v_mul_lo_u32 v4, v0, s6 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v5, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[8:9], v5, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc ; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s2, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s2, v2 ; GFX6-NEXT: s_lshr_b64 s[8:9], s[0:1], 12 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v3, vcc -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_hi_u32 v2, v0, s6 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, s6 -; GFX6-NEXT: v_mul_lo_u32 v4, v0, s6 -; GFX6-NEXT: s_movk_i32 s0, 0xfff -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, v0, v4 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v5, v3 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v4, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s2, v0 -; GFX6-NEXT: v_mul_hi_u32 v4, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v5, s3, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s3, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s3, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s0 -; GFX6-NEXT: v_mul_hi_u32 v5, v0, s0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0 -; GFX6-NEXT: v_mul_lo_u32 v8, v0, s0 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; GFX6-NEXT: v_add_i32_e32 v6, vcc, 1, v0 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_mov_b32_e32 v5, s3 -; GFX6-NEXT: v_sub_i32_e32 v8, vcc, s2, v8 -; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v5, v4, vcc -; GFX6-NEXT: v_subrev_i32_e32 v5, vcc, s0, v8 -; GFX6-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v4, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s3, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s3, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v6, v3, s10 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, s10, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, 2, v2 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v7, vcc, 1, v2 +; GFX6-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v6 +; GFX6-NEXT: v_mov_b32_e32 v6, s3 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GFX6-NEXT: v_subrev_i32_e32 v6, vcc, s10, v0 +; GFX6-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v1, vcc ; GFX6-NEXT: s_movk_i32 s0, 0xffe -; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s0, v5 -; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s0, v6 +; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 -; GFX6-NEXT: v_cndmask_b32_e32 v5, -1, v5, vcc -; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s0, v8 -; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 -; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4 -; GFX6-NEXT: v_cndmask_b32_e64 v4, -1, v5, s[0:1] -; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 -; GFX6-NEXT: v_cndmask_b32_e64 v3, v1, v3, s[0:1] -; GFX6-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc -; GFX6-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[0:1] +; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s0, v0 +; GFX6-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc +; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1] +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1 +; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 +; GFX6-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[0:1] +; GFX6-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc +; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v0 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v7, v4, vcc +; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, v5, s[0:1] +; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v0, s[0:1] ; GFX6-NEXT: v_mov_b32_e32 v0, s8 ; GFX6-NEXT: v_mov_b32_e32 v1, s9 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 @@ -10182,120 +10158,108 @@ ; GFX6-NEXT: v_mov_b32_e32 v1, 0x4f800000 ; GFX6-NEXT: v_madmk_f32 v0, v1, 0x438f8000, v0 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_movk_i32 s2, 0xfee0 -; GFX6-NEXT: s_mov_b32 s3, 0x689e0837 +; GFX6-NEXT: s_movk_i32 s4, 0xfee0 +; GFX6-NEXT: s_mov_b32 s5, 0x689e0837 ; GFX6-NEXT: v_mov_b32_e32 v8, 0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v1 ; GFX6-NEXT: v_mov_b32_e32 v7, 0 -; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s2 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s3 -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s3 +; GFX6-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9 +; GFX6-NEXT: v_mul_lo_u32 v2, v4, s4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, v5, s5 ; GFX6-NEXT: s_mov_b32 s12, 0x9761f7c9 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v1 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v5, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v5, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v2, v4, s4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, v5, s5 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_mov_b32 s8, s4 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s3 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v4, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v4, v3, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc +; GFX6-NEXT: s_mov_b32 s4, s8 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX6-NEXT: s_movk_i32 s8, 0x11f +; GFX6-NEXT: s_mov_b32 s5, s9 +; GFX6-NEXT: s_movk_i32 s9, 0x11e +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v5, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v5, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s2 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s3 -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s3 -; GFX6-NEXT: s_movk_i32 s4, 0x11f -; GFX6-NEXT: s_mov_b32 s9, s5 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s3 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_mul_lo_u32 v4, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 -; GFX6-NEXT: s_movk_i32 s5, 0x11e -; GFX6-NEXT: s_mov_b32 s11, 0xf000 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v3, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s10, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s10, v2 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s11, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s11, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s6, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s6, v0 -; GFX6-NEXT: v_mul_hi_u32 v4, s6, v1 -; GFX6-NEXT: v_mul_hi_u32 v5, s7, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s7, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s7, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s7, v0 -; GFX6-NEXT: s_mov_b32 s10, -1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s12 -; GFX6-NEXT: v_mul_lo_u32 v1, v1, s12 -; GFX6-NEXT: v_mul_lo_u32 v0, v0, s12 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s7, v1 -; GFX6-NEXT: v_mov_b32_e32 v3, s4 -; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v8, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v3, v0, s8 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, s12, 0 +; GFX6-NEXT: v_mul_lo_u32 v2, v2, s12 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s11, v1 +; GFX6-NEXT: v_mov_b32_e32 v3, s8 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s10, v0 ; GFX6-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v3, vcc ; GFX6-NEXT: v_subrev_i32_e64 v4, s[0:1], s12, v0 ; GFX6-NEXT: v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1] -; GFX6-NEXT: v_cmp_lt_u32_e64 s[2:3], s5, v5 -; GFX6-NEXT: s_mov_b32 s6, 0x9761f7c8 +; GFX6-NEXT: v_cmp_lt_u32_e64 s[2:3], s9, v5 +; GFX6-NEXT: s_mov_b32 s10, 0x9761f7c8 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[2:3] -; GFX6-NEXT: v_cmp_lt_u32_e64 s[2:3], s6, v4 +; GFX6-NEXT: v_cmp_lt_u32_e64 s[2:3], s10, v4 ; GFX6-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1] ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[2:3] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], s4, v5 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], s8, v5 ; GFX6-NEXT: v_subrev_i32_e64 v3, s[0:1], s12, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[2:3] ; GFX6-NEXT: v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1] ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v2, v5, v2, s[0:1] -; GFX6-NEXT: v_mov_b32_e32 v5, s7 +; GFX6-NEXT: v_mov_b32_e32 v5, s11 ; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v5, v1, vcc -; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s5, v1 +; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s9, v1 ; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc -; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s6, v0 +; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s10, v0 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s8, v1 ; GFX6-NEXT: v_cndmask_b32_e32 v5, v5, v6, vcc ; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ; GFX6-NEXT: v_cndmask_b32_e64 v2, v4, v3, s[0:1] ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 +; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: urem_i64_oddk_denom: @@ -10806,116 +10770,104 @@ ; GFX6-NEXT: v_mov_b32_e32 v0, 0x4f800000 ; GFX6-NEXT: v_madak_f32 v0, 0, v0, 0x4996c7d8 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_mov_b32 s5, 0xffed2705 +; GFX6-NEXT: s_mov_b32 s8, 0xffed2705 ; GFX6-NEXT: v_mov_b32_e32 v8, 0 ; GFX6-NEXT: v_mov_b32_e32 v7, 0 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v0 ; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, s5 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s5 -; GFX6-NEXT: v_mul_lo_u32 v4, v0, s5 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_mul_lo_u32 v2, v4, s8 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s8, v5, 0 +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, v5, v1 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v5, v0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v5, v0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v4, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s8, v6, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, v9, s8 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_ashr_i32 s8, s3, 31 +; GFX6-NEXT: v_mul_hi_u32 v10, v6, v0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v1, 0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v5, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc ; GFX6-NEXT: s_add_u32 s2, s2, s8 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v3, vcc -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, s5 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s5 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v0 ; GFX6-NEXT: s_mov_b32 s9, s8 ; GFX6-NEXT: s_addc_u32 s3, s3, s8 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc ; GFX6-NEXT: s_xor_b64 s[2:3], s[2:3], s[8:9] -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s5 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_mul_lo_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v10, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, v1, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v9, v6 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v6, v3 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v5, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s2, v0 -; GFX6-NEXT: v_mul_hi_u32 v4, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v5, s3, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s3, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s3, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s2, v2 ; GFX6-NEXT: s_mov_b32 s4, s0 -; GFX6-NEXT: s_mov_b32 s0, 0x12d8fb -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s0 -; GFX6-NEXT: v_mul_hi_u32 v5, v0, s0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0 -; GFX6-NEXT: v_mul_lo_u32 v8, v0, s0 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; GFX6-NEXT: v_add_i32_e32 v6, vcc, 1, v0 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_mov_b32_e32 v5, s3 -; GFX6-NEXT: v_sub_i32_e32 v8, vcc, s2, v8 -; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v5, v4, vcc -; GFX6-NEXT: v_subrev_i32_e32 v5, vcc, s0, v8 -; GFX6-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v4, vcc +; GFX6-NEXT: s_mov_b32 s5, s1 +; GFX6-NEXT: s_mov_b32 s9, 0x12d8fb +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s3, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s3, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v6, v3, s9 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s9, v2, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, 2, v2 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v7, vcc, 1, v2 +; GFX6-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v6 +; GFX6-NEXT: v_mov_b32_e32 v6, s3 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GFX6-NEXT: v_subrev_i32_e32 v6, vcc, s9, v0 +; GFX6-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v1, vcc ; GFX6-NEXT: s_mov_b32 s0, 0x12d8fa -; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s0, v5 -; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s0, v6 +; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 -; GFX6-NEXT: s_mov_b32 s5, s1 -; GFX6-NEXT: v_cndmask_b32_e32 v5, -1, v5, vcc -; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s0, v8 -; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 -; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4 -; GFX6-NEXT: v_cndmask_b32_e64 v4, -1, v5, s[0:1] -; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1] -; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1] -; GFX6-NEXT: v_xor_b32_e32 v0, s8, v0 +; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s0, v0 +; GFX6-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc +; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1] +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1 +; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 +; GFX6-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[0:1] +; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v0 +; GFX6-NEXT: v_cndmask_b32_e32 v1, v7, v4, vcc +; GFX6-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc +; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[0:1] +; GFX6-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1] ; GFX6-NEXT: v_xor_b32_e32 v1, s8, v1 -; GFX6-NEXT: v_mov_b32_e32 v2, s8 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s8, v0 -; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc +; GFX6-NEXT: v_xor_b32_e32 v2, s8, v0 +; GFX6-NEXT: v_mov_b32_e32 v3, s8 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s8, v1 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v3, vcc ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; @@ -11214,7 +11166,6 @@ ; GFX6-NEXT: s_load_dword s4, s[0:1], 0xd ; GFX6-NEXT: s_mov_b64 s[2:3], 0x1000 ; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: s_mov_b32 s6, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s4 ; GFX6-NEXT: s_ashr_i32 s8, s3, 31 @@ -11224,96 +11175,85 @@ ; GFX6-NEXT: s_xor_b64 s[10:11], s[2:3], s[8:9] ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s10 ; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s11 -; GFX6-NEXT: s_sub_u32 s4, 0, s10 -; GFX6-NEXT: s_subb_u32 s5, 0, s11 -; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: s_sub_u32 s6, 0, s10 +; GFX6-NEXT: s_subb_u32 s12, 0, s11 ; GFX6-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s12, s3, 31 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v0 +; GFX6-NEXT: v_mul_lo_u32 v2, s6, v4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, s12, v5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GFX6-NEXT: v_mul_hi_u32 v6, v5, v0 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v5, v3, 0 +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GFX6-NEXT: v_mov_b32_e32 v6, 0 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GFX6-NEXT: v_mov_b32_e32 v7, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v8, vcc, v5, v0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v4, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, s6, v9 +; GFX6-NEXT: v_mul_lo_u32 v5, s12, v8 +; GFX6-NEXT: v_mul_hi_u32 v10, v8, v0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v1, 0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_ashr_i32 s12, s3, 31 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc ; GFX6-NEXT: s_add_u32 s2, s2, s12 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v8, v0 ; GFX6-NEXT: s_mov_b32 s13, s12 -; GFX6-NEXT: v_mul_lo_u32 v2, s4, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s4, v0 -; GFX6-NEXT: v_mul_lo_u32 v5, s5, v0 -; GFX6-NEXT: v_mul_lo_u32 v4, s4, v0 ; GFX6-NEXT: s_addc_u32 s3, s3, s12 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, v4 -; GFX6-NEXT: v_mul_lo_u32 v5, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v4 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc ; GFX6-NEXT: s_xor_b64 s[2:3], s[2:3], s[12:13] -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v4, vcc -; GFX6-NEXT: v_mov_b32_e32 v4, 0 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v7, v4, vcc -; GFX6-NEXT: v_mov_b32_e32 v6, 0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s4, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s4, v0 -; GFX6-NEXT: v_mul_lo_u32 v5, s5, v0 -; GFX6-NEXT: s_mov_b32 s5, s1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, s4, v0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v2 -; GFX6-NEXT: v_mul_lo_u32 v8, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v10, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, v1, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v5, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v8, v3 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v7, vcc -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s2, v0 -; GFX6-NEXT: v_mul_hi_u32 v5, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v7, s3, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s3, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, s3, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s2, v2 ; GFX6-NEXT: s_mov_b32 s4, s0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v7, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v6, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s10, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s10, v0 -; GFX6-NEXT: v_mul_lo_u32 v4, s11, v0 +; GFX6-NEXT: s_mov_b32 s5, s1 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[14:15], s3, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[14:15], s3, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v7, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v4, s10, v3 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[14:15], s10, v2, 0 +; GFX6-NEXT: v_mul_lo_u32 v5, s11, v2 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s3, v1 ; GFX6-NEXT: v_mov_b32_e32 v5, s11 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, s10, v0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s3, v2 -; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s2, v3 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 ; GFX6-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s10, v3 +; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s10, v0 ; GFX6-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] ; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] @@ -11321,30 +11261,30 @@ ; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] ; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] -; GFX6-NEXT: v_add_i32_e64 v5, s[0:1], 2, v0 -; GFX6-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1] -; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 1, v0 -; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1] +; GFX6-NEXT: v_add_i32_e64 v5, s[0:1], 2, v2 +; GFX6-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v3, s[0:1] +; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 1, v2 +; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1] ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v4, v8, v6, s[0:1] ; GFX6-NEXT: v_mov_b32_e32 v6, s3 -; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v6, v2, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s11, v2 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s11, v1 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s10, v3 -; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s11, v2 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc -; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; GFX6-NEXT: v_cndmask_b32_e64 v2, v7, v5, s[0:1] -; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s10, v0 +; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s11, v1 +; GFX6-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc +; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GFX6-NEXT: v_cndmask_b32_e64 v1, v7, v5, s[0:1] +; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc ; GFX6-NEXT: s_xor_b64 s[0:1], s[12:13], s[8:9] -; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX6-NEXT: v_xor_b32_e32 v0, s0, v0 -; GFX6-NEXT: v_xor_b32_e32 v1, s1, v1 -; GFX6-NEXT: v_mov_b32_e32 v2, s1 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0 -; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc +; GFX6-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc +; GFX6-NEXT: v_xor_b32_e32 v1, s0, v1 +; GFX6-NEXT: v_xor_b32_e32 v2, s1, v0 +; GFX6-NEXT: v_mov_b32_e32 v3, s1 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v1 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v3, vcc ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; @@ -11719,120 +11659,108 @@ ; GFX6-NEXT: v_mov_b32_e32 v1, 0x4f800000 ; GFX6-NEXT: v_mac_f32_e32 v0, 0, v1 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_movk_i32 s6, 0xf001 +; GFX6-NEXT: s_movk_i32 s10, 0xf001 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd -; GFX6-NEXT: s_mov_b32 s7, 0xf000 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s10, v4, 0 +; GFX6-NEXT: v_mul_lo_u32 v2, v5, s10 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s8, s1, 31 -; GFX6-NEXT: s_lshr_b32 s8, s8, 20 -; GFX6-NEXT: v_mul_hi_u32 v2, v0, s6 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, s6 +; GFX6-NEXT: s_ashr_i32 s6, s1, 31 +; GFX6-NEXT: s_lshr_b32 s8, s6, 20 +; GFX6-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, v4, v1 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[6:7], v4, v3, 0 ; GFX6-NEXT: s_add_u32 s0, s0, s8 ; GFX6-NEXT: s_addc_u32 s1, s1, 0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v5, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v5, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GFX6-NEXT: v_mov_b32_e32 v6, 0 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GFX6-NEXT: v_mov_b32_e32 v7, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v8, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s10, v8, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, v9, s10 ; GFX6-NEXT: s_ashr_i64 s[8:9], s[0:1], 12 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s6 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_mul_lo_u32 v4, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 +; GFX6-NEXT: v_mul_hi_u32 v10, v8, v0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v9, v0, 0 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, v8, v1 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v8, v1, 0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v9, v1, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; GFX6-NEXT: s_ashr_i32 s10, s3, 31 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc ; GFX6-NEXT: s_add_u32 s0, s2, s10 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v3, vcc -; GFX6-NEXT: v_mov_b32_e32 v4, 0 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v7, v4, vcc -; GFX6-NEXT: v_mov_b32_e32 v6, 0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, s6 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s6 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v8, v0 ; GFX6-NEXT: s_mov_b32 s11, s10 ; GFX6-NEXT: s_addc_u32 s1, s3, s10 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc ; GFX6-NEXT: s_xor_b64 s[0:1], s[0:1], s[10:11] -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s6 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_mul_lo_u32 v8, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v10, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, v1, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v5, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v8, v3 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v7, vcc -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s0, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s0, v0 -; GFX6-NEXT: v_mul_hi_u32 v5, s0, v1 -; GFX6-NEXT: v_mul_hi_u32 v7, s1, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s1, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, s1, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s1, v0 -; GFX6-NEXT: s_movk_i32 s2, 0xfff +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s0, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s0, v2 +; GFX6-NEXT: s_movk_i32 s11, 0xfff +; GFX6-NEXT: s_mov_b32 s7, 0xf000 ; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v7, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v6, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s2 -; GFX6-NEXT: v_mul_hi_u32 v5, v0, s2 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0 -; GFX6-NEXT: v_mul_lo_u32 v8, v0, s2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; GFX6-NEXT: v_add_i32_e32 v6, vcc, 1, v0 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_mov_b32_e32 v5, s1 -; GFX6-NEXT: v_sub_i32_e32 v8, vcc, s0, v8 -; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v5, v4, vcc -; GFX6-NEXT: v_subrev_i32_e32 v5, vcc, s2, v8 -; GFX6-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v4, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s1, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s1, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v7, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v6, v3, s11 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s11, v2, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, 2, v2 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v7, vcc, 1, v2 +; GFX6-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v6 +; GFX6-NEXT: v_mov_b32_e32 v6, s1 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GFX6-NEXT: v_subrev_i32_e32 v6, vcc, s11, v0 +; GFX6-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v1, vcc ; GFX6-NEXT: s_movk_i32 s0, 0xffe -; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s0, v5 -; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s0, v6 +; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 -; GFX6-NEXT: v_cndmask_b32_e32 v5, -1, v5, vcc -; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s0, v8 -; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 -; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4 -; GFX6-NEXT: v_cndmask_b32_e64 v4, -1, v5, s[0:1] -; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1] -; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1] -; GFX6-NEXT: v_xor_b32_e32 v0, s10, v0 +; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s0, v0 +; GFX6-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc +; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1] +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1 +; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 +; GFX6-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[0:1] +; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v0 +; GFX6-NEXT: v_cndmask_b32_e32 v1, v7, v4, vcc +; GFX6-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc +; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[0:1] +; GFX6-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1] ; GFX6-NEXT: v_xor_b32_e32 v1, s10, v1 +; GFX6-NEXT: v_xor_b32_e32 v0, s10, v0 ; GFX6-NEXT: v_mov_b32_e32 v3, s10 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s10, v0 -; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc +; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s10, v1 +; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v0, v3, vcc ; GFX6-NEXT: v_mov_b32_e32 v0, s8 ; GFX6-NEXT: v_mov_b32_e32 v1, s9 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 @@ -12100,240 +12028,216 @@ ; GFX6-LABEL: sdiv_v2i64_pow2_shl_denom: ; GFX6: ; %bb.0: ; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; GFX6-NEXT: s_mov_b64 s[2:3], 0x1000 +; GFX6-NEXT: s_mov_b64 s[8:9], 0x1000 ; GFX6-NEXT: s_mov_b32 s18, 0x4f800000 ; GFX6-NEXT: s_mov_b32 s19, 0x5f7ffffc ; GFX6-NEXT: s_mov_b32 s20, 0x2f800000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b64 s[8:9], s[2:3], s6 -; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s4 -; GFX6-NEXT: s_ashr_i32 s12, s3, 31 -; GFX6-NEXT: s_add_u32 s2, s2, s12 -; GFX6-NEXT: s_mov_b32 s13, s12 -; GFX6-NEXT: s_addc_u32 s3, s3, s12 -; GFX6-NEXT: s_xor_b64 s[10:11], s[2:3], s[12:13] -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s10 -; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s11 +; GFX6-NEXT: s_lshl_b64 s[4:5], s[8:9], s4 +; GFX6-NEXT: s_lshl_b64 s[2:3], s[8:9], s6 +; GFX6-NEXT: s_ashr_i32 s14, s5, 31 +; GFX6-NEXT: s_add_u32 s4, s4, s14 +; GFX6-NEXT: s_mov_b32 s15, s14 +; GFX6-NEXT: s_addc_u32 s5, s5, s14 +; GFX6-NEXT: s_xor_b64 s[12:13], s[4:5], s[14:15] +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s12 +; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s13 ; GFX6-NEXT: s_mov_b32 s21, 0xcf800000 -; GFX6-NEXT: s_sub_u32 s6, 0, s10 -; GFX6-NEXT: s_subb_u32 s7, 0, s11 +; GFX6-NEXT: s_sub_u32 s6, 0, s12 +; GFX6-NEXT: s_subb_u32 s7, 0, s13 ; GFX6-NEXT: v_mac_f32_e32 v0, s18, v1 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd +; GFX6-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0xd ; GFX6-NEXT: v_mul_f32_e32 v0, s19, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, s20, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, s21, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v0 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s14, s1, 31 -; GFX6-NEXT: s_add_u32 s0, s0, s14 -; GFX6-NEXT: v_mul_lo_u32 v0, s6, v2 -; GFX6-NEXT: v_mul_hi_u32 v1, s6, v3 -; GFX6-NEXT: v_mul_lo_u32 v4, s7, v3 -; GFX6-NEXT: v_mul_lo_u32 v5, s6, v3 -; GFX6-NEXT: s_mov_b32 s15, s14 -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0 -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v0, v4 -; GFX6-NEXT: v_mul_lo_u32 v0, v3, v1 -; GFX6-NEXT: v_mul_hi_u32 v4, v3, v5 -; GFX6-NEXT: v_mul_hi_u32 v6, v3, v1 -; GFX6-NEXT: v_mul_hi_u32 v7, v2, v1 -; GFX6-NEXT: s_addc_u32 s1, s1, s14 -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v2, v5 -; GFX6-NEXT: v_mul_hi_u32 v5, v2, v5 -; GFX6-NEXT: s_xor_b64 s[16:17], s[0:1], s[14:15] -; GFX6-NEXT: s_xor_b64 s[14:15], s[14:15], s[12:13] -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v6 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, v2, v1 -; GFX6-NEXT: v_mov_b32_e32 v0, 0 -; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v7, v0, vcc +; GFX6-NEXT: s_ashr_i32 s16, s9, 31 +; GFX6-NEXT: s_mov_b32 s17, s16 +; GFX6-NEXT: v_mul_lo_u32 v2, s6, v4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, s7, v5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v5, v0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v7, v1, vcc ; GFX6-NEXT: v_mov_b32_e32 v1, 0 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v1, v6, vcc -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v2, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s6, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, s6, v3 -; GFX6-NEXT: v_mul_lo_u32 v6, s7, v3 -; GFX6-NEXT: s_ashr_i32 s12, s9, 31 -; GFX6-NEXT: s_add_u32 s8, s8, s12 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_mul_lo_u32 v5, s6, v3 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc +; GFX6-NEXT: v_mov_b32_e32 v0, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v8, vcc, v5, v2 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v4, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s6, v8, 0 +; GFX6-NEXT: v_mul_lo_u32 v6, s6, v9 +; GFX6-NEXT: v_mul_lo_u32 v7, s7, v8 +; GFX6-NEXT: v_mul_hi_u32 v10, v8, v2 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v9, v2, 0 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v7, v3 +; GFX6-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v3, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v10, v6 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc ; GFX6-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GFX6-NEXT: v_mul_lo_u32 v8, v3, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, v3, v5 -; GFX6-NEXT: v_mul_hi_u32 v10, v3, v4 -; GFX6-NEXT: v_mul_hi_u32 v7, v2, v5 -; GFX6-NEXT: v_mul_lo_u32 v5, v2, v5 -; GFX6-NEXT: v_mul_hi_u32 v6, v2, v4 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, v2, v4 -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v6, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v1, v6, vcc -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v2, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s16, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, s16, v3 -; GFX6-NEXT: v_mul_hi_u32 v6, s16, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, s17, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, s17, v2 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, s17, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, s17, v3 -; GFX6-NEXT: s_mov_b32 s13, s12 -; GFX6-NEXT: s_addc_u32 s9, s9, s12 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v3, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v1, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s10, v3 -; GFX6-NEXT: v_mul_hi_u32 v5, s10, v2 -; GFX6-NEXT: v_mul_lo_u32 v6, s11, v2 -; GFX6-NEXT: v_mov_b32_e32 v7, s11 -; GFX6-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13] -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_mul_lo_u32 v5, s10, v2 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s17, v4 -; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s16, v5 +; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc +; GFX6-NEXT: s_add_u32 s4, s8, s16 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v8, v2 +; GFX6-NEXT: s_addc_u32 s5, s9, s16 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v9, v3, vcc +; GFX6-NEXT: s_xor_b64 s[8:9], s[4:5], s[16:17] +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v5, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, s8, v4 +; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; GFX6-NEXT: s_xor_b64 s[16:17], s[16:17], s[14:15] +; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s9, v4, 0 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s9, v5, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v7, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v5, vcc, v2, v4 +; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v0, v3, vcc +; GFX6-NEXT: v_mul_lo_u32 v6, s12, v4 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s12, v5, 0 +; GFX6-NEXT: v_mul_lo_u32 v7, s13, v5 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7 +; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s9, v3 +; GFX6-NEXT: v_mov_b32_e32 v7, s13 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s8, v2 ; GFX6-NEXT: v_subb_u32_e64 v6, s[0:1], v6, v7, vcc -; GFX6-NEXT: v_subrev_i32_e64 v7, s[0:1], s10, v5 +; GFX6-NEXT: v_subrev_i32_e64 v7, s[0:1], s12, v2 ; GFX6-NEXT: v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v6 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v7 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v7 ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v6 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v6, v8, v7, s[0:1] -; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 2, v2 -; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1] -; GFX6-NEXT: v_add_i32_e64 v9, s[0:1], 1, v2 -; GFX6-NEXT: v_addc_u32_e64 v10, s[0:1], 0, v3, s[0:1] +; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 2, v5 +; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v4, s[0:1] +; GFX6-NEXT: v_add_i32_e64 v9, s[0:1], 1, v5 +; GFX6-NEXT: v_addc_u32_e64 v10, s[0:1], 0, v4, s[0:1] +; GFX6-NEXT: s_ashr_i32 s8, s3, 31 ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v6 +; GFX6-NEXT: s_add_u32 s2, s2, s8 ; GFX6-NEXT: v_cndmask_b32_e64 v6, v10, v8, s[0:1] +; GFX6-NEXT: v_mov_b32_e32 v8, s9 +; GFX6-NEXT: s_mov_b32 s9, s8 +; GFX6-NEXT: s_addc_u32 s3, s3, s8 +; GFX6-NEXT: s_xor_b64 s[14:15], s[2:3], s[8:9] +; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v8, v3, vcc +; GFX6-NEXT: v_cvt_f32_u32_e32 v8, s14 +; GFX6-NEXT: v_cvt_f32_u32_e32 v10, s15 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s13, v3 +; GFX6-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s12, v2 +; GFX6-NEXT: v_mac_f32_e32 v8, s18, v10 +; GFX6-NEXT: v_rcp_f32_e32 v8, v8 +; GFX6-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s13, v3 +; GFX6-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc +; GFX6-NEXT: v_cmp_ne_u32_e64 s[2:3], 0, v2 +; GFX6-NEXT: v_mul_f32_e32 v2, s19, v8 +; GFX6-NEXT: v_mul_f32_e32 v3, s20, v2 +; GFX6-NEXT: v_trunc_f32_e32 v3, v3 +; GFX6-NEXT: v_mac_f32_e32 v2, s21, v3 +; GFX6-NEXT: v_cndmask_b32_e64 v10, v4, v6, s[2:3] +; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GFX6-NEXT: v_cvt_u32_f32_e32 v8, v3 +; GFX6-NEXT: s_sub_u32 s18, 0, s14 +; GFX6-NEXT: s_subb_u32 s19, 0, s15 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[12:13], s18, v6, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, s18, v8 +; GFX6-NEXT: v_mul_lo_u32 v11, s19, v6 +; GFX6-NEXT: v_mul_hi_u32 v12, v6, v2 +; GFX6-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[0:1] +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3 +; GFX6-NEXT: v_add_i32_e32 v11, vcc, v3, v11 +; GFX6-NEXT: v_mad_u64_u32 v[3:4], s[12:13], v6, v11, 0 +; GFX6-NEXT: v_cndmask_b32_e64 v7, v5, v7, s[2:3] +; GFX6-NEXT: s_ashr_i32 s2, s11, 31 +; GFX6-NEXT: v_add_i32_e32 v9, vcc, v12, v3 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v8, v2, 0 +; GFX6-NEXT: v_addc_u32_e32 v12, vcc, 0, v4, vcc +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v8, v11, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v9, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v12, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v9, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v8, vcc, v8, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s18, v9, 0 +; GFX6-NEXT: v_mul_lo_u32 v6, s18, v8 +; GFX6-NEXT: v_xor_b32_e32 v11, s16, v7 +; GFX6-NEXT: v_mul_lo_u32 v7, s19, v9 +; GFX6-NEXT: v_mul_hi_u32 v12, v9, v2 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v7, v3 +; GFX6-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v9, v3, 0 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v8, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v8, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v12, v6 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc +; GFX6-NEXT: s_add_u32 s0, s10, s2 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v9, v2 +; GFX6-NEXT: s_mov_b32 s3, s2 +; GFX6-NEXT: s_addc_u32 s1, s11, s2 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v3, vcc +; GFX6-NEXT: s_xor_b64 s[10:11], s[0:1], s[2:3] +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s10, v5, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, s10, v4 +; GFX6-NEXT: v_xor_b32_e32 v7, s17, v10 ; GFX6-NEXT: v_mov_b32_e32 v8, s17 -; GFX6-NEXT: v_cvt_f32_u32_e32 v10, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v11, s9 -; GFX6-NEXT: v_subb_u32_e32 v4, vcc, v8, v4, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s11, v4 -; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s10, v5 -; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s11, v4 -; GFX6-NEXT: v_mac_f32_e32 v10, s18, v11 -; GFX6-NEXT: v_cndmask_b32_e32 v4, v8, v5, vcc -; GFX6-NEXT: v_rcp_f32_e32 v5, v10 -; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 -; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc -; GFX6-NEXT: v_cndmask_b32_e64 v4, v9, v7, s[0:1] -; GFX6-NEXT: v_mul_f32_e32 v5, s19, v5 -; GFX6-NEXT: v_mul_f32_e32 v6, s20, v5 -; GFX6-NEXT: v_trunc_f32_e32 v6, v6 -; GFX6-NEXT: v_mac_f32_e32 v5, s21, v6 -; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v6 -; GFX6-NEXT: s_sub_u32 s0, 0, s8 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX6-NEXT: v_mul_hi_u32 v4, s0, v5 -; GFX6-NEXT: v_mul_lo_u32 v7, s0, v6 -; GFX6-NEXT: s_subb_u32 s1, 0, s9 -; GFX6-NEXT: v_mul_lo_u32 v8, s1, v5 -; GFX6-NEXT: s_ashr_i32 s10, s3, 31 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GFX6-NEXT: v_mul_lo_u32 v7, s0, v5 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; GFX6-NEXT: v_mul_lo_u32 v8, v5, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, v5, v7 -; GFX6-NEXT: v_mul_hi_u32 v10, v5, v4 -; GFX6-NEXT: v_mul_hi_u32 v11, v6, v4 -; GFX6-NEXT: v_mul_lo_u32 v4, v6, v4 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v10, v6, v7 -; GFX6-NEXT: v_mul_hi_u32 v7, v6, v7 -; GFX6-NEXT: s_mov_b32 s11, s10 -; GFX6-NEXT: v_xor_b32_e32 v2, s14, v2 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v9, v7, vcc -; GFX6-NEXT: v_addc_u32_e32 v8, vcc, v11, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v7, v4 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v1, v8, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v6, v7, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, s0, v5 -; GFX6-NEXT: v_mul_hi_u32 v7, s0, v4 -; GFX6-NEXT: v_mul_lo_u32 v8, s1, v4 -; GFX6-NEXT: v_xor_b32_e32 v3, s15, v3 -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GFX6-NEXT: v_mul_lo_u32 v7, s0, v4 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v8, v6 -; GFX6-NEXT: v_mul_lo_u32 v10, v4, v6 -; GFX6-NEXT: v_mul_hi_u32 v11, v4, v7 -; GFX6-NEXT: v_mul_hi_u32 v12, v4, v6 -; GFX6-NEXT: v_mul_hi_u32 v9, v5, v7 -; GFX6-NEXT: v_mul_lo_u32 v7, v5, v7 -; GFX6-NEXT: v_mul_hi_u32 v8, v5, v6 -; GFX6-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; GFX6-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v5, v6 -; GFX6-NEXT: v_add_i32_e32 v7, vcc, v10, v7 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v11, v9, vcc -; GFX6-NEXT: v_addc_u32_e32 v8, vcc, v8, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v1, v8, vcc -; GFX6-NEXT: s_add_u32 s0, s2, s10 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GFX6-NEXT: s_addc_u32 s1, s3, s10 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc -; GFX6-NEXT: s_xor_b64 s[2:3], s[0:1], s[10:11] -; GFX6-NEXT: v_mul_lo_u32 v6, s2, v5 -; GFX6-NEXT: v_mul_hi_u32 v7, s2, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, s2, v5 -; GFX6-NEXT: v_mul_hi_u32 v10, s3, v5 -; GFX6-NEXT: v_mul_lo_u32 v5, s3, v5 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc -; GFX6-NEXT: v_mul_lo_u32 v9, s3, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, s3, v4 -; GFX6-NEXT: v_mov_b32_e32 v8, s15 -; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v9 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v4, vcc -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v10, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v1, v0, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, s8, v5 -; GFX6-NEXT: v_mul_hi_u32 v7, s8, v4 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s14, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, s9, v4 -; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v3, v8, vcc -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v7, v6 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, s8, v4 -; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s3, v2 -; GFX6-NEXT: v_mov_b32_e32 v7, s9 -; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s2, v3 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s11, v4, 0 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s11, v5, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v9, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v5, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v2, v4 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v0, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v6, s14, v5 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s14, v4, 0 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s16, v11 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v7, v8, vcc +; GFX6-NEXT: v_mul_lo_u32 v7, s15, v4 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7 +; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s11, v3 +; GFX6-NEXT: v_mov_b32_e32 v7, s15 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s10, v2 ; GFX6-NEXT: v_subb_u32_e64 v6, s[0:1], v6, v7, vcc -; GFX6-NEXT: v_subrev_i32_e64 v7, s[0:1], s8, v3 +; GFX6-NEXT: v_subrev_i32_e64 v7, s[0:1], s14, v2 ; GFX6-NEXT: v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v6 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v7 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v7 ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v6 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v6, v8, v7, s[0:1] ; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 2, v4 ; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v5, s[0:1] @@ -12341,24 +12245,25 @@ ; GFX6-NEXT: v_addc_u32_e64 v10, s[0:1], 0, v5, s[0:1] ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v6, v10, v8, s[0:1] -; GFX6-NEXT: v_mov_b32_e32 v8, s3 -; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v8, v2, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_mov_b32_e32 v8, s11 +; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v8, v3, vcc +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s15, v3 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 -; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s9, v2 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v8, v3, vcc +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s14, v2 +; GFX6-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s15, v3 +; GFX6-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc ; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v3, v9, v7, s[0:1] ; GFX6-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX6-NEXT: s_xor_b64 s[0:1], s[10:11], s[12:13] +; GFX6-NEXT: s_xor_b64 s[0:1], s[2:3], s[8:9] ; GFX6-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc ; GFX6-NEXT: v_xor_b32_e32 v3, s0, v3 ; GFX6-NEXT: v_xor_b32_e32 v4, s1, v2 ; GFX6-NEXT: v_mov_b32_e32 v5, s1 ; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s0, v3 ; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v4, v5, vcc +; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; @@ -12900,92 +12805,80 @@ ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v0 ; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, s4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s4 -; GFX6-NEXT: v_mul_lo_u32 v4, v0, s4 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_mul_lo_u32 v2, v4, s4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v5, 0 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: v_subrev_i32_e32 v6, vcc, v5, v1 +; GFX6-NEXT: v_mul_hi_u32 v3, v5, v0 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v5, v6, 0 +; GFX6-NEXT: v_add_i32_e32 v9, vcc, v3, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v10, vcc, v8, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v4, v6, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v9, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v5, v0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v4, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v6, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, v9, s4 +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: v_mul_hi_u32 v10, v6, v0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v1, 0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v5, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_ashr_i32 s8, s3, 31 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc ; GFX6-NEXT: s_add_u32 s2, s2, s8 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v3, vcc -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, s4 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, s4 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v0 ; GFX6-NEXT: s_mov_b32 s9, s8 ; GFX6-NEXT: s_addc_u32 s3, s3, s8 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc ; GFX6-NEXT: s_xor_b64 s[2:3], s[2:3], s[8:9] -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, v0, s4 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GFX6-NEXT: v_mul_lo_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v10, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, v1, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v9, v6 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v8, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v6, v3 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v5, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s2, v2 +; GFX6-NEXT: s_mov_b32 s9, 0x12d8fb +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s3, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s3, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s2, v0 -; GFX6-NEXT: v_mul_hi_u32 v4, s2, v1 -; GFX6-NEXT: v_mul_hi_u32 v5, s3, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s3, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s3, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v2, v1, s9 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s9, v0, 0 ; GFX6-NEXT: s_mov_b32 s4, s0 -; GFX6-NEXT: s_mov_b32 s0, 0x12d8fb -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v8, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v1, v1, s0 -; GFX6-NEXT: v_mul_hi_u32 v2, v0, s0 -; GFX6-NEXT: v_mul_lo_u32 v0, v0, s0 -; GFX6-NEXT: s_mov_b32 s5, s1 -; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: s_mov_b32 s0, 0x12d8fa +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GFX6-NEXT: v_mov_b32_e32 v2, s3 ; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 ; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s0, v0 +; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s9, v0 ; GFX6-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v1, vcc -; GFX6-NEXT: v_subrev_i32_e32 v4, vcc, s0, v2 +; GFX6-NEXT: v_subrev_i32_e32 v4, vcc, s9, v2 ; GFX6-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v3, vcc -; GFX6-NEXT: s_mov_b32 s0, 0x12d8fa ; GFX6-NEXT: v_cmp_lt_u32_e32 vcc, s0, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 ; GFX6-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc +; GFX6-NEXT: s_mov_b32 s5, s1 ; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 ; GFX6-NEXT: v_cmp_lt_u32_e64 s[0:1], s0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc @@ -13307,7 +13200,6 @@ ; GFX6-NEXT: s_load_dword s4, s[0:1], 0xd ; GFX6-NEXT: s_mov_b64 s[2:3], 0x1000 ; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: s_mov_b32 s6, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s4 ; GFX6-NEXT: s_ashr_i32 s4, s3, 31 @@ -13317,91 +13209,80 @@ ; GFX6-NEXT: s_xor_b64 s[8:9], s[2:3], s[4:5] ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GFX6-NEXT: s_sub_u32 s4, 0, s8 -; GFX6-NEXT: s_subb_u32 s5, 0, s9 -; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: s_sub_u32 s6, 0, s8 +; GFX6-NEXT: s_subb_u32 s10, 0, s9 ; GFX6-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s10, s3, 31 ; GFX6-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v0 +; GFX6-NEXT: v_mul_lo_u32 v2, s6, v4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, s10, v5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GFX6-NEXT: v_mul_hi_u32 v6, v5, v0 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v5, v3, 0 +; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GFX6-NEXT: v_mov_b32_e32 v6, 0 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GFX6-NEXT: v_mov_b32_e32 v7, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v8, vcc, v5, v0 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v4, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, s6, v9 +; GFX6-NEXT: v_mul_lo_u32 v5, s10, v8 +; GFX6-NEXT: v_mul_hi_u32 v10, v8, v0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v1, 0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_ashr_i32 s10, s3, 31 +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc ; GFX6-NEXT: s_add_u32 s2, s2, s10 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v8, v0 ; GFX6-NEXT: s_mov_b32 s11, s10 -; GFX6-NEXT: v_mul_lo_u32 v2, s4, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s4, v0 -; GFX6-NEXT: v_mul_lo_u32 v5, s5, v0 -; GFX6-NEXT: v_mul_lo_u32 v4, s4, v0 ; GFX6-NEXT: s_addc_u32 s3, s3, s10 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5 -; GFX6-NEXT: v_mul_hi_u32 v3, v0, v4 -; GFX6-NEXT: v_mul_lo_u32 v5, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, v1, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v1, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, v1, v4 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc ; GFX6-NEXT: s_xor_b64 s[12:13], s[2:3], s[10:11] -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v4, vcc -; GFX6-NEXT: v_mov_b32_e32 v4, 0 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v7, v4, vcc -; GFX6-NEXT: v_mov_b32_e32 v6, 0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s4, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s4, v0 -; GFX6-NEXT: v_mul_lo_u32 v5, s5, v0 -; GFX6-NEXT: s_mov_b32 s5, s1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, s4, v0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v2 -; GFX6-NEXT: v_mul_lo_u32 v8, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v9, v0, v3 -; GFX6-NEXT: v_mul_hi_u32 v10, v0, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, v1, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, v1, v3 -; GFX6-NEXT: v_mul_hi_u32 v5, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v8, v3 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v9, v7, vcc -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s12, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s12, v0 -; GFX6-NEXT: v_mul_hi_u32 v5, s12, v1 -; GFX6-NEXT: v_mul_hi_u32 v7, s13, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s13, v1 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, s13, v0 -; GFX6-NEXT: v_mul_hi_u32 v0, s13, v0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s12, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v4, s12, v2 ; GFX6-NEXT: s_mov_b32 s4, s0 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v7, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v6, v2, vcc -; GFX6-NEXT: v_mul_lo_u32 v1, s8, v1 -; GFX6-NEXT: v_mul_hi_u32 v2, s8, v0 -; GFX6-NEXT: v_mul_lo_u32 v3, s9, v0 -; GFX6-NEXT: v_mul_lo_u32 v0, s8, v0 -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: s_mov_b32 s5, s1 +; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s13, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s13, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GFX6-NEXT: v_mul_lo_u32 v3, s8, v0 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s8, v2, 0 +; GFX6-NEXT: v_mul_lo_u32 v2, s9, v2 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s13, v1 ; GFX6-NEXT: v_mov_b32_e32 v3, s9 ; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s12, v0 @@ -13842,11 +13723,10 @@ ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s16 ; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s17 ; GFX6-NEXT: s_mov_b32 s21, 0xcf800000 -; GFX6-NEXT: s_sub_u32 s2, 0, s16 -; GFX6-NEXT: s_subb_u32 s3, 0, s17 +; GFX6-NEXT: s_sub_u32 s4, 0, s16 +; GFX6-NEXT: s_subb_u32 s5, 0, s17 ; GFX6-NEXT: v_mac_f32_e32 v0, s18, v1 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 ; GFX6-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0xd ; GFX6-NEXT: s_mov_b32 s7, 0xf000 ; GFX6-NEXT: s_mov_b32 s6, -1 @@ -13854,205 +13734,182 @@ ; GFX6-NEXT: v_mul_f32_e32 v1, s20, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, s21, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v0 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v0 +; GFX6-NEXT: v_mul_lo_u32 v2, s4, v4 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v5, 0 +; GFX6-NEXT: v_mul_lo_u32 v3, s5, v5 +; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GFX6-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v5, v3, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, v5, v0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GFX6-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, v0, 0 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v2, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v4, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v7, v1, vcc +; GFX6-NEXT: v_mov_b32_e32 v1, 0 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc +; GFX6-NEXT: v_mov_b32_e32 v0, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v8, vcc, v5, v2 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, v4, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s4, v8, 0 +; GFX6-NEXT: v_mul_lo_u32 v6, s4, v9 +; GFX6-NEXT: v_mul_lo_u32 v7, s5, v8 +; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; GFX6-NEXT: v_mul_hi_u32 v10, v8, v2 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v7, v3 +; GFX6-NEXT: v_mad_u64_u32 v[6:7], s[2:3], v8, v3, 0 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[2:3], v9, v2, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v9, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v10, v6 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_ashr_i32 s12, s9, 31 -; GFX6-NEXT: s_add_u32 s0, s8, s12 -; GFX6-NEXT: v_mul_lo_u32 v0, s2, v2 -; GFX6-NEXT: v_mul_hi_u32 v1, s2, v3 -; GFX6-NEXT: v_mul_lo_u32 v4, s3, v3 -; GFX6-NEXT: v_mul_lo_u32 v5, s2, v3 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc +; GFX6-NEXT: s_add_u32 s2, s8, s12 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v8, v2 ; GFX6-NEXT: s_mov_b32 s13, s12 -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0 -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v0, v4 -; GFX6-NEXT: v_mul_lo_u32 v0, v3, v1 -; GFX6-NEXT: v_mul_hi_u32 v4, v3, v5 -; GFX6-NEXT: v_mul_hi_u32 v6, v3, v1 -; GFX6-NEXT: v_mul_hi_u32 v7, v2, v1 -; GFX6-NEXT: s_addc_u32 s1, s9, s12 -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v4, v0 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v2, v5 -; GFX6-NEXT: v_mul_hi_u32 v5, v2, v5 -; GFX6-NEXT: s_xor_b64 s[8:9], s[0:1], s[12:13] -; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v6 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, v2, v1 -; GFX6-NEXT: v_mov_b32_e32 v0, 0 -; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v7, v0, vcc -; GFX6-NEXT: v_mov_b32_e32 v1, 0 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v1, v6, vcc -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v2, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s2, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, s2, v3 -; GFX6-NEXT: v_mul_lo_u32 v6, s3, v3 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_mul_lo_u32 v5, s2, v3 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GFX6-NEXT: v_mul_lo_u32 v8, v3, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, v3, v5 -; GFX6-NEXT: v_mul_hi_u32 v10, v3, v4 -; GFX6-NEXT: v_mul_hi_u32 v7, v2, v5 -; GFX6-NEXT: v_mul_lo_u32 v5, v2, v5 -; GFX6-NEXT: v_mul_hi_u32 v6, v2, v4 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, v2, v4 -; GFX6-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v6, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v1, v6, vcc -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v2, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s8, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, s8, v3 -; GFX6-NEXT: v_mul_hi_u32 v6, s8, v2 -; GFX6-NEXT: v_mul_hi_u32 v7, s9, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, s9, v2 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, s9, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, s9, v3 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v3, vcc -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v1, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v3, s16, v3 -; GFX6-NEXT: v_mul_hi_u32 v4, s16, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, s17, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, s16, v2 -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3 +; GFX6-NEXT: s_addc_u32 s3, s9, s12 +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v9, v3, vcc +; GFX6-NEXT: s_xor_b64 s[8:9], s[2:3], s[12:13] +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s8, v5, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, s8, v4 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s9, v4, 0 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s9, v5, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v7, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v2, v4 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v0, v3, vcc +; GFX6-NEXT: v_mul_lo_u32 v5, s16, v2 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s16, v4, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, s17, v4 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 ; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s9, v3 ; GFX6-NEXT: v_mov_b32_e32 v5, s17 -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s8, v2 -; GFX6-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GFX6-NEXT: v_subrev_i32_e64 v6, s[0:1], s16, v2 -; GFX6-NEXT: v_subbrev_u32_e64 v7, s[2:3], 0, v4, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s17, v7 +; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s8, v2 +; GFX6-NEXT: v_subb_u32_e64 v2, s[0:1], v4, v5, vcc +; GFX6-NEXT: v_subrev_i32_e64 v7, s[0:1], s16, v6 +; GFX6-NEXT: v_subbrev_u32_e64 v4, s[2:3], 0, v2, s[0:1] +; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s17, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[2:3] -; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s16, v6 -; GFX6-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, s[0:1] +; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s16, v7 +; GFX6-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v5, s[0:1] ; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[2:3] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], s17, v7 -; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s16, v6 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], s17, v4 +; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s16, v7 ; GFX6-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[2:3] -; GFX6-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] +; GFX6-NEXT: v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1] ; GFX6-NEXT: s_ashr_i32 s2, s15, 31 ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v8 ; GFX6-NEXT: s_add_u32 s8, s14, s2 -; GFX6-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[0:1] -; GFX6-NEXT: v_mov_b32_e32 v7, s9 +; GFX6-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[0:1] +; GFX6-NEXT: v_mov_b32_e32 v4, s9 ; GFX6-NEXT: s_mov_b32 s3, s2 ; GFX6-NEXT: s_addc_u32 s9, s15, s2 ; GFX6-NEXT: s_xor_b64 s[8:9], s[8:9], s[2:3] -; GFX6-NEXT: v_cvt_f32_u32_e32 v8, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v9, s9 -; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v7, v3, vcc +; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v4, v3, vcc +; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v8, s9 ; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s17, v3 -; GFX6-NEXT: v_mac_f32_e32 v8, s18, v9 -; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s16, v2 -; GFX6-NEXT: v_rcp_f32_e32 v8, v8 +; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s16, v6 +; GFX6-NEXT: v_mac_f32_e32 v4, s18, v8 +; GFX6-NEXT: v_rcp_f32_e32 v4, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc ; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s17, v3 -; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc -; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7 -; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX6-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] -; GFX6-NEXT: v_mul_f32_e32 v5, s19, v8 -; GFX6-NEXT: v_mul_f32_e32 v6, s20, v5 -; GFX6-NEXT: v_trunc_f32_e32 v6, v6 -; GFX6-NEXT: v_mac_f32_e32 v5, s21, v6 -; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v6 -; GFX6-NEXT: s_sub_u32 s0, 0, s8 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX6-NEXT: v_mul_hi_u32 v4, s0, v5 -; GFX6-NEXT: v_mul_lo_u32 v7, s0, v6 -; GFX6-NEXT: s_subb_u32 s1, 0, s9 -; GFX6-NEXT: v_mul_lo_u32 v8, s1, v5 +; GFX6-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc +; GFX6-NEXT: v_cmp_ne_u32_e64 s[2:3], 0, v8 +; GFX6-NEXT: v_cndmask_b32_e64 v8, v3, v2, s[2:3] +; GFX6-NEXT: v_mul_f32_e32 v2, s19, v4 +; GFX6-NEXT: v_mul_f32_e32 v3, s20, v2 +; GFX6-NEXT: v_trunc_f32_e32 v3, v3 +; GFX6-NEXT: v_mac_f32_e32 v2, s21, v3 +; GFX6-NEXT: v_cvt_u32_f32_e32 v9, v2 +; GFX6-NEXT: v_cvt_u32_f32_e32 v10, v3 +; GFX6-NEXT: s_sub_u32 s13, 0, s8 +; GFX6-NEXT: s_subb_u32 s16, 0, s9 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[14:15], s13, v9, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, s13, v10 +; GFX6-NEXT: v_mul_lo_u32 v11, s16, v9 +; GFX6-NEXT: v_mul_hi_u32 v12, v9, v2 +; GFX6-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[0:1] +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3 +; GFX6-NEXT: v_add_i32_e32 v11, vcc, v3, v11 +; GFX6-NEXT: v_mad_u64_u32 v[3:4], s[14:15], v9, v11, 0 +; GFX6-NEXT: v_cndmask_b32_e64 v6, v6, v5, s[2:3] ; GFX6-NEXT: s_ashr_i32 s14, s11, 31 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GFX6-NEXT: v_mul_lo_u32 v7, s0, v5 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; GFX6-NEXT: v_mul_lo_u32 v8, v5, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, v5, v7 -; GFX6-NEXT: v_mul_hi_u32 v10, v5, v4 -; GFX6-NEXT: v_mul_hi_u32 v11, v6, v4 -; GFX6-NEXT: v_mul_lo_u32 v4, v6, v4 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc -; GFX6-NEXT: v_mul_lo_u32 v10, v6, v7 -; GFX6-NEXT: v_mul_hi_u32 v7, v6, v7 -; GFX6-NEXT: s_mov_b32 s15, s14 -; GFX6-NEXT: v_xor_b32_e32 v2, s12, v2 -; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v9, v7, vcc -; GFX6-NEXT: v_addc_u32_e32 v8, vcc, v11, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v7, v4 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v1, v8, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v6, v7, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, s0, v5 -; GFX6-NEXT: v_mul_hi_u32 v7, s0, v4 -; GFX6-NEXT: v_mul_lo_u32 v8, s1, v4 -; GFX6-NEXT: v_xor_b32_e32 v3, s12, v3 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GFX6-NEXT: v_mul_lo_u32 v7, s0, v4 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v8, v6 -; GFX6-NEXT: v_mul_lo_u32 v10, v4, v6 -; GFX6-NEXT: v_mul_hi_u32 v11, v4, v7 -; GFX6-NEXT: v_mul_hi_u32 v12, v4, v6 -; GFX6-NEXT: v_mul_hi_u32 v9, v5, v7 -; GFX6-NEXT: v_mul_lo_u32 v7, v5, v7 -; GFX6-NEXT: v_mul_hi_u32 v8, v5, v6 -; GFX6-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; GFX6-NEXT: v_addc_u32_e32 v11, vcc, 0, v12, vcc -; GFX6-NEXT: v_mul_lo_u32 v6, v5, v6 -; GFX6-NEXT: v_add_i32_e32 v7, vcc, v10, v7 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v11, v9, vcc -; GFX6-NEXT: v_addc_u32_e32 v8, vcc, v8, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v1, v8, vcc +; GFX6-NEXT: v_add_i32_e32 v7, vcc, v12, v3 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v10, v2, 0 +; GFX6-NEXT: v_addc_u32_e32 v12, vcc, 0, v4, vcc +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v10, v11, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v7, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v12, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v2 +; GFX6-NEXT: v_addc_u32_e32 v10, vcc, v10, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s13, v9, 0 +; GFX6-NEXT: v_xor_b32_e32 v11, s12, v6 +; GFX6-NEXT: v_mul_lo_u32 v6, s13, v10 +; GFX6-NEXT: v_mul_lo_u32 v7, s16, v9 +; GFX6-NEXT: v_mul_hi_u32 v12, v9, v2 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v10, v2, 0 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v7, v3 +; GFX6-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v9, v3, 0 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v10, v3, 0 +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v12, v6 +; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v3, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc ; GFX6-NEXT: s_add_u32 s0, s10, s14 -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6 +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v9, v2 +; GFX6-NEXT: s_mov_b32 s15, s14 ; GFX6-NEXT: s_addc_u32 s1, s11, s14 -; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc +; GFX6-NEXT: v_addc_u32_e32 v5, vcc, v10, v3, vcc ; GFX6-NEXT: s_xor_b64 s[10:11], s[0:1], s[14:15] -; GFX6-NEXT: v_mul_lo_u32 v6, s10, v5 -; GFX6-NEXT: v_mul_hi_u32 v7, s10, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, s10, v5 -; GFX6-NEXT: v_mul_hi_u32 v10, s11, v5 -; GFX6-NEXT: v_mul_lo_u32 v5, s11, v5 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GFX6-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc -; GFX6-NEXT: v_mul_lo_u32 v9, s11, v4 -; GFX6-NEXT: v_mul_hi_u32 v4, s11, v4 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s10, v5, 0 +; GFX6-NEXT: v_mul_hi_u32 v6, s10, v4 +; GFX6-NEXT: v_xor_b32_e32 v7, s12, v8 ; GFX6-NEXT: v_mov_b32_e32 v8, s12 -; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v9 -; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v7, v4, vcc -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v10, v0, vcc -; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v1, v0, vcc +; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s11, v4, 0 +; GFX6-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s11, v5, 0 +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v9, v3, vcc +; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v5, v1, vcc +; GFX6-NEXT: v_add_i32_e32 v4, vcc, v2, v4 +; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v0, v1, vcc ; GFX6-NEXT: v_mul_lo_u32 v5, s8, v0 -; GFX6-NEXT: v_mul_hi_u32 v6, s8, v4 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s12, v2 -; GFX6-NEXT: v_mul_lo_u32 v2, s9, v4 -; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v3, v8, vcc -; GFX6-NEXT: v_add_i32_e32 v3, vcc, v6, v5 -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GFX6-NEXT: v_mul_lo_u32 v3, s8, v4 -; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s11, v2 +; GFX6-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s8, v4, 0 +; GFX6-NEXT: v_mul_lo_u32 v4, s9, v4 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s12, v11 +; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v7, v8, vcc +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 +; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s11, v3 ; GFX6-NEXT: v_mov_b32_e32 v5, s9 -; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s10, v3 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s10, v2 ; GFX6-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GFX6-NEXT: v_subrev_i32_e64 v6, s[0:1], s8, v3 +; GFX6-NEXT: v_subrev_i32_e64 v6, s[0:1], s8, v2 ; GFX6-NEXT: v_subbrev_u32_e64 v7, s[2:3], 0, v4, s[0:1] ; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s9, v7 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[2:3] @@ -14066,22 +13923,22 @@ ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v8 ; GFX6-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[0:1] ; GFX6-NEXT: v_mov_b32_e32 v7, s11 -; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v7, v2, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v7, v3, vcc +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s8, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s9, v3 ; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc ; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7 -; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX6-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] ; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc +; GFX6-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] +; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GFX6-NEXT: v_xor_b32_e32 v2, s14, v2 ; GFX6-NEXT: v_xor_b32_e32 v3, s14, v3 -; GFX6-NEXT: v_xor_b32_e32 v4, s14, v2 -; GFX6-NEXT: v_mov_b32_e32 v5, s14 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s14, v3 -; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v4, v5, vcc +; GFX6-NEXT: v_mov_b32_e32 v4, s14 +; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s14, v2 +; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 ; GFX6-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll @@ -774,12 +774,10 @@ ; GFX7LESS-NEXT: s_mov_b32 s12, s6 ; GFX7LESS-NEXT: s_mov_b32 s13, s7 ; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[8:9] -; GFX7LESS-NEXT: s_mul_i32 s7, s1, s6 -; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 -; GFX7LESS-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX7LESS-NEXT: s_mul_i32 s6, s0, s6 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0 +; GFX7LESS-NEXT: s_mul_i32 s8, s1, s6 ; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s0, v0, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s8, v1 ; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc ; GFX7LESS-NEXT: s_waitcnt vmcnt(0) @@ -791,14 +789,13 @@ ; GFX7LESS-NEXT: s_mov_b32 s6, -1 ; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v0 ; GFX7LESS-NEXT: v_readfirstlane_b32 s3, v1 +; GFX7LESS-NEXT: v_mul_lo_u32 v3, s1, v2 ; GFX7LESS-NEXT: s_waitcnt expcnt(0) -; GFX7LESS-NEXT: v_mul_lo_u32 v0, s1, v2 -; GFX7LESS-NEXT: v_mul_hi_u32 v1, s0, v2 -; GFX7LESS-NEXT: v_mul_lo_u32 v2, s0, v2 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v0 -; GFX7LESS-NEXT: v_mov_b32_e32 v3, s3 -; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s2, v2 -; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s0, v2, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3 +; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s2, v0 +; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc ; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX7LESS-NEXT: s_endpgm ; @@ -818,32 +815,29 @@ ; GFX8-NEXT: s_mov_b32 s12, s6 ; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[8:9] ; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX8-NEXT: s_mov_b32 s13, s7 -; GFX8-NEXT: s_mul_i32 s7, s1, s6 -; GFX8-NEXT: s_mul_i32 s6, s0, s6 +; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s0, v0, 0 +; GFX8-NEXT: s_mul_i32 s6, s1, s6 ; GFX8-NEXT: s_mov_b32 s15, 0xf000 ; GFX8-NEXT: s_mov_b32 s14, -1 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, s7, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s6 +; GFX8-NEXT: s_mov_b32 s13, s7 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: buffer_wbinvl1_vol ; GFX8-NEXT: BB4_2: ; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX8-NEXT: v_readfirstlane_b32 s2, v0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mul_lo_u32 v0, s1, v2 -; GFX8-NEXT: v_mul_hi_u32 v3, s0, v2 +; GFX8-NEXT: v_mul_lo_u32 v4, s1, v2 +; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s0, v2, 0 +; GFX8-NEXT: v_readfirstlane_b32 s0, v0 ; GFX8-NEXT: v_readfirstlane_b32 s1, v1 -; GFX8-NEXT: v_mul_lo_u32 v1, s0, v2 -; GFX8-NEXT: s_mov_b32 s7, 0xf000 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v0 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v4 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 -; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v1 +; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2 +; GFX8-NEXT: s_mov_b32 s7, 0xf000 ; GFX8-NEXT: s_mov_b32 s6, -1 -; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v3, v2, vcc +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc ; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX8-NEXT: s_endpgm ; @@ -878,17 +872,16 @@ ; GFX9-NEXT: BB4_2: ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX9-NEXT: v_mul_hi_u32 v4, s2, v2 +; GFX9-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0 ; GFX9-NEXT: v_readfirstlane_b32 s0, v0 -; GFX9-NEXT: v_mul_lo_u32 v0, s2, v2 ; GFX9-NEXT: v_readfirstlane_b32 s1, v1 -; GFX9-NEXT: v_add_u32_e32 v1, v4, v3 -; GFX9-NEXT: v_mov_b32_e32 v2, s1 -; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0 +; GFX9-NEXT: v_add_u32_e32 v1, v3, v4 +; GFX9-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v2 ; GFX9-NEXT: s_mov_b32 s7, 0xf000 ; GFX9-NEXT: s_mov_b32 s6, -1 -; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc ; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX9-NEXT: s_endpgm ; @@ -927,14 +920,13 @@ ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX1064-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1064-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1064-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1064-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1064-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0 ; GFX1064-NEXT: v_readfirstlane_b32 s0, v0 ; GFX1064-NEXT: v_readfirstlane_b32 s1, v1 ; GFX1064-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1064-NEXT: s_mov_b32 s6, -1 -; GFX1064-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1064-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1064-NEXT: v_add_co_u32 v0, vcc, s0, v2 ; GFX1064-NEXT: v_add_co_ci_u32_e32 v1, vcc, s1, v1, vcc ; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 @@ -974,14 +966,13 @@ ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX1032-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1032-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1032-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1032-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1032-NEXT: v_mad_u64_u32 v[2:3], s0, s2, v2, 0 ; GFX1032-NEXT: v_readfirstlane_b32 s0, v0 ; GFX1032-NEXT: v_readfirstlane_b32 s1, v1 ; GFX1032-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1032-NEXT: s_mov_b32 s6, -1 -; GFX1032-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1032-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1032-NEXT: v_add_co_u32 v0, vcc_lo, s0, v2 ; GFX1032-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo ; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 @@ -1911,12 +1902,10 @@ ; GFX7LESS-NEXT: s_mov_b32 s12, s6 ; GFX7LESS-NEXT: s_mov_b32 s13, s7 ; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[8:9] -; GFX7LESS-NEXT: s_mul_i32 s7, s1, s6 -; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 -; GFX7LESS-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX7LESS-NEXT: s_mul_i32 s6, s0, s6 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0 +; GFX7LESS-NEXT: s_mul_i32 s8, s1, s6 ; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s0, v0, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s8, v1 ; GFX7LESS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX7LESS-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[12:15], 0 glc ; GFX7LESS-NEXT: s_waitcnt vmcnt(0) @@ -1928,14 +1917,13 @@ ; GFX7LESS-NEXT: s_mov_b32 s6, -1 ; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v0 ; GFX7LESS-NEXT: v_readfirstlane_b32 s3, v1 +; GFX7LESS-NEXT: v_mul_lo_u32 v3, s1, v2 ; GFX7LESS-NEXT: s_waitcnt expcnt(0) -; GFX7LESS-NEXT: v_mul_lo_u32 v0, s1, v2 -; GFX7LESS-NEXT: v_mul_hi_u32 v1, s0, v2 -; GFX7LESS-NEXT: v_mul_lo_u32 v2, s0, v2 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v0 -; GFX7LESS-NEXT: v_mov_b32_e32 v3, s3 -; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s2, v2 -; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s0, v2, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3 +; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc ; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX7LESS-NEXT: s_endpgm ; @@ -1955,32 +1943,29 @@ ; GFX8-NEXT: s_mov_b32 s12, s6 ; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[8:9] ; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX8-NEXT: s_mov_b32 s13, s7 -; GFX8-NEXT: s_mul_i32 s7, s1, s6 -; GFX8-NEXT: s_mul_i32 s6, s0, s6 +; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s0, v0, 0 +; GFX8-NEXT: s_mul_i32 s6, s1, s6 ; GFX8-NEXT: s_mov_b32 s15, 0xf000 ; GFX8-NEXT: s_mov_b32 s14, -1 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, s7, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s6 +; GFX8-NEXT: s_mov_b32 s13, s7 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1 ; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX8-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[12:15], 0 glc ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: buffer_wbinvl1_vol ; GFX8-NEXT: BB10_2: ; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX8-NEXT: v_readfirstlane_b32 s2, v0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mul_lo_u32 v0, s1, v2 -; GFX8-NEXT: v_mul_hi_u32 v3, s0, v2 +; GFX8-NEXT: v_mul_lo_u32 v4, s1, v2 +; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s0, v2, 0 +; GFX8-NEXT: v_readfirstlane_b32 s0, v0 ; GFX8-NEXT: v_readfirstlane_b32 s1, v1 -; GFX8-NEXT: v_mul_lo_u32 v1, s0, v2 -; GFX8-NEXT: s_mov_b32 s7, 0xf000 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v0 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v4 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 -; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s2, v1 +; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v2 +; GFX8-NEXT: s_mov_b32 s7, 0xf000 ; GFX8-NEXT: s_mov_b32 s6, -1 -; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc +; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc ; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX8-NEXT: s_endpgm ; @@ -2015,17 +2000,16 @@ ; GFX9-NEXT: BB10_2: ; GFX9-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX9-NEXT: v_mul_hi_u32 v4, s2, v2 +; GFX9-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0 ; GFX9-NEXT: v_readfirstlane_b32 s0, v0 -; GFX9-NEXT: v_mul_lo_u32 v0, s2, v2 ; GFX9-NEXT: v_readfirstlane_b32 s1, v1 -; GFX9-NEXT: v_add_u32_e32 v1, v4, v3 -; GFX9-NEXT: v_mov_b32_e32 v2, s1 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v0 +; GFX9-NEXT: v_add_u32_e32 v1, v3, v4 +; GFX9-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v2 ; GFX9-NEXT: s_mov_b32 s7, 0xf000 ; GFX9-NEXT: s_mov_b32 s6, -1 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v1, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc ; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX9-NEXT: s_endpgm ; @@ -2064,14 +2048,13 @@ ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX1064-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1064-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1064-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1064-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1064-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0 ; GFX1064-NEXT: v_readfirstlane_b32 s0, v0 ; GFX1064-NEXT: v_readfirstlane_b32 s1, v1 ; GFX1064-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1064-NEXT: s_mov_b32 s6, -1 -; GFX1064-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1064-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1064-NEXT: v_sub_co_u32 v0, vcc, s0, v2 ; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s1, v1, vcc ; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 @@ -2111,14 +2094,13 @@ ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX1032-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1032-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1032-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1032-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1032-NEXT: v_mad_u64_u32 v[2:3], s0, s2, v2, 0 ; GFX1032-NEXT: v_readfirstlane_b32 s0, v0 ; GFX1032-NEXT: v_readfirstlane_b32 s1, v1 ; GFX1032-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1032-NEXT: s_mov_b32 s6, -1 -; GFX1032-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1032-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1032-NEXT: v_sub_co_u32 v0, vcc_lo, s0, v2 ; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo ; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll @@ -914,12 +914,10 @@ ; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7] ; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0 ; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) -; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6 -; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 -; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0 -; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0 +; GFX7LESS-NEXT: s_mul_i32 s8, s3, s6 ; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s8, v1 ; GFX7LESS-NEXT: s_mov_b32 m0, -1 ; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) ; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1] @@ -931,15 +929,14 @@ ; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) ; GFX7LESS-NEXT: s_mov_b32 s4, s0 ; GFX7LESS-NEXT: s_mov_b32 s5, s1 -; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0 -; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v1 -; GFX7LESS-NEXT: v_mul_lo_u32 v0, s3, v2 -; GFX7LESS-NEXT: v_mul_hi_u32 v1, s2, v2 -; GFX7LESS-NEXT: v_mul_lo_u32 v2, s2, v2 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v0 -; GFX7LESS-NEXT: v_mov_b32_e32 v3, s1 -; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s0, v2 -; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc +; GFX7LESS-NEXT: v_readfirstlane_b32 s8, v0 +; GFX7LESS-NEXT: v_readfirstlane_b32 s9, v1 +; GFX7LESS-NEXT: v_mul_lo_u32 v3, s3, v2 +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v2, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s9 +; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s8, v0 +; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc ; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX7LESS-NEXT: s_endpgm ; @@ -954,15 +951,13 @@ ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX8-NEXT: s_cbranch_execz BB5_2 ; GFX8-NEXT: ; %bb.1: -; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX8-NEXT: v_mov_b32_e32 v0, s6 +; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[6:7] +; GFX8-NEXT: v_mov_b32_e32 v0, s8 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0 -; GFX8-NEXT: s_mul_i32 s7, s3, s6 -; GFX8-NEXT: s_mul_i32 s6, s2, s6 +; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0 +; GFX8-NEXT: s_mul_i32 s6, s3, s8 ; GFX8-NEXT: v_mov_b32_e32 v3, 0 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, s7, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s6 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1 ; GFX8-NEXT: s_mov_b32 m0, -1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1] @@ -971,18 +966,17 @@ ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 s4, s0 -; GFX8-NEXT: v_readfirstlane_b32 s0, v0 -; GFX8-NEXT: v_mul_lo_u32 v0, s3, v2 -; GFX8-NEXT: v_mul_hi_u32 v3, s2, v2 ; GFX8-NEXT: s_mov_b32 s5, s1 +; GFX8-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0 +; GFX8-NEXT: v_readfirstlane_b32 s0, v0 ; GFX8-NEXT: v_readfirstlane_b32 s1, v1 -; GFX8-NEXT: v_mul_lo_u32 v1, s2, v2 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v0 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v4 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 -; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v1 +; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2 ; GFX8-NEXT: s_mov_b32 s7, 0xf000 ; GFX8-NEXT: s_mov_b32 s6, -1 -; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v3, v2, vcc +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc ; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX8-NEXT: s_endpgm ; @@ -1012,19 +1006,18 @@ ; GFX9-NEXT: BB5_2: ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0 ; GFX9-NEXT: s_mov_b32 s4, s0 -; GFX9-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX9-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX9-NEXT: v_readfirstlane_b32 s0, v0 -; GFX9-NEXT: v_mul_lo_u32 v0, s2, v2 ; GFX9-NEXT: s_mov_b32 s5, s1 +; GFX9-NEXT: v_readfirstlane_b32 s0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s1, v1 -; GFX9-NEXT: v_add_u32_e32 v1, v4, v3 -; GFX9-NEXT: v_mov_b32_e32 v2, s1 -; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0 +; GFX9-NEXT: v_add_u32_e32 v1, v3, v4 +; GFX9-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v2 ; GFX9-NEXT: s_mov_b32 s7, 0xf000 ; GFX9-NEXT: s_mov_b32 s6, -1 -; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc ; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX9-NEXT: s_endpgm ; @@ -1057,13 +1050,12 @@ ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1064-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1064-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1064-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1064-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0 ; GFX1064-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1064-NEXT: v_readfirstlane_b32 s4, v1 ; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1064-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1064-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1064-NEXT: v_add_co_u32 v0, vcc, s2, v2 ; GFX1064-NEXT: s_mov_b32 s2, -1 ; GFX1064-NEXT: v_add_co_ci_u32_e32 v1, vcc, s4, v1, vcc @@ -1098,13 +1090,12 @@ ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1032-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1032-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1032-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1032-NEXT: v_mad_u64_u32 v[2:3], s2, s2, v2, 0 ; GFX1032-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1032-NEXT: v_readfirstlane_b32 s4, v1 ; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1032-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1032-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1032-NEXT: v_add_co_u32 v0, vcc_lo, s2, v2 ; GFX1032-NEXT: s_mov_b32 s2, -1 ; GFX1032-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo @@ -2093,12 +2084,10 @@ ; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7] ; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0 ; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) -; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6 -; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 -; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0 -; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0 +; GFX7LESS-NEXT: s_mul_i32 s8, s3, s6 ; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6 +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s8, v1 ; GFX7LESS-NEXT: s_mov_b32 m0, -1 ; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) ; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1] @@ -2110,15 +2099,14 @@ ; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) ; GFX7LESS-NEXT: s_mov_b32 s4, s0 ; GFX7LESS-NEXT: s_mov_b32 s5, s1 -; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0 -; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v1 -; GFX7LESS-NEXT: v_mul_lo_u32 v0, s3, v2 -; GFX7LESS-NEXT: v_mul_hi_u32 v1, s2, v2 -; GFX7LESS-NEXT: v_mul_lo_u32 v2, s2, v2 -; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v0 -; GFX7LESS-NEXT: v_mov_b32_e32 v3, s1 -; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s0, v2 -; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc +; GFX7LESS-NEXT: v_readfirstlane_b32 s8, v0 +; GFX7LESS-NEXT: v_readfirstlane_b32 s9, v1 +; GFX7LESS-NEXT: v_mul_lo_u32 v3, s3, v2 +; GFX7LESS-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v2, 0 +; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s9 +; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s8, v0 +; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc ; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX7LESS-NEXT: s_endpgm ; @@ -2133,15 +2121,13 @@ ; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX8-NEXT: s_cbranch_execz BB12_2 ; GFX8-NEXT: ; %bb.1: -; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX8-NEXT: v_mov_b32_e32 v0, s6 +; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[6:7] +; GFX8-NEXT: v_mov_b32_e32 v0, s8 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0 -; GFX8-NEXT: s_mul_i32 s7, s3, s6 -; GFX8-NEXT: s_mul_i32 s6, s2, s6 +; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0 +; GFX8-NEXT: s_mul_i32 s6, s3, s8 ; GFX8-NEXT: v_mov_b32_e32 v3, 0 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, s7, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s6 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1 ; GFX8-NEXT: s_mov_b32 m0, -1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1] @@ -2150,18 +2136,17 @@ ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_mov_b32 s4, s0 -; GFX8-NEXT: v_readfirstlane_b32 s0, v0 -; GFX8-NEXT: v_mul_lo_u32 v0, s3, v2 -; GFX8-NEXT: v_mul_hi_u32 v3, s2, v2 ; GFX8-NEXT: s_mov_b32 s5, s1 +; GFX8-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v2, 0 +; GFX8-NEXT: v_readfirstlane_b32 s0, v0 ; GFX8-NEXT: v_readfirstlane_b32 s1, v1 -; GFX8-NEXT: v_mul_lo_u32 v1, s2, v2 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v0 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v4 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 -; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v1 +; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v2 ; GFX8-NEXT: s_mov_b32 s7, 0xf000 ; GFX8-NEXT: s_mov_b32 s6, -1 -; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc +; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc ; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX8-NEXT: s_endpgm ; @@ -2191,19 +2176,18 @@ ; GFX9-NEXT: BB12_2: ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0 ; GFX9-NEXT: s_mov_b32 s4, s0 -; GFX9-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX9-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX9-NEXT: v_readfirstlane_b32 s0, v0 -; GFX9-NEXT: v_mul_lo_u32 v0, s2, v2 ; GFX9-NEXT: s_mov_b32 s5, s1 +; GFX9-NEXT: v_readfirstlane_b32 s0, v0 ; GFX9-NEXT: v_readfirstlane_b32 s1, v1 -; GFX9-NEXT: v_add_u32_e32 v1, v4, v3 -; GFX9-NEXT: v_mov_b32_e32 v2, s1 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v0 +; GFX9-NEXT: v_add_u32_e32 v1, v3, v4 +; GFX9-NEXT: v_mov_b32_e32 v3, s1 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v2 ; GFX9-NEXT: s_mov_b32 s7, 0xf000 ; GFX9-NEXT: s_mov_b32 s6, -1 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v1, vcc +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc ; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX9-NEXT: s_endpgm ; @@ -2236,13 +2220,12 @@ ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1064-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1064-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1064-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1064-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s2, v2, 0 ; GFX1064-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1064-NEXT: v_readfirstlane_b32 s4, v1 ; GFX1064-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1064-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1064-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1064-NEXT: v_sub_co_u32 v0, vcc, s2, v2 ; GFX1064-NEXT: s_mov_b32 s2, -1 ; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc @@ -2277,13 +2260,12 @@ ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-NEXT: v_mul_lo_u32 v3, s3, v2 -; GFX1032-NEXT: v_mul_hi_u32 v4, s2, v2 -; GFX1032-NEXT: v_mul_lo_u32 v2, s2, v2 +; GFX1032-NEXT: v_mul_lo_u32 v4, s3, v2 +; GFX1032-NEXT: v_mad_u64_u32 v[2:3], s2, s2, v2, 0 ; GFX1032-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1032-NEXT: v_readfirstlane_b32 s4, v1 ; GFX1032-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1032-NEXT: v_add_nc_u32_e32 v1, v4, v3 +; GFX1032-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; GFX1032-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v2 ; GFX1032-NEXT: s_mov_b32 s2, -1 ; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo diff --git a/llvm/test/CodeGen/AMDGPU/bypass-div.ll b/llvm/test/CodeGen/AMDGPU/bypass-div.ll --- a/llvm/test/CodeGen/AMDGPU/bypass-div.ll +++ b/llvm/test/CodeGen/AMDGPU/bypass-div.ll @@ -16,120 +16,108 @@ ; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5] ; GFX9-NEXT: s_cbranch_execz BB0_2 ; GFX9-NEXT: ; %bb.1: -; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3 -; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 -; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc -; GFX9-NEXT: v_xor_b32_e32 v3, v3, v4 -; GFX9-NEXT: v_xor_b32_e32 v2, v2, v4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v6, v3 -; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, 0, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, 0, v3, vcc -; GFX9-NEXT: v_mac_f32_e32 v5, 0x4f800000, v6 -; GFX9-NEXT: v_rcp_f32_e32 v5, v5 +; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v3 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v9 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v9, vcc +; GFX9-NEXT: v_xor_b32_e32 v10, v3, v9 +; GFX9-NEXT: v_xor_b32_e32 v11, v2, v9 +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v11 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v10 +; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, 0, v11 +; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, 0, v10, vcc +; GFX9-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 +; GFX9-NEXT: v_rcp_f32_e32 v2, v2 ; GFX9-NEXT: v_mov_b32_e32 v15, 0 ; GFX9-NEXT: v_mov_b32_e32 v14, 0 -; GFX9-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5 -; GFX9-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5 -; GFX9-NEXT: v_trunc_f32_e32 v6, v6 -; GFX9-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6 -; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v6 -; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: v_mul_lo_u32 v11, v7, v6 -; GFX9-NEXT: v_mul_lo_u32 v9, v8, v5 -; GFX9-NEXT: v_mul_hi_u32 v10, v7, v5 -; GFX9-NEXT: v_mul_lo_u32 v12, v7, v5 -; GFX9-NEXT: v_add3_u32 v9, v10, v11, v9 -; GFX9-NEXT: v_mul_lo_u32 v11, v5, v9 -; GFX9-NEXT: v_mul_hi_u32 v13, v5, v12 -; GFX9-NEXT: v_mul_hi_u32 v10, v5, v9 -; GFX9-NEXT: v_mul_hi_u32 v16, v6, v9 -; GFX9-NEXT: v_mul_lo_u32 v9, v6, v9 -; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v13, v11 -; GFX9-NEXT: v_mul_lo_u32 v13, v6, v12 -; GFX9-NEXT: v_mul_hi_u32 v12, v6, v12 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v10, vcc -; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v13 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v10, v12, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v16, v14, vcc -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v10, v9 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v11, vcc -; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v9 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v10, vcc -; GFX9-NEXT: v_mul_lo_u32 v9, v7, v6 -; GFX9-NEXT: v_mul_lo_u32 v8, v8, v5 -; GFX9-NEXT: v_mul_hi_u32 v10, v7, v5 -; GFX9-NEXT: v_mul_lo_u32 v7, v7, v5 -; GFX9-NEXT: v_add3_u32 v8, v10, v9, v8 -; GFX9-NEXT: v_mul_lo_u32 v11, v5, v8 -; GFX9-NEXT: v_mul_hi_u32 v12, v5, v7 -; GFX9-NEXT: v_mul_hi_u32 v13, v5, v8 -; GFX9-NEXT: v_mul_hi_u32 v10, v6, v7 -; GFX9-NEXT: v_mul_lo_u32 v7, v6, v7 -; GFX9-NEXT: v_mul_hi_u32 v9, v6, v8 -; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v12, v11 -; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, v15, v13, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v8 -; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v11, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v12, v10, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v14, vcc -; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v15, v9, vcc -; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v8, vcc -; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v1 -; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v7 -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v7 -; GFX9-NEXT: v_mul_lo_u32 v8, v0, v6 -; GFX9-NEXT: v_mul_hi_u32 v9, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v10, v0, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v7, vcc -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v7 -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v9, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v15, v10, vcc -; GFX9-NEXT: v_mul_lo_u32 v10, v1, v5 -; GFX9-NEXT: v_mul_hi_u32 v5, v1, v5 -; GFX9-NEXT: v_mul_hi_u32 v11, v1, v6 -; GFX9-NEXT: v_mul_lo_u32 v6, v1, v6 -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v8, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v9, v5, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v11, v14, vcc -; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v15, v8, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v3, v5 -; GFX9-NEXT: v_mul_lo_u32 v9, v2, v6 -; GFX9-NEXT: v_mul_hi_u32 v10, v2, v5 -; GFX9-NEXT: v_mul_lo_u32 v11, v2, v5 -; GFX9-NEXT: v_add3_u32 v8, v10, v9, v8 -; GFX9-NEXT: v_sub_u32_e32 v9, v1, v8 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v11 -; GFX9-NEXT: v_subb_co_u32_e64 v9, s[4:5], v9, v3, vcc -; GFX9-NEXT: v_sub_co_u32_e64 v10, s[4:5], v0, v2 -; GFX9-NEXT: v_subbrev_co_u32_e64 v9, s[4:5], 0, v9, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v2 -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v9, v11, v10, s[4:5] -; GFX9-NEXT: v_add_co_u32_e64 v10, s[4:5], 2, v5 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v8, vcc -; GFX9-NEXT: v_addc_co_u32_e64 v11, s[4:5], 0, v6, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_add_co_u32_e64 v12, s[4:5], 1, v5 -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GFX9-NEXT: v_addc_co_u32_e64 v13, s[4:5], 0, v6, s[4:5] +; GFX9-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 +; GFX9-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 +; GFX9-NEXT: v_trunc_f32_e32 v3, v3 +; GFX9-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 +; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v3 +; GFX9-NEXT: v_mul_lo_u32 v4, v8, v6 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v6, 0 +; GFX9-NEXT: v_mul_lo_u32 v5, v7, v12 +; GFX9-NEXT: v_mul_hi_u32 v13, v6, v2 +; GFX9-NEXT: v_add3_u32 v5, v3, v5, v4 +; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v13, v3 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v2, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v16, vcc, v15, v4, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v5, 0 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v13, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v16, v3, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v14, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v15, v3, vcc +; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v6, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, v12, v3, vcc +; GFX9-NEXT: v_mul_lo_u32 v4, v7, v12 +; GFX9-NEXT: v_mul_lo_u32 v5, v8, v13 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v13, 0 +; GFX9-NEXT: v_add3_u32 v5, v3, v4, v5 +; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v5, 0 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v5, 0 +; GFX9-NEXT: v_mul_hi_u32 v16, v13, v2 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v2, 0 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v16, v5 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v15, v6, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v7 +; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v5, v8, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v14, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v3 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v15, v4, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v13, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v12, v3, vcc +; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v1 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4 +; GFX9-NEXT: v_xor_b32_e32 v6, v0, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v4, vcc +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0 +; GFX9-NEXT: v_mul_hi_u32 v7, v6, v2 +; GFX9-NEXT: v_xor_b32_e32 v5, v5, v4 +; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v15, v1, vcc +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v7, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v8, v1, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v14, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v15, v1, vcc +; GFX9-NEXT: v_mul_lo_u32 v7, v10, v2 +; GFX9-NEXT: v_mul_lo_u32 v8, v11, v3 +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v2, 0 +; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7 +; GFX9-NEXT: v_sub_u32_e32 v7, v5, v1 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v6, v0 +; GFX9-NEXT: v_subb_co_u32_e64 v6, s[4:5], v7, v10, vcc +; GFX9-NEXT: v_sub_co_u32_e64 v7, s[4:5], v0, v11 +; GFX9-NEXT: v_subbrev_co_u32_e64 v6, s[4:5], 0, v6, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v11 +; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v7, s[4:5] +; GFX9-NEXT: v_add_co_u32_e64 v7, s[4:5], 2, v2 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v5, v1, vcc +; GFX9-NEXT: v_addc_co_u32_e64 v8, s[4:5], 0, v3, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v10 +; GFX9-NEXT: v_add_co_u32_e64 v12, s[4:5], 1, v2 +; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v11 +; GFX9-NEXT: v_addc_co_u32_e64 v13, s[4:5], 0, v3, s[4:5] ; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v10 +; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v12, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v13, v11, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: v_xor_b32_e32 v2, v7, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v1, v12, v7, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v6, v13, v8, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GFX9-NEXT: v_xor_b32_e32 v2, v4, v9 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc ; GFX9-NEXT: v_xor_b32_e32 v1, v1, v2 ; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2 ; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v1, v2 @@ -184,107 +172,95 @@ ; GFX9-NEXT: ; %bb.1: ; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v3 -; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, 0, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v10, vcc, 0, v2 +; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v3, vcc ; GFX9-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 ; GFX9-NEXT: v_rcp_f32_e32 v4, v4 +; GFX9-NEXT: v_mov_b32_e32 v14, 0 ; GFX9-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 ; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 ; GFX9-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v9, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v10, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v11, v6, v4 -; GFX9-NEXT: v_add3_u32 v8, v10, v8, v9 -; GFX9-NEXT: v_mul_hi_u32 v9, v4, v11 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v14, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v15, v5, v8 -; GFX9-NEXT: v_mul_lo_u32 v8, v5, v8 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v13, v14, vcc -; GFX9-NEXT: v_mul_lo_u32 v14, v5, v11 -; GFX9-NEXT: v_mul_hi_u32 v11, v5, v11 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v14 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v10, v11, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v12, vcc -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v9, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v13, v10, vcc -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v9, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v7, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v9, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v6, v6, v4 -; GFX9-NEXT: v_add3_u32 v7, v9, v8, v7 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v11, v4, v6 -; GFX9-NEXT: v_mul_hi_u32 v14, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v9, v5, v6 -; GFX9-NEXT: v_mul_lo_u32 v6, v5, v6 -; GFX9-NEXT: v_mul_hi_u32 v8, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v11, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v13, v14, vcc -; GFX9-NEXT: v_mul_lo_u32 v7, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v10, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v11, v9, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v12, vcc -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v13, v8, vcc +; GFX9-NEXT: v_cvt_u32_f32_e32 v8, v5 +; GFX9-NEXT: v_cvt_u32_f32_e32 v9, v4 +; GFX9-NEXT: v_mul_lo_u32 v6, v10, v8 +; GFX9-NEXT: v_mul_lo_u32 v7, v11, v9 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0 +; GFX9-NEXT: v_add3_u32 v7, v5, v6, v7 +; GFX9-NEXT: v_mul_hi_u32 v12, v9, v4 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v5 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v14, v6, vcc +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v12, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v15, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v13, vcc ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v7, vcc -; GFX9-NEXT: v_mul_lo_u32 v6, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v7, v0, v4 -; GFX9-NEXT: v_mul_hi_u32 v8, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v9, v1, v5 -; GFX9-NEXT: v_mul_lo_u32 v5, v1, v5 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v7, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v13, v8, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v1, v4 -; GFX9-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v7, v4, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v9, v12, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v5, vcc +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v9, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v8, v5, vcc +; GFX9-NEXT: v_mul_lo_u32 v6, v10, v15 +; GFX9-NEXT: v_mul_lo_u32 v7, v11, v12 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0 +; GFX9-NEXT: v_add3_u32 v7, v5, v6, v7 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v15, v7, 0 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v7, 0 +; GFX9-NEXT: v_mul_hi_u32 v11, v12, v4 +; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v4, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v11, v7 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v14, v8, vcc +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v9 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v7, v10, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v13, vcc ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v13, v6, vcc -; GFX9-NEXT: v_mul_lo_u32 v6, v3, v4 -; GFX9-NEXT: v_mul_lo_u32 v7, v2, v5 -; GFX9-NEXT: v_mul_hi_u32 v8, v2, v4 -; GFX9-NEXT: v_mul_lo_u32 v9, v2, v4 -; GFX9-NEXT: v_add3_u32 v6, v8, v7, v6 -; GFX9-NEXT: v_sub_u32_e32 v7, v1, v6 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v9 -; GFX9-NEXT: v_subb_co_u32_e64 v7, s[4:5], v7, v3, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v6, vcc +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v12, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v15, v5, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0 +; GFX9-NEXT: v_mul_hi_u32 v8, v0, v6 +; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v8, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v14, v5, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0 +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v8, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v9, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v13, vcc +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v4, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v14, v5, vcc +; GFX9-NEXT: v_mul_lo_u32 v8, v3, v6 +; GFX9-NEXT: v_mul_lo_u32 v9, v2, v7 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0 +; GFX9-NEXT: v_add3_u32 v5, v5, v9, v8 +; GFX9-NEXT: v_sub_u32_e32 v8, v1, v5 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4 +; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v8, v3, vcc ; GFX9-NEXT: v_sub_co_u32_e64 v8, s[4:5], v0, v2 -; GFX9-NEXT: v_subbrev_co_u32_e64 v7, s[4:5], 0, v7, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v3 +; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[4:5], 0, v4, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5] ; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v2 ; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v9, v8, s[4:5] -; GFX9-NEXT: v_add_co_u32_e64 v8, s[4:5], 2, v4 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, v5, s[4:5] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v3 +; GFX9-NEXT: v_cndmask_b32_e64 v4, v9, v8, s[4:5] +; GFX9-NEXT: v_add_co_u32_e64 v8, s[4:5], 2, v6 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, v7, s[4:5] ; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_add_co_u32_e64 v10, s[4:5], 1, v4 -; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc +; GFX9-NEXT: v_add_co_u32_e64 v10, s[4:5], 1, v6 +; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc ; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GFX9-NEXT: v_addc_co_u32_e64 v11, s[4:5], 0, v5, s[4:5] +; GFX9-NEXT: v_addc_co_u32_e64 v11, s[4:5], 0, v7, s[4:5] ; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v7, v11, v9, s[4:5] +; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v4, v11, v9, s[4:5] ; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 ; GFX9-NEXT: v_cndmask_b32_e64 v0, v10, v8, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v4, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v0, vcc ; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX9-NEXT: BB1_2: ; %Flow @@ -336,119 +312,107 @@ ; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3 ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc -; GFX9-NEXT: v_xor_b32_e32 v3, v3, v4 -; GFX9-NEXT: v_xor_b32_e32 v2, v2, v4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v3 -; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, 0, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc -; GFX9-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 -; GFX9-NEXT: v_rcp_f32_e32 v4, v4 +; GFX9-NEXT: v_xor_b32_e32 v9, v3, v4 +; GFX9-NEXT: v_xor_b32_e32 v10, v2, v4 +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v10 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v9 +; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, 0, v10 +; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, 0, v9, vcc +; GFX9-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 +; GFX9-NEXT: v_rcp_f32_e32 v2, v2 ; GFX9-NEXT: v_mov_b32_e32 v14, 0 ; GFX9-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 -; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 -; GFX9-NEXT: v_trunc_f32_e32 v5, v5 -; GFX9-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GFX9-NEXT: v_mul_lo_u32 v10, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v8, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v9, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v11, v6, v4 -; GFX9-NEXT: v_add3_u32 v8, v9, v10, v8 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v12, v4, v11 -; GFX9-NEXT: v_mul_hi_u32 v9, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v15, v5, v8 -; GFX9-NEXT: v_mul_lo_u32 v8, v5, v8 -; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v12, v10 -; GFX9-NEXT: v_mul_lo_u32 v12, v5, v11 -; GFX9-NEXT: v_mul_hi_u32 v11, v5, v11 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v14, v9, vcc -; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v10, v12 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v11, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v13, vcc -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v9, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v14, v10, vcc -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v9, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v7, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v9, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v6, v6, v4 -; GFX9-NEXT: v_add3_u32 v7, v9, v8, v7 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v11, v4, v6 -; GFX9-NEXT: v_mul_hi_u32 v12, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v9, v5, v6 -; GFX9-NEXT: v_mul_lo_u32 v6, v5, v6 -; GFX9-NEXT: v_mul_hi_u32 v8, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v11, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v14, v12, vcc -; GFX9-NEXT: v_mul_lo_u32 v7, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v10, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v11, v9, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v13, vcc -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v14, v8, vcc -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v7, vcc -; GFX9-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v6 -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v6 -; GFX9-NEXT: v_mul_lo_u32 v7, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v8, v0, v4 -; GFX9-NEXT: v_mul_hi_u32 v9, v0, v5 -; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v6 -; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v8, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v14, v9, vcc -; GFX9-NEXT: v_mul_lo_u32 v9, v1, v4 -; GFX9-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX9-NEXT: v_mul_hi_u32 v10, v1, v5 -; GFX9-NEXT: v_mul_lo_u32 v5, v1, v5 -; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v9 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v8, v4, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v10, v13, vcc -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v7, vcc -; GFX9-NEXT: v_mul_lo_u32 v7, v3, v4 -; GFX9-NEXT: v_mul_lo_u32 v5, v2, v5 -; GFX9-NEXT: v_mul_hi_u32 v8, v2, v4 -; GFX9-NEXT: v_mul_lo_u32 v4, v2, v4 -; GFX9-NEXT: v_add3_u32 v5, v8, v5, v7 -; GFX9-NEXT: v_sub_u32_e32 v7, v1, v5 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4 -; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v7, v3, vcc -; GFX9-NEXT: v_sub_co_u32_e64 v7, s[4:5], v0, v2 -; GFX9-NEXT: v_subbrev_co_u32_e64 v8, s[6:7], 0, v4, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v8, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v7, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[6:7] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v8, v3 -; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v4, v3, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[6:7] -; GFX9-NEXT: v_sub_co_u32_e64 v10, s[4:5], v7, v2 -; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[4:5], 0, v4, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; GFX9-NEXT: v_cndmask_b32_e64 v2, v7, v10, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] +; GFX9-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 +; GFX9-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 +; GFX9-NEXT: v_trunc_f32_e32 v3, v3 +; GFX9-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 +; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v3 +; GFX9-NEXT: v_mul_lo_u32 v4, v8, v6 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v6, 0 +; GFX9-NEXT: v_mul_lo_u32 v5, v7, v11 +; GFX9-NEXT: v_mul_hi_u32 v12, v6, v2 +; GFX9-NEXT: v_add3_u32 v5, v3, v5, v4 +; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v3 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v2, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v14, v4, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v11, v5, 0 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v12, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v15, v3, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v13, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v14, v3, vcc +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v6, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v11, v3, vcc +; GFX9-NEXT: v_mul_lo_u32 v4, v7, v11 +; GFX9-NEXT: v_mul_lo_u32 v5, v8, v12 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v12, 0 +; GFX9-NEXT: v_add3_u32 v5, v3, v4, v5 +; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v5, 0 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v5, 0 +; GFX9-NEXT: v_mul_hi_u32 v15, v12, v2 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v2, 0 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v15, v5 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v6, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v7 +; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v5, v8, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v13, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v3 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v14, v4, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v12, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v11, v3, vcc +; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v1 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v5 +; GFX9-NEXT: v_xor_b32_e32 v6, v0, v5 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0 +; GFX9-NEXT: v_mul_hi_u32 v7, v6, v2 +; GFX9-NEXT: v_xor_b32_e32 v4, v4, v5 +; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v14, v1, vcc +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v7, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v8, v1, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v13, vcc +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v14, v1, vcc +; GFX9-NEXT: v_mul_lo_u32 v2, v9, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, v10, v1 +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v0, 0 +; GFX9-NEXT: v_add3_u32 v1, v1, v3, v2 +; GFX9-NEXT: v_sub_u32_e32 v2, v4, v1 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v6, v0 +; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v9, vcc +; GFX9-NEXT: v_sub_co_u32_e64 v3, s[4:5], v0, v10 +; GFX9-NEXT: v_subbrev_co_u32_e64 v6, s[6:7], 0, v2, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v9 +; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[6:7] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v6, v9 +; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v9, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[6:7] +; GFX9-NEXT: v_sub_co_u32_e64 v8, s[4:5], v3, v10 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v4, v1, vcc +; GFX9-NEXT: v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v9 +; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7 +; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v9 +; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v8, s[4:5] ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v6 -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v6 -; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v0, v6 -; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v6, vcc +; GFX9-NEXT: v_xor_b32_e32 v0, v0, v5 +; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5 +; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v0, v5 +; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v5, vcc ; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX9-NEXT: BB2_2: ; %Flow @@ -497,79 +461,67 @@ ; GFX9-NEXT: ; %bb.1: ; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v3 -; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, 0, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v10, vcc, 0, v2 +; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v3, vcc ; GFX9-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 ; GFX9-NEXT: v_rcp_f32_e32 v4, v4 +; GFX9-NEXT: v_mov_b32_e32 v14, 0 ; GFX9-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 ; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 ; GFX9-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v9, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v10, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v11, v6, v4 -; GFX9-NEXT: v_add3_u32 v8, v10, v8, v9 -; GFX9-NEXT: v_mul_hi_u32 v9, v4, v11 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v14, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v15, v5, v8 -; GFX9-NEXT: v_mul_lo_u32 v8, v5, v8 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v13, v14, vcc -; GFX9-NEXT: v_mul_lo_u32 v14, v5, v11 -; GFX9-NEXT: v_mul_hi_u32 v11, v5, v11 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v14 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v10, v11, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v12, vcc -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v9, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v13, v10, vcc -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v9, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v7, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v9, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v6, v6, v4 -; GFX9-NEXT: v_add3_u32 v7, v9, v8, v7 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v11, v4, v6 -; GFX9-NEXT: v_mul_hi_u32 v14, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v9, v5, v6 -; GFX9-NEXT: v_mul_lo_u32 v6, v5, v6 -; GFX9-NEXT: v_mul_hi_u32 v8, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v11, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v13, v14, vcc -; GFX9-NEXT: v_mul_lo_u32 v7, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v10, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v11, v9, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v12, vcc -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v13, v8, vcc +; GFX9-NEXT: v_cvt_u32_f32_e32 v8, v5 +; GFX9-NEXT: v_cvt_u32_f32_e32 v9, v4 +; GFX9-NEXT: v_mul_lo_u32 v6, v10, v8 +; GFX9-NEXT: v_mul_lo_u32 v7, v11, v9 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0 +; GFX9-NEXT: v_add3_u32 v7, v5, v6, v7 +; GFX9-NEXT: v_mul_hi_u32 v12, v9, v4 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v5 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v14, v6, vcc +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v12, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v15, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v13, vcc ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v7, vcc -; GFX9-NEXT: v_mul_lo_u32 v6, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v7, v0, v4 -; GFX9-NEXT: v_mul_hi_u32 v8, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v9, v1, v5 -; GFX9-NEXT: v_mul_lo_u32 v5, v1, v5 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v7, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v13, v8, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v1, v4 -; GFX9-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v7, v4, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v9, v12, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v5, vcc +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v9, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v8, v5, vcc +; GFX9-NEXT: v_mul_lo_u32 v6, v10, v15 +; GFX9-NEXT: v_mul_lo_u32 v7, v11, v12 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0 +; GFX9-NEXT: v_add3_u32 v7, v5, v6, v7 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v15, v7, 0 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v7, 0 +; GFX9-NEXT: v_mul_hi_u32 v11, v12, v4 +; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v4, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v11, v7 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v14, v8, vcc +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v9 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v7, v10, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v13, vcc ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v13, v6, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v6, vcc +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v12, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v15, v5, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0 +; GFX9-NEXT: v_mul_hi_u32 v8, v0, v6 +; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v8, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v14, v5, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0 +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v8, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v9, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v13, vcc +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v5, vcc ; GFX9-NEXT: v_mul_lo_u32 v6, v3, v4 -; GFX9-NEXT: v_mul_lo_u32 v5, v2, v5 -; GFX9-NEXT: v_mul_hi_u32 v7, v2, v4 -; GFX9-NEXT: v_mul_lo_u32 v4, v2, v4 -; GFX9-NEXT: v_add3_u32 v5, v7, v5, v6 +; GFX9-NEXT: v_mul_lo_u32 v7, v2, v5 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v4, 0 +; GFX9-NEXT: v_add3_u32 v5, v5, v7, v6 ; GFX9-NEXT: v_sub_u32_e32 v6, v1, v5 ; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4 ; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v6, v3, vcc @@ -769,132 +721,120 @@ ; GFX9-NEXT: s_xor_b64 s[10:11], exec, s[4:5] ; GFX9-NEXT: s_cbranch_execz BB8_2 ; GFX9-NEXT: ; %bb.1: -; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3 -; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 -; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc -; GFX9-NEXT: v_xor_b32_e32 v3, v3, v4 -; GFX9-NEXT: v_xor_b32_e32 v2, v2, v4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v6, v3 -; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, 0, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, 0, v3, vcc -; GFX9-NEXT: v_mac_f32_e32 v5, 0x4f800000, v6 -; GFX9-NEXT: v_rcp_f32_e32 v5, v5 +; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v3 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v9 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v9, vcc +; GFX9-NEXT: v_xor_b32_e32 v10, v3, v9 +; GFX9-NEXT: v_xor_b32_e32 v11, v2, v9 +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v11 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v10 +; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, 0, v11 +; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, 0, v10, vcc +; GFX9-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 +; GFX9-NEXT: v_rcp_f32_e32 v2, v2 ; GFX9-NEXT: v_mov_b32_e32 v15, 0 ; GFX9-NEXT: v_mov_b32_e32 v14, 0 -; GFX9-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5 -; GFX9-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5 -; GFX9-NEXT: v_trunc_f32_e32 v6, v6 -; GFX9-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6 -; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v6 -; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: v_mul_lo_u32 v11, v7, v6 -; GFX9-NEXT: v_mul_lo_u32 v9, v8, v5 -; GFX9-NEXT: v_mul_hi_u32 v10, v7, v5 -; GFX9-NEXT: v_mul_lo_u32 v12, v7, v5 -; GFX9-NEXT: v_add3_u32 v9, v10, v11, v9 -; GFX9-NEXT: v_mul_lo_u32 v11, v5, v9 -; GFX9-NEXT: v_mul_hi_u32 v13, v5, v12 -; GFX9-NEXT: v_mul_hi_u32 v10, v5, v9 -; GFX9-NEXT: v_mul_hi_u32 v16, v6, v9 -; GFX9-NEXT: v_mul_lo_u32 v9, v6, v9 -; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v13, v11 -; GFX9-NEXT: v_mul_lo_u32 v13, v6, v12 -; GFX9-NEXT: v_mul_hi_u32 v12, v6, v12 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v10, vcc -; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v13 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v10, v12, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v16, v14, vcc -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v10, v9 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v11, vcc -; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v9 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v10, vcc -; GFX9-NEXT: v_mul_lo_u32 v9, v7, v6 -; GFX9-NEXT: v_mul_lo_u32 v8, v8, v5 -; GFX9-NEXT: v_mul_hi_u32 v10, v7, v5 -; GFX9-NEXT: v_mul_lo_u32 v7, v7, v5 -; GFX9-NEXT: v_add3_u32 v8, v10, v9, v8 -; GFX9-NEXT: v_mul_lo_u32 v11, v5, v8 -; GFX9-NEXT: v_mul_hi_u32 v12, v5, v7 -; GFX9-NEXT: v_mul_hi_u32 v13, v5, v8 -; GFX9-NEXT: v_mul_hi_u32 v10, v6, v7 -; GFX9-NEXT: v_mul_lo_u32 v7, v6, v7 -; GFX9-NEXT: v_mul_hi_u32 v9, v6, v8 -; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v12, v11 -; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, v15, v13, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v8 -; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v11, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v12, v10, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v14, vcc -; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v15, v9, vcc -; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v8, vcc +; GFX9-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 +; GFX9-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 +; GFX9-NEXT: v_trunc_f32_e32 v3, v3 +; GFX9-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 +; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v3 +; GFX9-NEXT: v_mul_lo_u32 v4, v8, v6 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v6, 0 +; GFX9-NEXT: v_mul_lo_u32 v5, v7, v12 +; GFX9-NEXT: v_mul_hi_u32 v13, v6, v2 +; GFX9-NEXT: v_add3_u32 v5, v3, v5, v4 +; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v13, v3 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v2, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v16, vcc, v15, v4, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v5, 0 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v13, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v16, v3, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v14, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v15, v3, vcc +; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v6, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, v12, v3, vcc +; GFX9-NEXT: v_mul_lo_u32 v4, v7, v12 +; GFX9-NEXT: v_mul_lo_u32 v5, v8, v13 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v13, 0 +; GFX9-NEXT: v_add3_u32 v5, v3, v4, v5 +; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v5, 0 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v13, v5, 0 +; GFX9-NEXT: v_mul_hi_u32 v16, v13, v2 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v2, 0 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v16, v5 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v15, v6, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v7 +; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v5, v8, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v14, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v3 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v15, v4, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v13, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v12, v3, vcc ; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v1 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v7 -; GFX9-NEXT: v_xor_b32_e32 v0, v0, v7 -; GFX9-NEXT: v_mul_lo_u32 v8, v0, v6 -; GFX9-NEXT: v_mul_hi_u32 v9, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v10, v0, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v7, vcc -; GFX9-NEXT: v_xor_b32_e32 v1, v1, v7 -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v9, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v15, v10, vcc -; GFX9-NEXT: v_mul_lo_u32 v10, v1, v5 -; GFX9-NEXT: v_mul_hi_u32 v5, v1, v5 -; GFX9-NEXT: v_mul_hi_u32 v11, v1, v6 -; GFX9-NEXT: v_mul_lo_u32 v6, v1, v6 -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v8, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v9, v5, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v11, v14, vcc -; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v15, v8, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v3, v5 -; GFX9-NEXT: v_mul_lo_u32 v9, v2, v6 -; GFX9-NEXT: v_mul_hi_u32 v10, v2, v5 -; GFX9-NEXT: v_mul_lo_u32 v11, v2, v5 -; GFX9-NEXT: v_add3_u32 v8, v10, v9, v8 -; GFX9-NEXT: v_sub_u32_e32 v9, v1, v8 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v11 -; GFX9-NEXT: v_subb_co_u32_e64 v9, s[4:5], v9, v3, vcc -; GFX9-NEXT: v_sub_co_u32_e64 v10, s[4:5], v0, v2 -; GFX9-NEXT: v_subbrev_co_u32_e64 v11, s[6:7], 0, v9, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v11, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[6:7] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v10, v2 +; GFX9-NEXT: v_xor_b32_e32 v5, v0, v7 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v7, vcc +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v3, 0 +; GFX9-NEXT: v_mul_hi_u32 v6, v5, v2 +; GFX9-NEXT: v_xor_b32_e32 v4, v4, v7 +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v15, v1, vcc +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0 +; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v8, v1, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v14, vcc +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v15, v1, vcc +; GFX9-NEXT: v_mul_lo_u32 v6, v10, v2 +; GFX9-NEXT: v_mul_lo_u32 v8, v11, v3 +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v2, 0 +; GFX9-NEXT: v_add3_u32 v1, v1, v8, v6 +; GFX9-NEXT: v_sub_u32_e32 v6, v4, v1 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v5, v0 +; GFX9-NEXT: v_subb_co_u32_e64 v6, s[4:5], v6, v10, vcc +; GFX9-NEXT: v_sub_co_u32_e64 v8, s[4:5], v0, v11 +; GFX9-NEXT: v_subbrev_co_u32_e64 v12, s[6:7], 0, v6, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v12, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[6:7] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v8, v11 ; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[6:7] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v11, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[6:7] -; GFX9-NEXT: v_add_co_u32_e64 v13, s[6:7], 2, v5 -; GFX9-NEXT: v_addc_co_u32_e64 v14, s[6:7], 0, v6, s[6:7] -; GFX9-NEXT: v_add_co_u32_e64 v15, s[6:7], 1, v5 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v8, vcc -; GFX9-NEXT: v_addc_co_u32_e64 v16, s[6:7], 0, v6, s[6:7] -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v12 -; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e64 v12, v16, v14, s[6:7] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v12, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v13, s[6:7] +; GFX9-NEXT: v_add_co_u32_e64 v13, s[6:7], 2, v2 +; GFX9-NEXT: v_addc_co_u32_e64 v14, s[6:7], 0, v3, s[6:7] +; GFX9-NEXT: v_add_co_u32_e64 v15, s[6:7], 1, v2 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v4, v1, vcc +; GFX9-NEXT: v_addc_co_u32_e64 v16, s[6:7], 0, v3, s[6:7] +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v10 +; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v5 +; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc +; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v11 +; GFX9-NEXT: v_cndmask_b32_e64 v5, v16, v14, s[6:7] ; GFX9-NEXT: v_cndmask_b32_e64 v14, 0, -1, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v8, v14, vcc -; GFX9-NEXT: v_subb_co_u32_e64 v3, s[4:5], v9, v3, s[4:5] -; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8 -; GFX9-NEXT: v_cndmask_b32_e64 v8, v15, v13, s[6:7] -; GFX9-NEXT: v_sub_co_u32_e64 v2, s[4:5], v10, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc -; GFX9-NEXT: v_xor_b32_e32 v8, v7, v4 -; GFX9-NEXT: v_subbrev_co_u32_e64 v3, s[4:5], 0, v3, s[4:5] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v12, vcc -; GFX9-NEXT: v_xor_b32_e32 v4, v5, v8 -; GFX9-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v10 +; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX9-NEXT: v_cndmask_b32_e64 v4, v15, v13, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GFX9-NEXT: v_xor_b32_e32 v5, v7, v9 +; GFX9-NEXT: v_xor_b32_e32 v2, v2, v5 +; GFX9-NEXT: v_xor_b32_e32 v3, v3, v5 +; GFX9-NEXT: v_sub_co_u32_e64 v4, s[8:9], v2, v5 +; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v6, v10, s[4:5] +; GFX9-NEXT: v_subb_co_u32_e64 v5, s[8:9], v3, v5, s[8:9] +; GFX9-NEXT: v_sub_co_u32_e64 v3, s[4:5], v8, v11 +; GFX9-NEXT: v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5] +; GFX9-NEXT: v_cndmask_b32_e64 v2, v12, v2, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, v3, s[6:7] ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GFX9-NEXT: v_xor_b32_e32 v6, v6, v8 -; GFX9-NEXT: v_sub_co_u32_e64 v4, s[8:9], v4, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GFX9-NEXT: v_xor_b32_e32 v0, v0, v7 -; GFX9-NEXT: v_subb_co_u32_e64 v5, s[8:9], v6, v8, s[8:9] ; GFX9-NEXT: v_xor_b32_e32 v1, v1, v7 ; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v0, v7 ; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v1, v7, vcc @@ -957,113 +897,101 @@ ; GFX9-NEXT: ; %bb.1: ; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v3 -; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, 0, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, 0, v3, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v10, vcc, 0, v2 +; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v3, vcc ; GFX9-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 ; GFX9-NEXT: v_rcp_f32_e32 v4, v4 +; GFX9-NEXT: v_mov_b32_e32 v14, 0 ; GFX9-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 ; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 ; GFX9-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v9, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v10, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v11, v6, v4 -; GFX9-NEXT: v_add3_u32 v8, v10, v8, v9 -; GFX9-NEXT: v_mul_hi_u32 v9, v4, v11 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v14, v4, v8 -; GFX9-NEXT: v_mul_hi_u32 v15, v5, v8 -; GFX9-NEXT: v_mul_lo_u32 v8, v5, v8 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v13, v14, vcc -; GFX9-NEXT: v_mul_lo_u32 v14, v5, v11 -; GFX9-NEXT: v_mul_hi_u32 v11, v5, v11 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v14 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v10, v11, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v15, v12, vcc -; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v9, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v13, v10, vcc -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v9, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v7, v7, v4 -; GFX9-NEXT: v_mul_hi_u32 v9, v6, v4 -; GFX9-NEXT: v_mul_lo_u32 v6, v6, v4 -; GFX9-NEXT: v_add3_u32 v7, v9, v8, v7 -; GFX9-NEXT: v_mul_lo_u32 v10, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v11, v4, v6 -; GFX9-NEXT: v_mul_hi_u32 v14, v4, v7 -; GFX9-NEXT: v_mul_hi_u32 v9, v5, v6 -; GFX9-NEXT: v_mul_lo_u32 v6, v5, v6 -; GFX9-NEXT: v_mul_hi_u32 v8, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v11, v10 -; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, v13, v14, vcc -; GFX9-NEXT: v_mul_lo_u32 v7, v5, v7 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v10, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v11, v9, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v8, v12, vcc -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v13, v8, vcc +; GFX9-NEXT: v_cvt_u32_f32_e32 v8, v5 +; GFX9-NEXT: v_cvt_u32_f32_e32 v9, v4 +; GFX9-NEXT: v_mul_lo_u32 v6, v10, v8 +; GFX9-NEXT: v_mul_lo_u32 v7, v11, v9 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0 +; GFX9-NEXT: v_add3_u32 v7, v5, v6, v7 +; GFX9-NEXT: v_mul_hi_u32 v12, v9, v4 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v5 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0 +; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v14, v6, vcc +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v12, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v15, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v13, vcc ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v7, vcc -; GFX9-NEXT: v_mul_lo_u32 v6, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v7, v0, v4 -; GFX9-NEXT: v_mul_hi_u32 v8, v0, v5 -; GFX9-NEXT: v_mul_hi_u32 v9, v1, v5 -; GFX9-NEXT: v_mul_lo_u32 v5, v1, v5 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v7, v6 -; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v13, v8, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v1, v4 -; GFX9-NEXT: v_mul_hi_u32 v4, v1, v4 -; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v7, v4, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v9, v12, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v5, vcc +; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v9, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v8, v5, vcc +; GFX9-NEXT: v_mul_lo_u32 v6, v10, v15 +; GFX9-NEXT: v_mul_lo_u32 v7, v11, v12 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0 +; GFX9-NEXT: v_add3_u32 v7, v5, v6, v7 +; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v15, v7, 0 +; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v7, 0 +; GFX9-NEXT: v_mul_hi_u32 v11, v12, v4 +; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v4, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v11, v7 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v14, v8, vcc +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v9 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v7, v10, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v13, vcc ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v13, v6, vcc -; GFX9-NEXT: v_mul_lo_u32 v6, v3, v4 -; GFX9-NEXT: v_mul_lo_u32 v7, v2, v5 -; GFX9-NEXT: v_mul_hi_u32 v8, v2, v4 -; GFX9-NEXT: v_mul_lo_u32 v9, v2, v4 -; GFX9-NEXT: v_add3_u32 v6, v8, v7, v6 -; GFX9-NEXT: v_sub_u32_e32 v7, v1, v6 -; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v9 -; GFX9-NEXT: v_subb_co_u32_e64 v7, s[4:5], v7, v3, vcc -; GFX9-NEXT: v_sub_co_u32_e64 v8, s[4:5], v0, v2 -; GFX9-NEXT: v_subbrev_co_u32_e64 v9, s[6:7], 0, v7, s[4:5] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v9, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[6:7] -; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v8, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v14, v6, vcc +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v12, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v15, v5, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0 +; GFX9-NEXT: v_mul_hi_u32 v8, v0, v6 +; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v8, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v14, v5, vcc +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0 +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v8, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v9, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v13, vcc +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v4, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v14, v5, vcc +; GFX9-NEXT: v_mul_lo_u32 v8, v3, v6 +; GFX9-NEXT: v_mul_lo_u32 v9, v2, v7 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0 +; GFX9-NEXT: v_add3_u32 v5, v5, v9, v8 +; GFX9-NEXT: v_sub_u32_e32 v8, v1, v5 +; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4 +; GFX9-NEXT: v_subb_co_u32_e64 v8, s[4:5], v8, v3, vcc +; GFX9-NEXT: v_sub_co_u32_e64 v9, s[4:5], v0, v2 +; GFX9-NEXT: v_subbrev_co_u32_e64 v10, s[6:7], 0, v8, s[4:5] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v10, v3 +; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[6:7] +; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v9, v2 ; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[6:7] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v9, v3 -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[6:7] -; GFX9-NEXT: v_add_co_u32_e64 v11, s[6:7], 2, v4 -; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, v5, s[6:7] -; GFX9-NEXT: v_add_co_u32_e64 v13, s[6:7], 1, v4 -; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9-NEXT: v_addc_co_u32_e64 v14, s[6:7], 0, v5, s[6:7] +; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v10, v3 +; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v11, s[6:7] +; GFX9-NEXT: v_add_co_u32_e64 v11, s[6:7], 2, v6 +; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, v7, s[6:7] +; GFX9-NEXT: v_add_co_u32_e64 v13, s[6:7], 1, v6 +; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc +; GFX9-NEXT: v_addc_co_u32_e64 v14, s[6:7], 0, v7, s[6:7] ; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v10 -; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc +; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v4 +; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc ; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GFX9-NEXT: v_cndmask_b32_e64 v10, v14, v12, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v4, v14, v12, s[6:7] ; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GFX9-NEXT: v_subb_co_u32_e64 v3, s[4:5], v7, v3, s[4:5] -; GFX9-NEXT: v_sub_co_u32_e64 v2, s[4:5], v8, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v12, vcc +; GFX9-NEXT: v_subb_co_u32_e64 v3, s[4:5], v8, v3, s[4:5] +; GFX9-NEXT: v_sub_co_u32_e64 v2, s[4:5], v9, v2 +; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc ; GFX9-NEXT: v_subbrev_co_u32_e64 v3, s[4:5], 0, v3, s[4:5] -; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e64 v6, v13, v11, s[6:7] +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GFX9-NEXT: v_cndmask_b32_e64 v3, v10, v3, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v4, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v4, v13, v11, s[6:7] ; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v1, v8, v2, s[6:7] -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v2, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc ; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3 ; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll --- a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll @@ -7,22 +7,24 @@ ; SI-LABEL: umulo_i64_v_v: ; SI: ; %bb.0: ; %bb ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_hi_u32 v4, v1, v2 -; SI-NEXT: v_mul_lo_u32 v5, v1, v2 -; SI-NEXT: v_mul_hi_u32 v6, v0, v3 -; SI-NEXT: v_mul_lo_u32 v7, v0, v3 -; SI-NEXT: v_mul_hi_u32 v8, v0, v2 -; SI-NEXT: v_mul_hi_u32 v9, v1, v3 -; SI-NEXT: v_mul_lo_u32 v3, v1, v3 -; SI-NEXT: v_mul_lo_u32 v0, v0, v2 -; SI-NEXT: v_add_i32_e32 v1, vcc, v8, v7 -; SI-NEXT: v_addc_u32_e32 v2, vcc, 0, v6, vcc -; SI-NEXT: v_add_i32_e32 v6, vcc, v1, v5 -; SI-NEXT: v_add_i32_e64 v1, s[4:5], v1, v5 -; SI-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc -; SI-NEXT: v_addc_u32_e32 v4, vcc, 0, v9, vcc -; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc +; SI-NEXT: v_mov_b32_e32 v4, v1 +; SI-NEXT: v_mov_b32_e32 v5, v0 +; SI-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v4, v2, 0 +; SI-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v3, 0 +; SI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0 +; SI-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v4, v3, 0 +; SI-NEXT: v_mul_lo_u32 v3, v5, v3 +; SI-NEXT: v_mul_lo_u32 v4, v4, v2 +; SI-NEXT: v_mov_b32_e32 v2, v1 +; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v8 +; SI-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc +; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v6 +; SI-NEXT: v_add_i32_e64 v1, s[4:5], v1, v4 +; SI-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc +; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v11, vcc +; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v10 +; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; SI-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; SI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; SI-NEXT: s_setpc_b64 s[30:31] @@ -30,23 +32,24 @@ ; GFX9-LABEL: umulo_i64_v_v: ; GFX9: ; %bb.0: ; %bb ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mul_lo_u32 v5, v0, v3 -; GFX9-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX9-NEXT: v_mul_hi_u32 v8, v0, v3 -; GFX9-NEXT: v_mul_lo_u32 v7, v1, v2 -; GFX9-NEXT: v_mul_hi_u32 v4, v1, v2 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v6, v5 -; GFX9-NEXT: v_mul_hi_u32 v10, v1, v3 -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, 0, v8, vcc -; GFX9-NEXT: v_mul_lo_u32 v1, v1, v3 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v8, v4, vcc -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, 0, v10, vcc -; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v4, v1 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v8, vcc -; GFX9-NEXT: v_mul_lo_u32 v0, v0, v2 -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[3:4] -; GFX9-NEXT: v_add3_u32 v1, v6, v5, v7 +; GFX9-NEXT: v_mov_b32_e32 v5, v0 +; GFX9-NEXT: v_mov_b32_e32 v4, v1 +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0 +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0 +; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0 +; GFX9-NEXT: v_mov_b32_e32 v10, v1 +; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v10, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, 0, v7, vcc +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v4, v3, 0 +; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v10, v8 +; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v11, v9, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc +; GFX9-NEXT: v_mul_lo_u32 v4, v4, v2 +; GFX9-NEXT: v_mul_lo_u32 v5, v5, v3 +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v8, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v7, vcc +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] +; GFX9-NEXT: v_add3_u32 v1, v1, v5, v4 ; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -54,23 +57,24 @@ ; GFX10: ; %bb.0: ; %bb ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: v_mul_lo_u32 v5, v0, v3 -; GFX10-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX10-NEXT: v_mul_hi_u32 v4, v0, v3 -; GFX10-NEXT: v_mul_lo_u32 v8, v1, v2 -; GFX10-NEXT: v_mul_hi_u32 v7, v1, v2 -; GFX10-NEXT: v_mul_hi_u32 v9, v1, v3 -; GFX10-NEXT: v_mul_lo_u32 v1, v1, v3 -; GFX10-NEXT: v_mul_lo_u32 v0, v0, v2 -; GFX10-NEXT: v_add_co_u32 v10, vcc_lo, v6, v5 -; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v4, vcc_lo -; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, v10, v8 -; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v4, v7, vcc_lo -; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v9, vcc_lo -; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, v3, v1 -; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v4, vcc_lo -; GFX10-NEXT: v_add3_u32 v1, v6, v5, v8 -; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4] +; GFX10-NEXT: v_mov_b32_e32 v4, v0 +; GFX10-NEXT: v_mov_b32_e32 v5, v1 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s4, v4, v2, 0 +; GFX10-NEXT: v_mad_u64_u32 v[6:7], s4, v4, v3, 0 +; GFX10-NEXT: v_mad_u64_u32 v[9:10], s4, v5, v2, 0 +; GFX10-NEXT: v_mad_u64_u32 v[11:12], s4, v5, v3, 0 +; GFX10-NEXT: v_mov_b32_e32 v8, v1 +; GFX10-NEXT: v_mul_lo_u32 v5, v5, v2 +; GFX10-NEXT: v_mul_lo_u32 v4, v4, v3 +; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v8, v6 +; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; GFX10-NEXT: v_add3_u32 v1, v1, v4, v5 +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v6, v9 +; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v7, v10, vcc_lo +; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, 0, v12, vcc_lo +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v11 +; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v6, vcc_lo +; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] ; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] bb: @@ -82,74 +86,75 @@ ; SI-LABEL: smulo_i64_s_s: ; SI: ; %bb.0: ; %bb ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_hi_u32 v6, v1, v2 -; SI-NEXT: v_mul_lo_u32 v5, v1, v2 -; SI-NEXT: v_mul_hi_u32 v7, v0, v3 -; SI-NEXT: v_mul_lo_u32 v8, v0, v3 -; SI-NEXT: v_mul_hi_u32 v9, v0, v2 -; SI-NEXT: v_mul_hi_i32 v10, v1, v3 -; SI-NEXT: v_mul_lo_u32 v11, v1, v3 -; SI-NEXT: v_mov_b32_e32 v12, 0 -; SI-NEXT: v_mul_lo_u32 v4, v0, v2 -; SI-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc -; SI-NEXT: v_add_i32_e32 v9, vcc, v8, v5 -; SI-NEXT: v_add_i32_e64 v5, s[4:5], v8, v5 -; SI-NEXT: v_addc_u32_e32 v8, vcc, v7, v6, vcc -; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v5 -; SI-NEXT: v_addc_u32_e32 v9, vcc, 0, v10, vcc +; SI-NEXT: v_mov_b32_e32 v4, v1 +; SI-NEXT: v_mov_b32_e32 v5, v0 +; SI-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v4, v2, 0 +; SI-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v3, 0 +; SI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0 +; SI-NEXT: v_mad_i64_i32 v[10:11], s[4:5], v4, v3, 0 +; SI-NEXT: v_mov_b32_e32 v13, 0 +; SI-NEXT: v_mul_lo_u32 v14, v5, v3 +; SI-NEXT: v_mul_lo_u32 v15, v4, v2 +; SI-NEXT: v_mov_b32_e32 v12, v1 +; SI-NEXT: v_add_i32_e32 v8, vcc, v12, v8 +; SI-NEXT: v_addc_u32_e32 v9, vcc, 0, v9, vcc +; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v14 +; SI-NEXT: v_add_i32_e32 v6, vcc, v8, v6 +; SI-NEXT: v_add_i32_e64 v1, s[4:5], v1, v15 +; SI-NEXT: v_addc_u32_e32 v8, vcc, v9, v7, vcc +; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v1 +; SI-NEXT: v_addc_u32_e32 v9, vcc, 0, v11, vcc ; SI-NEXT: v_mov_b32_e32 v7, v6 -; SI-NEXT: v_add_i32_e32 v8, vcc, v8, v11 -; SI-NEXT: v_addc_u32_e32 v9, vcc, v12, v9, vcc +; SI-NEXT: v_add_i32_e32 v8, vcc, v8, v10 +; SI-NEXT: v_addc_u32_e32 v9, vcc, v13, v9, vcc ; SI-NEXT: v_sub_i32_e32 v2, vcc, v8, v2 -; SI-NEXT: v_subb_u32_e32 v10, vcc, v9, v12, vcc -; SI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1 -; SI-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc +; SI-NEXT: v_subb_u32_e32 v10, vcc, v9, v13, vcc +; SI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v4 +; SI-NEXT: v_cndmask_b32_e32 v4, v9, v10, vcc ; SI-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; SI-NEXT: v_sub_i32_e32 v0, vcc, v2, v0 -; SI-NEXT: v_subb_u32_e32 v8, vcc, v1, v12, vcc +; SI-NEXT: v_sub_i32_e32 v5, vcc, v2, v5 +; SI-NEXT: v_subb_u32_e32 v8, vcc, v4, v13, vcc ; SI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3 -; SI-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc -; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc -; SI-NEXT: v_cmp_ne_u64_e32 vcc, v[0:1], v[6:7] +; SI-NEXT: v_cndmask_b32_e32 v3, v4, v8, vcc +; SI-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc +; SI-NEXT: v_cmp_ne_u64_e32 vcc, v[2:3], v[6:7] ; SI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; SI-NEXT: v_mov_b32_e32 v0, v4 -; SI-NEXT: v_mov_b32_e32 v1, v5 ; SI-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: smulo_i64_s_s: ; GFX9: ; %bb.0: ; %bb ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mul_lo_u32 v5, v0, v3 -; GFX9-NEXT: v_mul_hi_u32 v6, v0, v2 -; GFX9-NEXT: v_mul_hi_u32 v8, v0, v3 -; GFX9-NEXT: v_mul_lo_u32 v7, v1, v2 -; GFX9-NEXT: v_mul_hi_u32 v4, v1, v2 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v6, v5 -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, 0, v8, vcc -; GFX9-NEXT: v_mul_hi_i32 v10, v1, v3 -; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v7 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v8, v4, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, v1, v3 -; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v10, vcc -; GFX9-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8 -; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v10, v9, vcc -; GFX9-NEXT: v_sub_co_u32_e32 v9, vcc, v4, v2 -; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v8, v10, vcc -; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v11, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v9, vcc -; GFX9-NEXT: v_sub_co_u32_e32 v9, vcc, v8, v0 -; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v10, vcc +; GFX9-NEXT: v_mov_b32_e32 v5, v0 +; GFX9-NEXT: v_mov_b32_e32 v4, v1 +; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0 +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0 +; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0 +; GFX9-NEXT: v_mov_b32_e32 v10, v1 +; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v10, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, 0, v7, vcc +; GFX9-NEXT: v_mad_i64_i32 v[6:7], s[4:5], v4, v3, 0 +; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v10, v8 +; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v11, v9, vcc +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc +; GFX9-NEXT: v_mov_b32_e32 v9, 0 +; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v9, v7, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v6, v2 +; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v7, v9, vcc +; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v4 +; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc +; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v6, v5 +; GFX9-NEXT: v_mul_lo_u32 v4, v4, v2 +; GFX9-NEXT: v_mul_lo_u32 v5, v5, v3 +; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v7, v9, vcc ; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, 0, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc -; GFX9-NEXT: v_add3_u32 v1, v6, v5, v7 -; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc -; GFX9-NEXT: v_mul_lo_u32 v0, v0, v2 -; GFX9-NEXT: v_mov_b32_e32 v6, v5 -; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, v[3:4], v[5:6] +; GFX9-NEXT: v_add3_u32 v1, v1, v5, v4 +; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v1 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc +; GFX9-NEXT: v_mov_b32_e32 v5, v4 +; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, v[2:3], v[4:5] ; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -157,35 +162,36 @@ ; GFX10: ; %bb.0: ; %bb ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: v_mul_lo_u32 v4, v0, v3 -; GFX10-NEXT: v_mul_hi_u32 v5, v0, v2 -; GFX10-NEXT: v_mul_hi_u32 v6, v0, v3 -; GFX10-NEXT: v_mul_lo_u32 v8, v1, v2 -; GFX10-NEXT: v_mul_hi_u32 v7, v1, v2 -; GFX10-NEXT: v_mul_hi_i32 v9, v1, v3 -; GFX10-NEXT: v_mul_lo_u32 v11, v1, v3 -; GFX10-NEXT: v_add_co_u32 v10, vcc_lo, v5, v4 -; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, 0, v6, vcc_lo -; GFX10-NEXT: v_add_co_u32 v10, vcc_lo, v10, v8 -; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v6, v7, vcc_lo -; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v9, vcc_lo +; GFX10-NEXT: v_mov_b32_e32 v4, v0 +; GFX10-NEXT: v_mov_b32_e32 v5, v1 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s4, v4, v2, 0 +; GFX10-NEXT: v_mad_u64_u32 v[6:7], s4, v4, v3, 0 +; GFX10-NEXT: v_mad_u64_u32 v[9:10], s4, v5, v2, 0 +; GFX10-NEXT: v_mad_i64_i32 v[11:12], s4, v5, v3, 0 +; GFX10-NEXT: v_mov_b32_e32 v8, v1 +; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v8, v6 +; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; GFX10-NEXT: v_mul_lo_u32 v8, v5, v2 +; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v6, v9 +; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v7, v10, vcc_lo +; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v12, vcc_lo +; GFX10-NEXT: v_mul_lo_u32 v9, v4, v3 ; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v6, v11 ; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v9, vcc_lo, v6, v2 +; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, v6, v2 ; GFX10-NEXT: v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v7, vcc_lo -; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v1 -; GFX10-NEXT: v_add3_u32 v1, v5, v4, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo -; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v1 -; GFX10-NEXT: v_sub_co_u32 v8, vcc_lo, v6, v0 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v9, vcc_lo, 0, v7, vcc_lo +; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v5 +; GFX10-NEXT: v_add3_u32 v1, v1, v9, v8 +; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v10, vcc_lo +; GFX10-NEXT: v_ashrrev_i32_e32 v2, 31, v1 +; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v6, v4 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo ; GFX10-NEXT: v_cmp_gt_i32_e32 vcc_lo, 0, v3 -; GFX10-NEXT: v_mov_b32_e32 v5, v4 -; GFX10-NEXT: v_mul_lo_u32 v0, v0, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo -; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[4:5] +; GFX10-NEXT: v_mov_b32_e32 v3, v2 +; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo +; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3] ; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] bb: @@ -199,29 +205,27 @@ ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s7, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s2 -; SI-NEXT: v_mul_hi_u32 v1, s1, v0 -; SI-NEXT: s_mul_i32 s4, s1, s2 -; SI-NEXT: v_mov_b32_e32 v2, s3 -; SI-NEXT: v_mul_hi_u32 v3, s0, v2 -; SI-NEXT: s_mul_i32 s5, s0, s3 -; SI-NEXT: v_mul_hi_u32 v0, s0, v0 -; SI-NEXT: v_mul_hi_u32 v2, s1, v2 -; SI-NEXT: s_mul_i32 s1, s1, s3 -; SI-NEXT: s_mul_i32 s0, s0, s2 -; SI-NEXT: v_add_i32_e32 v4, vcc, s5, v0 +; SI-NEXT: v_mov_b32_e32 v4, s2 +; SI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s1, v4, 0 +; SI-NEXT: v_mov_b32_e32 v6, s3 +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s0, v6, 0 +; SI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s0, v4, 0 +; SI-NEXT: v_mad_u64_u32 v[6:7], s[4:5], s1, v6, 0 +; SI-NEXT: s_mul_i32 s0, s0, s3 +; SI-NEXT: s_mul_i32 s1, s1, s2 +; SI-NEXT: v_mov_b32_e32 v8, v5 +; SI-NEXT: v_add_i32_e32 v2, vcc, v8, v2 ; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc -; SI-NEXT: v_mov_b32_e32 v5, s0 -; SI-NEXT: v_add_i32_e32 v4, vcc, s4, v4 -; SI-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc -; SI-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc -; SI-NEXT: v_add_i32_e32 v3, vcc, s5, v0 -; SI-NEXT: v_add_i32_e32 v0, vcc, s1, v1 -; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc -; SI-NEXT: v_add_i32_e32 v2, vcc, s4, v3 +; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; SI-NEXT: v_addc_u32_e32 v0, vcc, v3, v1, vcc +; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc +; SI-NEXT: v_add_i32_e32 v2, vcc, s0, v5 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v6 +; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; SI-NEXT: v_add_i32_e32 v2, vcc, s1, v2 ; SI-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] ; SI-NEXT: v_cndmask_b32_e64 v1, v2, 0, vcc -; SI-NEXT: v_cndmask_b32_e64 v0, v5, 0, vcc +; SI-NEXT: v_cndmask_b32_e64 v0, v4, 0, vcc ; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; SI-NEXT: s_endpgm @@ -296,46 +300,44 @@ ; SI-LABEL: smulo_i64_s: ; SI: ; %bb.0: ; %bb ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; SI-NEXT: v_mov_b32_e32 v0, 0 +; SI-NEXT: v_mov_b32_e32 v9, 0 ; SI-NEXT: s_mov_b32 s7, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v1, s2 -; SI-NEXT: v_mul_hi_u32 v2, s1, v1 -; SI-NEXT: s_mul_i32 s4, s1, s2 -; SI-NEXT: v_mov_b32_e32 v3, s3 -; SI-NEXT: v_mul_hi_u32 v4, s0, v3 -; SI-NEXT: s_mul_i32 s5, s0, s3 -; SI-NEXT: v_mul_hi_u32 v1, s0, v1 -; SI-NEXT: v_mul_hi_i32 v3, s1, v3 -; SI-NEXT: s_mul_i32 s6, s1, s3 +; SI-NEXT: v_mov_b32_e32 v4, s2 +; SI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s1, v4, 0 +; SI-NEXT: v_mov_b32_e32 v6, s3 +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s0, v6, 0 +; SI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s0, v4, 0 +; SI-NEXT: v_mad_i64_i32 v[6:7], s[4:5], s1, v6, 0 ; SI-NEXT: s_cmp_lt_i32 s1, 0 -; SI-NEXT: s_mul_i32 s1, s0, s2 -; SI-NEXT: v_add_i32_e32 v5, vcc, s5, v1 -; SI-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc -; SI-NEXT: v_mov_b32_e32 v6, s1 -; SI-NEXT: v_add_i32_e32 v5, vcc, s4, v5 -; SI-NEXT: v_addc_u32_e32 v2, vcc, v4, v2, vcc +; SI-NEXT: s_mul_i32 s4, s0, s3 +; SI-NEXT: s_mul_i32 s1, s1, s2 +; SI-NEXT: v_mov_b32_e32 v8, v5 +; SI-NEXT: v_add_i32_e32 v2, vcc, v8, v2 ; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc -; SI-NEXT: v_add_i32_e32 v1, vcc, s5, v1 -; SI-NEXT: v_add_i32_e32 v2, vcc, s6, v2 -; SI-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc -; SI-NEXT: v_add_i32_e32 v4, vcc, s4, v1 -; SI-NEXT: v_subrev_i32_e32 v1, vcc, s2, v2 -; SI-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v3, vcc +; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; SI-NEXT: v_addc_u32_e32 v0, vcc, v3, v1, vcc +; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc +; SI-NEXT: v_add_i32_e32 v2, vcc, s4, v5 +; SI-NEXT: v_add_i32_e32 v3, vcc, v0, v6 +; SI-NEXT: v_addc_u32_e32 v1, vcc, v9, v1, vcc +; SI-NEXT: v_add_i32_e32 v5, vcc, s1, v2 +; SI-NEXT: v_subrev_i32_e32 v2, vcc, s2, v3 +; SI-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v1, vcc ; SI-NEXT: s_cselect_b64 vcc, -1, 0 ; SI-NEXT: s_cmp_lt_i32 s3, 0 -; SI-NEXT: v_ashrrev_i32_e32 v0, 31, v4 -; SI-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; SI-NEXT: v_cndmask_b32_e32 v2, v2, v1, vcc +; SI-NEXT: v_ashrrev_i32_e32 v0, 31, v5 +; SI-NEXT: v_cndmask_b32_e32 v6, v1, v6, vcc +; SI-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc ; SI-NEXT: v_mov_b32_e32 v1, v0 -; SI-NEXT: v_subrev_i32_e32 v5, vcc, s0, v2 -; SI-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v3, vcc +; SI-NEXT: v_subrev_i32_e32 v7, vcc, s0, v2 +; SI-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v6, vcc ; SI-NEXT: s_cselect_b64 vcc, -1, 0 -; SI-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; SI-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc +; SI-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc +; SI-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc ; SI-NEXT: v_cmp_ne_u64_e32 vcc, v[2:3], v[0:1] -; SI-NEXT: v_cndmask_b32_e64 v1, v4, 0, vcc -; SI-NEXT: v_cndmask_b32_e64 v0, v6, 0, vcc +; SI-NEXT: v_cndmask_b32_e64 v1, v5, 0, vcc +; SI-NEXT: v_cndmask_b32_e64 v0, v4, 0, vcc ; SI-NEXT: s_mov_b32 s6, -1 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; SI-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll --- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll +++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll @@ -12,9 +12,8 @@ ; SI-LABEL: mad_i64_i32_sextops: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v4, v0, v1 -; SI-NEXT: v_mul_hi_i32 v1, v0, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, v4, v2 +; SI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i64 @@ -34,9 +33,8 @@ ; SI-LABEL: mad_i64_i32_sextops_commute: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v4, v0, v1 -; SI-NEXT: v_mul_hi_i32 v1, v0, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v4 +; SI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; SI-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i64 @@ -56,9 +54,8 @@ ; SI-LABEL: mad_u64_u32_zextops: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v4, v0, v1 -; SI-NEXT: v_mul_hi_u32 v1, v0, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, v4, v2 +; SI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %sext0 = zext i32 %arg0 to i64 @@ -78,9 +75,8 @@ ; SI-LABEL: mad_u64_u32_zextops_commute: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v4, v0, v1 -; SI-NEXT: v_mul_hi_u32 v1, v0, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v4 +; SI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; SI-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %sext0 = zext i32 %arg0 to i64 @@ -94,61 +90,56 @@ ; CI-LABEL: mad_i64_i32_sextops_i32_i128: ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CI-NEXT: v_mul_hi_u32 v6, v0, v1 -; CI-NEXT: v_ashrrev_i32_e32 v12, 31, v0 -; CI-NEXT: v_mov_b32_e32 v7, 0 -; CI-NEXT: v_ashrrev_i32_e32 v13, 31, v1 -; CI-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v12, v1, v[6:7] -; CI-NEXT: v_mul_hi_i32 v11, v1, v12 -; CI-NEXT: v_mul_lo_u32 v10, v1, v12 -; CI-NEXT: v_mov_b32_e32 v6, v9 -; CI-NEXT: v_mov_b32_e32 v9, v7 -; CI-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v0, v13, v[8:9] -; CI-NEXT: v_mad_i64_i32 v[10:11], s[4:5], v13, v0, v[10:11] -; CI-NEXT: v_add_i32_e32 v8, vcc, v6, v8 -; CI-NEXT: v_addc_u32_e64 v9, s[4:5], 0, 0, vcc -; CI-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v12, v13, v[8:9] -; CI-NEXT: v_mul_lo_u32 v0, v0, v1 -; CI-NEXT: v_mov_b32_e32 v1, v7 -; CI-NEXT: v_add_i32_e32 v6, vcc, v8, v10 -; CI-NEXT: v_addc_u32_e32 v8, vcc, v9, v11, vcc -; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; CI-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0 +; CI-NEXT: v_ashrrev_i32_e32 v13, 31, v0 +; CI-NEXT: v_mov_b32_e32 v8, 0 +; CI-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v1, v[7:8] +; CI-NEXT: v_ashrrev_i32_e32 v14, 31, v1 +; CI-NEXT: v_mad_i64_i32 v[11:12], s[4:5], v1, v13, 0 +; CI-NEXT: v_mov_b32_e32 v7, v10 +; CI-NEXT: v_mov_b32_e32 v10, v8 +; CI-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[9:10] +; CI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[11:12] +; CI-NEXT: v_add_i32_e32 v9, vcc, v7, v9 +; CI-NEXT: v_addc_u32_e64 v10, s[4:5], 0, 0, vcc +; CI-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v14, v[9:10] +; CI-NEXT: v_add_i32_e32 v7, vcc, v9, v0 +; CI-NEXT: v_addc_u32_e32 v9, vcc, v10, v1, vcc +; CI-NEXT: v_mov_b32_e32 v1, v8 +; CI-NEXT: v_add_i32_e32 v0, vcc, v6, v2 ; CI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; CI-NEXT: v_addc_u32_e32 v2, vcc, v6, v4, vcc -; CI-NEXT: v_addc_u32_e32 v3, vcc, v8, v5, vcc +; CI-NEXT: v_addc_u32_e32 v2, vcc, v7, v4, vcc +; CI-NEXT: v_addc_u32_e32 v3, vcc, v9, v5, vcc ; CI-NEXT: s_setpc_b64 s[30:31] ; ; SI-LABEL: mad_i64_i32_sextops_i32_i128: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_ashrrev_i32_e32 v6, 31, v0 -; SI-NEXT: v_mul_lo_u32 v11, v6, v1 -; SI-NEXT: v_mul_hi_u32 v12, v0, v1 -; SI-NEXT: v_ashrrev_i32_e32 v7, 31, v1 -; SI-NEXT: v_mul_hi_u32 v14, v6, v1 -; SI-NEXT: v_mul_lo_u32 v13, v0, v7 -; SI-NEXT: v_mul_hi_u32 v10, v0, v7 -; SI-NEXT: v_add_i32_e32 v12, vcc, v11, v12 -; SI-NEXT: v_addc_u32_e32 v14, vcc, 0, v14, vcc -; SI-NEXT: v_mul_hi_u32 v8, v6, v7 -; SI-NEXT: v_add_i32_e32 v12, vcc, v13, v12 -; SI-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc -; SI-NEXT: v_mul_i32_i24_e32 v9, v6, v7 -; SI-NEXT: v_add_i32_e32 v10, vcc, v14, v10 -; SI-NEXT: v_mul_hi_i32 v6, v1, v6 -; SI-NEXT: v_mul_hi_i32 v7, v7, v0 -; SI-NEXT: v_addc_u32_e64 v14, s[4:5], 0, 0, vcc -; SI-NEXT: v_add_i32_e32 v9, vcc, v9, v10 -; SI-NEXT: v_addc_u32_e32 v8, vcc, v8, v14, vcc -; SI-NEXT: v_add_i32_e32 v10, vcc, v13, v11 -; SI-NEXT: v_mul_lo_u32 v0, v0, v1 -; SI-NEXT: v_addc_u32_e32 v6, vcc, v7, v6, vcc -; SI-NEXT: v_add_i32_e32 v7, vcc, v9, v10 -; SI-NEXT: v_addc_u32_e32 v6, vcc, v8, v6, vcc -; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; SI-NEXT: v_addc_u32_e32 v1, vcc, v12, v3, vcc +; SI-NEXT: v_ashrrev_i32_e32 v12, 31, v0 +; SI-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0 +; SI-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v12, v1, 0 +; SI-NEXT: v_ashrrev_i32_e32 v13, 31, v1 +; SI-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v0, v13, 0 +; SI-NEXT: v_add_i32_e32 v7, vcc, v8, v7 +; SI-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc +; SI-NEXT: v_add_i32_e32 v9, vcc, v10, v7 +; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v11, vcc +; SI-NEXT: v_add_i32_e32 v10, vcc, v8, v7 +; SI-NEXT: v_mad_i64_i32 v[7:8], s[4:5], v1, v12, 0 +; SI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v13, v0, 0 +; SI-NEXT: v_mul_i32_i24_e32 v15, v12, v13 +; SI-NEXT: v_mul_hi_i32_i24_e32 v14, v12, v13 +; SI-NEXT: v_addc_u32_e64 v11, s[4:5], 0, 0, vcc +; SI-NEXT: v_add_i32_e32 v10, vcc, v15, v10 +; SI-NEXT: v_addc_u32_e32 v11, vcc, v14, v11, vcc +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v7 +; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc +; SI-NEXT: v_add_i32_e32 v7, vcc, v10, v0 +; SI-NEXT: v_addc_u32_e32 v8, vcc, v11, v1, vcc +; SI-NEXT: v_add_i32_e32 v0, vcc, v6, v2 +; SI-NEXT: v_addc_u32_e32 v1, vcc, v9, v3, vcc ; SI-NEXT: v_addc_u32_e32 v2, vcc, v7, v4, vcc -; SI-NEXT: v_addc_u32_e32 v3, vcc, v6, v5, vcc +; SI-NEXT: v_addc_u32_e32 v3, vcc, v8, v5, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i128 %sext1 = sext i32 %arg1 to i128 @@ -169,9 +160,8 @@ ; SI-LABEL: mad_i64_i32_sextops_i32_i63: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v4, v0, v1 -; SI-NEXT: v_mul_hi_i32 v1, v0, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, v4, v2 +; SI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i32 %arg0 to i63 @@ -199,10 +189,9 @@ ; SI-NEXT: v_lshlrev_b32_e32 v1, 1, v1 ; SI-NEXT: v_ashr_i64 v[4:5], v[3:4], 33 ; SI-NEXT: v_ashr_i64 v[0:1], v[0:1], 33 -; SI-NEXT: v_mul_lo_u32 v1, v4, v0 -; SI-NEXT: v_mul_hi_i32 v4, v4, v0 -; SI-NEXT: v_add_i32_e32 v0, vcc, v1, v2 -; SI-NEXT: v_addc_u32_e32 v1, vcc, v4, v3, vcc +; SI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v4, v0, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %sext0 = sext i31 %arg0 to i63 %sext1 = sext i31 %arg1 to i63 @@ -221,10 +210,9 @@ ; SI-LABEL: mad_u64_u32_bitops: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v1, v0, v2 -; SI-NEXT: v_mul_hi_u32 v2, v0, v2 -; SI-NEXT: v_add_i32_e32 v0, vcc, v1, v4 -; SI-NEXT: v_addc_u32_e32 v1, vcc, v2, v5, vcc +; SI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %trunc.lhs = and i64 %arg0, 4294967295 %trunc.rhs = and i64 %arg1, 4294967295 @@ -238,10 +226,9 @@ ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_and_b32_e32 v1, 1, v1 -; GCN-NEXT: v_mul_hi_u32 v3, v0, v2 -; GCN-NEXT: v_mul_lo_u32 v1, v1, v2 -; GCN-NEXT: v_mul_lo_u32 v0, v0, v2 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 +; GCN-NEXT: v_mul_lo_u32 v3, v1, v2 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 ; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] @@ -257,10 +244,9 @@ ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_and_b32_e32 v1, 1, v3 -; GCN-NEXT: v_mul_hi_u32 v3, v0, v2 -; GCN-NEXT: v_mul_lo_u32 v1, v0, v1 -; GCN-NEXT: v_mul_lo_u32 v0, v0, v2 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 +; GCN-NEXT: v_mul_lo_u32 v3, v0, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 ; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] @@ -281,10 +267,9 @@ ; SI-LABEL: mad_i64_i32_bitops: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v1, v0, v2 -; SI-NEXT: v_mul_hi_i32 v2, v0, v2 -; SI-NEXT: v_add_i32_e32 v0, vcc, v1, v4 -; SI-NEXT: v_addc_u32_e32 v1, vcc, v2, v5, vcc +; SI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], v0, v2, 0 +; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +; SI-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc ; SI-NEXT: s_setpc_b64 s[30:31] %shl.lhs = shl i64 %arg0, 32 %trunc.lhs = ashr i64 %shl.lhs, 32 @@ -306,8 +291,7 @@ ; SI-LABEL: mad_i64_i32_unpack_i64ops: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_mul_lo_u32 v2, v1, v0 -; SI-NEXT: v_mul_hi_u32 v3, v1, v0 +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v1, v0, 0 ; SI-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; SI-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc ; SI-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll --- a/llvm/test/CodeGen/AMDGPU/mul.ll +++ b/llvm/test/CodeGen/AMDGPU/mul.ll @@ -73,8 +73,7 @@ ; FUNC-LABEL: {{^}}mul64_sext_c: ; EG-DAG: MULLO_INT ; EG-DAG: MULHI_INT -; GCN-DAG: s_mul_i32 -; GCN-DAG: v_mul_hi_i32 +; GCN: v_mad_i64_i32 define amdgpu_kernel void @mul64_sext_c(i64 addrspace(1)* %out, i32 %in) { entry: %0 = sext i32 %in to i64 @@ -86,8 +85,7 @@ ; FUNC-LABEL: {{^}}v_mul64_sext_c: ; EG-DAG: MULLO_INT ; EG-DAG: MULHI_INT -; GCN-DAG: v_mul_lo_u32 -; GCN-DAG: v_mul_hi_i32 +; GCN: v_mad_i64_i32 ; GCN: s_endpgm define amdgpu_kernel void @v_mul64_sext_c(i64 addrspace(1)* %out, i32 addrspace(1)* %in) { %val = load i32, i32 addrspace(1)* %in, align 4 @@ -98,8 +96,7 @@ } ; FUNC-LABEL: {{^}}v_mul64_sext_inline_imm: -; GCN-DAG: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, 9 -; GCN-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9 +; GCN: v_mad_i64_i32 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, 9, 0 ; GCN: s_endpgm define amdgpu_kernel void @v_mul64_sext_inline_imm(i64 addrspace(1)* %out, i32 addrspace(1)* %in) { %val = load i32, i32 addrspace(1)* %in, align 4 @@ -184,8 +181,7 @@ } ; FUNC-LABEL: {{^}}mul64_in_branch: -; GCN-DAG: s_mul_i32 -; GCN-DAG: v_mul_hi_u32 +; GCN: v_mad_u64_u32 ; GCN: s_endpgm define amdgpu_kernel void @mul64_in_branch(i64 addrspace(1)* %out, i64 addrspace(1)* %in, i64 %a, i64 %b, i64 %c) { entry: @@ -211,34 +207,27 @@ ; GCN: s_load_dwordx4 ; GCN: s_load_dwordx4 -; SI: v_mul_hi_u32 -; SI: v_mul_hi_u32 -; SI: s_mul_i32 -; SI: v_mul_hi_u32 -; SI: s_mul_i32 -; SI: s_mul_i32 - +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: s_mul_i32 ; SI-DAG: s_mul_i32 -; SI-DAG: v_mul_hi_u32 -; SI-DAG: v_mul_hi_u32 ; SI-DAG: s_mul_i32 ; SI-DAG: s_mul_i32 -; SI-DAG: v_mul_hi_u32 - -; VI: v_mul_hi_u32 -; VI: s_mul_i32 -; VI: s_mul_i32 -; VI: v_mul_hi_u32 -; VI: v_mul_hi_u32 -; VI: s_mul_i32 -; VI: v_mad_u64_u32 -; VI: s_mul_i32 -; VI: v_mad_u64_u32 -; VI: s_mul_i32 -; VI: s_mul_i32 -; VI: v_mad_u64_u32 -; VI: s_mul_i32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: s_mul_i32 +; VI-DAG: s_mul_i32 +; VI-DAG: s_mul_i32 +; VI-DAG: s_mul_i32 ; GCN: buffer_store_dwordx4 define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, [8 x i32], i128 %a, [8 x i32], i128 %b) nounwind #0 { @@ -251,30 +240,27 @@ ; GCN: {{buffer|flat}}_load_dwordx4 ; GCN: {{buffer|flat}}_load_dwordx4 -; SI-DAG: v_mul_lo_u32 -; SI-DAG: v_mul_hi_u32 -; SI-DAG: v_mul_hi_u32 -; SI-DAG: v_mul_lo_u32 -; SI-DAG: v_mul_hi_u32 -; SI-DAG: v_mul_hi_u32 -; SI-DAG: v_mul_lo_u32 -; SI-DAG: v_mul_lo_u32 -; SI-DAG: v_add_i32_e32 - -; SI-DAG: v_mul_hi_u32 -; SI-DAG: v_mul_lo_u32 -; SI-DAG: v_mul_hi_u32 -; SI-DAG: v_mul_lo_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 +; SI-DAG: v_mad_u64_u32 ; SI-DAG: v_mul_lo_u32 ; SI-DAG: v_mul_lo_u32 ; SI-DAG: v_mul_lo_u32 ; SI-DAG: v_mul_lo_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mad_u64_u32 +; VI-DAG: v_mul_lo_u32 +; VI-DAG: v_mul_lo_u32 +; VI-DAG: v_mul_lo_u32 ; VI-DAG: v_mul_lo_u32 -; VI-DAG: v_mul_hi_u32 -; VI: v_mad_u64_u32 -; VI: v_mad_u64_u32 -; VI: v_mad_u64_u32 ; GCN: {{buffer|flat}}_store_dwordx4 define amdgpu_kernel void @v_mul_i128(i128 addrspace(1)* %out, i128 addrspace(1)* %aptr, i128 addrspace(1)* %bptr) #0 { diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll --- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll +++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll @@ -313,9 +313,8 @@ ; SI-NEXT: s_bfe_i32 s1, s2, 0x180000 ; SI-NEXT: s_bfe_i32 s0, s0, 0x180000 ; SI-NEXT: v_mov_b32_e32 v0, s1 -; SI-NEXT: s_mul_i32 s1, s0, s1 ; SI-NEXT: v_mul_hi_i32_i24_e32 v1, s0, v0 -; SI-NEXT: v_mov_b32_e32 v0, s1 +; SI-NEXT: v_mul_i32_i24_e32 v0, s0, v0 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -331,8 +330,7 @@ ; VI-NEXT: s_bfe_i32 s0, s0, 0x180000 ; VI-NEXT: v_mov_b32_e32 v0, s1 ; VI-NEXT: v_mul_hi_i32_i24_e32 v1, s0, v0 -; VI-NEXT: s_mul_i32 s0, s0, s1 -; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mul_i32_i24_e32 v0, s0, v0 ; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; VI-NEXT: s_endpgm ; @@ -413,9 +411,8 @@ ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_bfe_i32 s4, s4, 0x180000 -; SI-NEXT: s_mul_i32 s5, s4, s4 ; SI-NEXT: v_mul_hi_i32_i24_e64 v1, s4, s4 -; SI-NEXT: v_mov_b32_e32 v0, s5 +; SI-NEXT: v_mul_i32_i24_e64 v0, s4, s4 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -428,8 +425,7 @@ ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_bfe_i32 s0, s0, 0x180000 ; VI-NEXT: v_mul_hi_i32_i24_e64 v1, s0, s0 -; VI-NEXT: s_mul_i32 s0, s0, s0 -; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mul_i32_i24_e64 v0, s0, s0 ; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; VI-NEXT: s_endpgm ; @@ -506,9 +502,8 @@ ; SI-NEXT: s_ashr_i64 s[2:3], s[2:3], 40 ; SI-NEXT: s_ashr_i64 s[0:1], s[0:1], 40 ; SI-NEXT: v_mov_b32_e32 v0, s2 -; SI-NEXT: s_mul_i32 s1, s0, s2 ; SI-NEXT: v_mul_hi_i32_i24_e32 v1, s0, v0 -; SI-NEXT: v_mov_b32_e32 v0, s1 +; SI-NEXT: v_mul_i32_i24_e32 v0, s0, v0 ; SI-NEXT: v_lshl_b64 v[0:1], v[0:1], 31 ; SI-NEXT: v_ashr_i64 v[0:1], v[0:1], 31 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 @@ -528,8 +523,7 @@ ; VI-NEXT: s_ashr_i64 s[0:1], s[0:1], 40 ; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mul_hi_i32_i24_e32 v1, s0, v0 -; VI-NEXT: s_mul_i32 s0, s0, s2 -; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mul_i32_i24_e32 v0, s0, v0 ; VI-NEXT: v_lshlrev_b64 v[0:1], 31, v[0:1] ; VI-NEXT: v_ashrrev_i64 v[0:1], 31, v[0:1] ; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 diff --git a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll --- a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll +++ b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll @@ -508,16 +508,12 @@ ; SI-NEXT: s_load_dword s7, s[0:1], 0xd ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 -; SI-NEXT: s_mov_b32 s8, 0xffffff ; SI-NEXT: s_mov_b32 s0, s4 ; SI-NEXT: s_mov_b32 s1, s5 -; SI-NEXT: s_and_b32 s4, s6, s8 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_and_b32 s5, s7, s8 ; SI-NEXT: v_mov_b32_e32 v0, s7 -; SI-NEXT: s_mul_i32 s4, s4, s5 ; SI-NEXT: v_mul_hi_u32_u24_e32 v1, s6, v0 -; SI-NEXT: v_mov_b32_e32 v0, s4 +; SI-NEXT: v_mul_u32_u24_e32 v0, s6, v0 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -529,15 +525,11 @@ ; VI-NEXT: s_mov_b32 s3, 0xf000 ; VI-NEXT: s_mov_b32 s2, -1 ; VI-NEXT: s_mov_b32 s0, s4 -; VI-NEXT: s_mov_b32 s4, 0xffffff -; VI-NEXT: s_mov_b32 s1, s5 -; VI-NEXT: s_and_b32 s5, s6, s4 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s4, s7, s4 -; VI-NEXT: s_mul_i32 s5, s5, s4 ; VI-NEXT: v_mov_b32_e32 v0, s7 +; VI-NEXT: s_mov_b32 s1, s5 ; VI-NEXT: v_mul_hi_u32_u24_e32 v1, s6, v0 -; VI-NEXT: v_mov_b32_e32 v0, s5 +; VI-NEXT: v_mul_u32_u24_e32 v0, s6, v0 ; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; VI-NEXT: s_endpgm ; @@ -610,10 +602,8 @@ ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_and_b32 s5, s4, 0xffffff -; SI-NEXT: s_mul_i32 s5, s5, s5 ; SI-NEXT: v_mul_hi_u32_u24_e64 v1, s4, s4 -; SI-NEXT: v_mov_b32_e32 v0, s5 +; SI-NEXT: v_mul_u32_u24_e64 v0, s4, s4 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -624,10 +614,8 @@ ; VI-NEXT: s_mov_b32 s7, 0xf000 ; VI-NEXT: s_mov_b32 s6, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s1, s0, 0xffffff -; VI-NEXT: s_mul_i32 s1, s1, s1 ; VI-NEXT: v_mul_hi_u32_u24_e64 v1, s0, s0 -; VI-NEXT: v_mov_b32_e32 v0, s1 +; VI-NEXT: v_mul_u32_u24_e64 v0, s0, s0 ; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; VI-NEXT: s_endpgm ; @@ -712,21 +700,17 @@ define amdgpu_kernel void @test_umul24_i33(i64 addrspace(1)* %out, i33 %a, i33 %b) { ; SI-LABEL: test_umul24_i33: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; SI-NEXT: s_load_dword s2, s[0:1], 0xb -; SI-NEXT: s_load_dword s0, s[0:1], 0xd -; SI-NEXT: s_mov_b32 s7, 0xf000 -; SI-NEXT: s_mov_b32 s6, -1 -; SI-NEXT: s_mov_b32 s1, 0xffffff +; SI-NEXT: s_load_dword s4, s[0:1], 0xd +; SI-NEXT: s_load_dword s5, s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_and_b32 s3, s2, s1 -; SI-NEXT: s_and_b32 s1, s0, s1 -; SI-NEXT: v_mov_b32_e32 v0, s0 -; SI-NEXT: v_mul_hi_u32_u24_e32 v0, s2, v0 -; SI-NEXT: s_mul_i32 s3, s3, s1 -; SI-NEXT: v_and_b32_e32 v1, 1, v0 -; SI-NEXT: v_mov_b32_e32 v0, s3 -; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: v_mov_b32_e32 v1, s4 +; SI-NEXT: v_mul_u32_u24_e32 v0, s5, v1 +; SI-NEXT: v_mul_hi_u32_u24_e32 v1, s5, v1 +; SI-NEXT: v_and_b32_e32 v1, 1, v1 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: test_umul24_i33: @@ -734,17 +718,13 @@ ; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 ; VI-NEXT: s_load_dword s2, s[0:1], 0x2c ; VI-NEXT: s_load_dword s0, s[0:1], 0x34 -; VI-NEXT: s_mov_b32 s1, 0xffffff ; VI-NEXT: s_mov_b32 s7, 0xf000 ; VI-NEXT: s_mov_b32 s6, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s2, s1 -; VI-NEXT: s_and_b32 s1, s0, s1 -; VI-NEXT: v_mov_b32_e32 v0, s0 -; VI-NEXT: s_mul_i32 s3, s3, s1 -; VI-NEXT: v_mul_hi_u32_u24_e32 v0, s2, v0 -; VI-NEXT: v_and_b32_e32 v1, 1, v0 -; VI-NEXT: v_mov_b32_e32 v0, s3 +; VI-NEXT: v_mov_b32_e32 v1, s0 +; VI-NEXT: v_mul_u32_u24_e32 v0, s2, v1 +; VI-NEXT: v_mul_hi_u32_u24_e32 v1, s2, v1 +; VI-NEXT: v_and_b32_e32 v1, 1, v1 ; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; VI-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll --- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll +++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll @@ -6,9 +6,9 @@ ; GCN-LABEL: s_test_sdiv: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd +; GCN-NEXT: v_mov_b32_e32 v8, 0 ; GCN-NEXT: v_mov_b32_e32 v7, 0 ; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_ashr_i32 s8, s3, 31 ; GCN-NEXT: s_add_u32 s2, s2, s8 @@ -17,95 +17,83 @@ ; GCN-NEXT: s_xor_b64 s[10:11], s[2:3], s[8:9] ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s10 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s11 -; GCN-NEXT: s_sub_u32 s4, 0, s10 -; GCN-NEXT: s_subb_u32 s5, 0, s11 +; GCN-NEXT: s_sub_u32 s6, 0, s10 +; GCN-NEXT: s_subb_u32 s12, 0, s11 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s6, v4 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v5, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s12, v5 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v6, v5, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v5, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v4, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s6, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s12, v6 +; GCN-NEXT: v_mul_hi_u32 v10, v6, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_ashr_i32 s12, s3, 31 -; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v2, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc ; GCN-NEXT: s_add_u32 s2, s2, s12 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v6, v0 ; GCN-NEXT: s_mov_b32 s13, s12 -; GCN-NEXT: v_mul_lo_u32 v3, s4, v2 -; GCN-NEXT: v_mul_hi_u32 v4, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s5, v0 -; GCN-NEXT: v_mul_lo_u32 v5, s4, v0 ; GCN-NEXT: s_addc_u32 s3, s3, s12 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v5 -; GCN-NEXT: v_mul_lo_u32 v6, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v8, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v9, v2, v3 -; GCN-NEXT: v_mul_lo_u32 v3, v2, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v7, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v5, v2, v5 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc ; GCN-NEXT: s_xor_b64 s[2:3], s[2:3], s[12:13] -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v6, v5, vcc -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v1, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v3, s4, v2 -; GCN-NEXT: v_mul_hi_u32 v4, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v5, s5, v0 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_mul_lo_u32 v4, s4, v0 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v5, v3 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v6, v2, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v5, v2, v3 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v7, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v2, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v1, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v3, s2, v2 -; GCN-NEXT: v_mul_hi_u32 v4, s2, v0 -; GCN-NEXT: v_mul_hi_u32 v5, s2, v2 -; GCN-NEXT: v_mul_hi_u32 v6, s3, v2 -; GCN-NEXT: v_mul_lo_u32 v2, s3, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v5, s3, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v4, s2, v2 ; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v4, v0, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v2, s10, v1 -; GCN-NEXT: v_mul_hi_u32 v3, s10, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s11, v0 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[14:15], s3, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[14:15], s3, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v4, s10, v3 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[14:15], s10, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v5, s11, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_sub_i32_e32 v4, vcc, s3, v1 ; GCN-NEXT: v_mov_b32_e32 v5, s11 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_mul_lo_u32 v3, s10, v0 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, s3, v2 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, s2, v3 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 ; GCN-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GCN-NEXT: v_subrev_i32_e64 v5, s[0:1], s10, v3 +; GCN-NEXT: v_subrev_i32_e64 v5, s[0:1], s10, v0 ; GCN-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] ; GCN-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v4 ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] @@ -113,30 +101,30 @@ ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] ; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v4 ; GCN-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] -; GCN-NEXT: v_add_i32_e64 v5, s[0:1], 2, v0 -; GCN-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1] -; GCN-NEXT: v_add_i32_e64 v7, s[0:1], 1, v0 -; GCN-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1] +; GCN-NEXT: v_add_i32_e64 v5, s[0:1], 2, v2 +; GCN-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v3, s[0:1] +; GCN-NEXT: v_add_i32_e64 v7, s[0:1], 1, v2 +; GCN-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1] ; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 ; GCN-NEXT: v_cndmask_b32_e64 v4, v8, v6, s[0:1] ; GCN-NEXT: v_mov_b32_e32 v6, s3 -; GCN-NEXT: v_subb_u32_e32 v2, vcc, v6, v2, vcc -; GCN-NEXT: v_cmp_le_u32_e32 vcc, s11, v2 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GCN-NEXT: v_cmp_le_u32_e32 vcc, s11, v1 ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc -; GCN-NEXT: v_cmp_le_u32_e32 vcc, s10, v3 -; GCN-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s11, v2 -; GCN-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; GCN-NEXT: v_cndmask_b32_e64 v2, v7, v5, s[0:1] -; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GCN-NEXT: v_cmp_le_u32_e32 vcc, s10, v0 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s11, v1 +; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_cndmask_b32_e64 v1, v7, v5, s[0:1] +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc ; GCN-NEXT: s_xor_b64 s[0:1], s[12:13], s[8:9] -; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GCN-NEXT: v_xor_b32_e32 v0, s0, v0 -; GCN-NEXT: v_xor_b32_e32 v1, s1, v1 -; GCN-NEXT: v_mov_b32_e32 v2, s1 -; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc +; GCN-NEXT: v_xor_b32_e32 v1, s0, v1 +; GCN-NEXT: v_xor_b32_e32 v2, s1, v0 +; GCN-NEXT: v_mov_b32_e32 v3, s1 +; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s0, v1 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v2, v3, vcc ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; @@ -251,123 +239,111 @@ ; GCN-LABEL: v_test_sdiv: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: v_ashrrev_i32_e32 v4, 31, v3 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc -; GCN-NEXT: v_xor_b32_e32 v3, v3, v4 -; GCN-NEXT: v_xor_b32_e32 v2, v2, v4 -; GCN-NEXT: v_cvt_f32_u32_e32 v5, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v6, v3 -; GCN-NEXT: v_sub_i32_e32 v7, vcc, 0, v2 -; GCN-NEXT: v_subb_u32_e32 v8, vcc, 0, v3, vcc -; GCN-NEXT: v_mac_f32_e32 v5, 0x4f800000, v6 -; GCN-NEXT: v_rcp_f32_e32 v5, v5 +; GCN-NEXT: v_ashrrev_i32_e32 v8, 31, v3 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v8 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc +; GCN-NEXT: v_xor_b32_e32 v9, v3, v8 +; GCN-NEXT: v_xor_b32_e32 v10, v2, v8 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v10 +; GCN-NEXT: v_cvt_f32_u32_e32 v3, v9 +; GCN-NEXT: v_sub_i32_e32 v11, vcc, 0, v10 +; GCN-NEXT: v_subb_u32_e32 v12, vcc, 0, v9, vcc +; GCN-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_mov_b32_e32 v15, 0 ; GCN-NEXT: v_mov_b32_e32 v14, 0 -; GCN-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5 -; GCN-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5 -; GCN-NEXT: v_trunc_f32_e32 v6, v6 -; GCN-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GCN-NEXT: v_cvt_u32_f32_e32 v6, v6 -; GCN-NEXT: v_mul_hi_u32 v9, v7, v5 -; GCN-NEXT: v_mul_lo_u32 v10, v7, v6 -; GCN-NEXT: v_mul_lo_u32 v11, v8, v5 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v10 -; GCN-NEXT: v_mul_lo_u32 v10, v7, v5 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v11 -; GCN-NEXT: v_mul_lo_u32 v12, v5, v9 -; GCN-NEXT: v_mul_hi_u32 v13, v5, v10 -; GCN-NEXT: v_mul_hi_u32 v11, v5, v9 -; GCN-NEXT: v_mul_hi_u32 v16, v6, v9 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v9 -; GCN-NEXT: v_add_i32_e32 v12, vcc, v13, v12 -; GCN-NEXT: v_mul_lo_u32 v13, v6, v10 -; GCN-NEXT: v_mul_hi_u32 v10, v6, v10 -; GCN-NEXT: v_addc_u32_e32 v11, vcc, v15, v11, vcc -; GCN-NEXT: v_add_i32_e32 v12, vcc, v12, v13 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v11, v10, vcc -; GCN-NEXT: v_addc_u32_e32 v11, vcc, v16, v14, vcc -; GCN-NEXT: v_add_i32_e32 v9, vcc, v10, v9 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v15, v11, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v9 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v9, v7, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v7, v5 -; GCN-NEXT: v_mul_lo_u32 v8, v8, v5 -; GCN-NEXT: v_mul_lo_u32 v7, v7, v5 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v10, v9 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_mul_lo_u32 v11, v5, v8 -; GCN-NEXT: v_mul_hi_u32 v12, v5, v7 -; GCN-NEXT: v_mul_hi_u32 v13, v5, v8 -; GCN-NEXT: v_mul_hi_u32 v10, v6, v7 -; GCN-NEXT: v_mul_lo_u32 v7, v6, v7 -; GCN-NEXT: v_mul_hi_u32 v9, v6, v8 -; GCN-NEXT: v_add_i32_e32 v11, vcc, v12, v11 -; GCN-NEXT: v_addc_u32_e32 v12, vcc, v15, v13, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v6, v8 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v11, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v10, vcc -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v9, v14, vcc -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v8 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v15, v9, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v8, vcc -; GCN-NEXT: v_ashrrev_i32_e32 v7, 31, v1 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v7 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v7 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v6 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v6 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc -; GCN-NEXT: v_xor_b32_e32 v1, v1, v7 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v15, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v10, v1, v5 -; GCN-NEXT: v_mul_hi_u32 v5, v1, v5 -; GCN-NEXT: v_mul_hi_u32 v11, v1, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v1, v6 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v5, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v11, v14, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v15, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v9, v2, v5 -; GCN-NEXT: v_mul_lo_u32 v10, v3, v5 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_mul_lo_u32 v9, v2, v5 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GCN-NEXT: v_sub_i32_e32 v10, vcc, v1, v8 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v9 -; GCN-NEXT: v_subb_u32_e64 v9, s[4:5], v10, v3, vcc -; GCN-NEXT: v_sub_i32_e64 v10, s[4:5], v0, v2 -; GCN-NEXT: v_subbrev_u32_e64 v9, s[4:5], 0, v9, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v9, v3 +; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 +; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 +; GCN-NEXT: v_trunc_f32_e32 v3, v3 +; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 +; GCN-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v11, v7 +; GCN-NEXT: v_mul_lo_u32 v5, v12, v6 +; GCN-NEXT: v_mul_hi_u32 v13, v6, v2 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v3, v5 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GCN-NEXT: v_add_i32_e32 v13, vcc, v13, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v16, vcc, v15, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v13, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v16, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v14, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v15, v3, vcc +; GCN-NEXT: v_add_i32_e32 v13, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v16, vcc, v7, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v13, 0 +; GCN-NEXT: v_mul_lo_u32 v6, v11, v16 +; GCN-NEXT: v_mul_lo_u32 v7, v12, v13 +; GCN-NEXT: v_mul_hi_u32 v11, v13, v2 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v2, 0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v7 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v13, v3, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v16, v3, 0 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v11, v6 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v15, v7, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v14, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v15, v3, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v13, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v16, v3, vcc +; GCN-NEXT: v_ashrrev_i32_e32 v4, 31, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +; GCN-NEXT: v_xor_b32_e32 v6, v0, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v1, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v7, v6, v2 +; GCN-NEXT: v_xor_b32_e32 v5, v5, v4 +; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v0 +; GCN-NEXT: v_addc_u32_e32 v11, vcc, v15, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v7, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v11, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v14, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v15, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v7, v10, v3 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v11, v9, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v7 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v11 +; GCN-NEXT: v_sub_i32_e32 v7, vcc, v5, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v6, v0 +; GCN-NEXT: v_subb_u32_e64 v6, s[4:5], v7, v9, vcc +; GCN-NEXT: v_sub_i32_e64 v7, s[4:5], v0, v10 +; GCN-NEXT: v_subbrev_u32_e64 v6, s[4:5], 0, v6, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v9 ; GCN-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v10, v2 -; GCN-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5] -; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v9, v3 -; GCN-NEXT: v_cndmask_b32_e64 v9, v11, v10, s[4:5] -; GCN-NEXT: v_add_i32_e64 v10, s[4:5], 2, v5 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v8, vcc -; GCN-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v6, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GCN-NEXT: v_add_i32_e64 v12, s[4:5], 1, v5 -; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e64 v13, s[4:5], 0, v6, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v10 +; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5] +; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v9 +; GCN-NEXT: v_cndmask_b32_e64 v6, v11, v7, s[4:5] +; GCN-NEXT: v_add_i32_e64 v7, s[4:5], 2, v2 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v3, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v9 +; GCN-NEXT: v_add_i32_e64 v12, s[4:5], 1, v2 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v10 +; GCN-NEXT: v_addc_u32_e64 v13, s[4:5], 0, v3, s[4:5] ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v9 -; GCN-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc -; GCN-NEXT: v_cndmask_b32_e64 v9, v13, v11, s[4:5] +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v9 +; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6 +; GCN-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc +; GCN-NEXT: v_cndmask_b32_e64 v6, v13, v11, s[4:5] ; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 -; GCN-NEXT: v_cndmask_b32_e64 v1, v12, v10, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc -; GCN-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GCN-NEXT: v_xor_b32_e32 v2, v7, v4 +; GCN-NEXT: v_cndmask_b32_e64 v1, v12, v7, s[4:5] +; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GCN-NEXT: v_xor_b32_e32 v2, v4, v8 ; GCN-NEXT: v_xor_b32_e32 v3, v0, v2 ; GCN-NEXT: v_xor_b32_e32 v0, v1, v2 ; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 @@ -1113,9 +1089,9 @@ ; GCN-LABEL: s_test_sdiv_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: v_mov_b32_e32 v7, 0 +; GCN-NEXT: v_mov_b32_e32 v6, 0 ; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_ashr_i32 s8, s3, 31 ; GCN-NEXT: s_add_u32 s2, s2, s8 @@ -1124,77 +1100,75 @@ ; GCN-NEXT: s_xor_b64 s[2:3], s[2:3], s[8:9] ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s3 -; GCN-NEXT: s_sub_u32 s4, 0, s2 -; GCN-NEXT: s_subb_u32 s5, 0, s3 +; GCN-NEXT: s_sub_u32 s6, 0, s2 +; GCN-NEXT: s_subb_u32 s9, 0, s3 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s4, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v7, s5, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s4, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v2, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v10, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s4, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s5, v0 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, s4, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v2, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v1, 24 -; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 -; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 -; GCN-NEXT: v_mov_b32_e32 v5, s3 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s6, v5 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s9, v8 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v9, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v8, v10, 0 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v0, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v10, 0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, v9, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s6, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s9, v8 +; GCN-NEXT: v_mul_hi_u32 v10, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], 24, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 ; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v2, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v1, s3, v0 -; GCN-NEXT: v_mul_hi_u32 v3, s2, v0 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-NEXT: v_mul_lo_u32 v3, s2, v0 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[10:11], 0, v3, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 0, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v7, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v4, s2, v3 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[10:11], s2, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v5, s3, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 ; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v1 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, 24, v3 +; GCN-NEXT: v_mov_b32_e32 v5, s3 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 ; GCN-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GCN-NEXT: v_subrev_i32_e64 v5, s[0:1], s2, v3 +; GCN-NEXT: v_subrev_i32_e64 v5, s[0:1], s2, v0 ; GCN-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] ; GCN-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v4 ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] @@ -1202,28 +1176,28 @@ ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] ; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v4 ; GCN-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] -; GCN-NEXT: v_add_i32_e64 v5, s[0:1], 2, v0 -; GCN-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v2, s[0:1] -; GCN-NEXT: v_add_i32_e64 v7, s[0:1], 1, v0 +; GCN-NEXT: v_add_i32_e64 v5, s[0:1], 2, v2 +; GCN-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v3, s[0:1] +; GCN-NEXT: v_add_i32_e64 v7, s[0:1], 1, v2 +; GCN-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1] ; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc -; GCN-NEXT: v_addc_u32_e64 v2, s[0:1], 0, v2, s[0:1] -; GCN-NEXT: v_cmp_le_u32_e32 vcc, s3, v1 ; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 -; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v3 -; GCN-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc +; GCN-NEXT: v_cmp_le_u32_e32 vcc, s3, v1 +; GCN-NEXT: v_cndmask_b32_e64 v4, v8, v6, s[0:1] +; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc +; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v0 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc ; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s3, v1 -; GCN-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc -; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1] -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 -; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc -; GCN-NEXT: v_cndmask_b32_e64 v2, v7, v5, s[0:1] -; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GCN-NEXT: v_xor_b32_e32 v0, s8, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_cndmask_b32_e64 v1, v7, v5, s[0:1] +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc ; GCN-NEXT: v_xor_b32_e32 v1, s8, v1 -; GCN-NEXT: v_mov_b32_e32 v2, s8 -; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s8, v0 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc +; GCN-NEXT: v_xor_b32_e32 v2, s8, v0 +; GCN-NEXT: v_mov_b32_e32 v3, s8 +; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s8, v1 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v2, v3, vcc ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; @@ -1320,110 +1294,108 @@ ; GCN-LABEL: v_test_sdiv_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v1 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc -; GCN-NEXT: v_xor_b32_e32 v1, v1, v2 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, v0 -; GCN-NEXT: v_cvt_f32_u32_e32 v4, v1 -; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v0 -; GCN-NEXT: v_subb_u32_e32 v6, vcc, 0, v1, vcc -; GCN-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4 -; GCN-NEXT: v_rcp_f32_e32 v3, v3 +; GCN-NEXT: v_ashrrev_i32_e32 v6, 31, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v6 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_xor_b32_e32 v7, v1, v6 +; GCN-NEXT: v_xor_b32_e32 v8, v0, v6 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, v8 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v7 +; GCN-NEXT: v_sub_i32_e32 v9, vcc, 0, v8 +; GCN-NEXT: v_subb_u32_e32 v10, vcc, 0, v7, vcc +; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-NEXT: v_mov_b32_e32 v13, 0 ; GCN-NEXT: v_mov_b32_e32 v12, 0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3 -; GCN-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3 -; GCN-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v5, v3 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v8 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v9 -; GCN-NEXT: v_mul_lo_u32 v10, v3, v7 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v8 -; GCN-NEXT: v_mul_hi_u32 v9, v3, v7 -; GCN-NEXT: v_mul_hi_u32 v14, v4, v7 -; GCN-NEXT: v_mul_lo_u32 v7, v4, v7 -; GCN-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; GCN-NEXT: v_mul_lo_u32 v11, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v8, v4, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v13, v9, vcc -; GCN-NEXT: v_add_i32_e32 v10, vcc, v10, v11 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v9, v8, vcc -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v12, vcc -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v13, v9, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v7 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v4, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v7, v5, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v5, v3 -; GCN-NEXT: v_mul_lo_u32 v6, v6, v3 -; GCN-NEXT: v_mul_lo_u32 v5, v5, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_mul_lo_u32 v9, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v8, v4, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v4, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v6 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v10, v9 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v13, v11, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v4, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v9, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v8, vcc -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v7, v12, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v13, v7, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v4, 24 -; GCN-NEXT: v_mul_hi_u32 v3, v3, 24 -; GCN-NEXT: v_mul_hi_u32 v4, v4, 24 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v13, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v1, v3 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, v0, v3 -; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v4 -; GCN-NEXT: v_sub_i32_e32 v5, vcc, 24, v5 -; GCN-NEXT: v_subb_u32_e64 v6, s[4:5], v6, v1, vcc -; GCN-NEXT: v_sub_i32_e64 v7, s[4:5], v5, v0 -; GCN-NEXT: v_subbrev_u32_e64 v6, s[4:5], 0, v6, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v1 -; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v0 -; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5] -; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v1 -; GCN-NEXT: v_cndmask_b32_e64 v6, v8, v7, s[4:5] -; GCN-NEXT: v_add_i32_e64 v7, s[4:5], 2, v3 -; GCN-NEXT: v_addc_u32_e64 v8, s[4:5], 0, v13, s[4:5] -; GCN-NEXT: v_add_i32_e64 v9, s[4:5], 1, v3 -; GCN-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v13, s[4:5] -; GCN-NEXT: v_subb_u32_e32 v4, vcc, 0, v4, vcc -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v4, v1 -; GCN-NEXT: v_cndmask_b32_e64 v6, v10, v8, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v5, v0 +; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v9, v5 +; GCN-NEXT: v_mul_lo_u32 v3, v10, v4 +; GCN-NEXT: v_mul_hi_u32 v11, v4, v0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v4, v3, 0 +; GCN-NEXT: v_add_i32_e32 v11, vcc, v11, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v14, vcc, v13, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v11, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v14, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v12, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v13, v1, vcc +; GCN-NEXT: v_add_i32_e32 v11, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v14, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v11, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v9, v14 +; GCN-NEXT: v_mul_lo_u32 v5, v10, v11 +; GCN-NEXT: v_mul_hi_u32 v9, v11, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v14, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v11, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v9, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v13, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v13, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v11, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v14, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], 24, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v13, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], 0, v3, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 0, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v13, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v4, v8, v3 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v5, v7, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v7, vcc +; GCN-NEXT: v_sub_i32_e64 v5, s[4:5], v0, v8 +; GCN-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v7 +; GCN-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v8 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5] +; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v7 +; GCN-NEXT: v_cndmask_b32_e64 v4, v9, v5, s[4:5] +; GCN-NEXT: v_add_i32_e64 v5, s[4:5], 2, v2 +; GCN-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v3, s[4:5] +; GCN-NEXT: v_add_i32_e64 v10, s[4:5], 1, v2 +; GCN-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v3, s[4:5] +; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc +; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v7 +; GCN-NEXT: v_cndmask_b32_e64 v4, v11, v9, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8 ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v7 +; GCN-NEXT: v_cndmask_b32_e32 v0, v9, v0, vcc ; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 -; GCN-NEXT: v_cndmask_b32_e64 v1, v9, v7, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc -; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GCN-NEXT: v_xor_b32_e32 v3, v0, v2 -; GCN-NEXT: v_xor_b32_e32 v0, v1, v2 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc +; GCN-NEXT: v_cndmask_b32_e64 v1, v10, v5, s[4:5] +; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GCN-NEXT: v_xor_b32_e32 v2, v0, v6 +; GCN-NEXT: v_xor_b32_e32 v0, v1, v6 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v2, v6, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_sdiv_k_num_i64: @@ -1522,111 +1494,109 @@ ; GCN-LABEL: v_test_sdiv_pow2_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v1 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc -; GCN-NEXT: v_xor_b32_e32 v1, v1, v2 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, v0 -; GCN-NEXT: v_cvt_f32_u32_e32 v4, v1 -; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v0 -; GCN-NEXT: v_subb_u32_e32 v6, vcc, 0, v1, vcc -; GCN-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4 -; GCN-NEXT: v_rcp_f32_e32 v3, v3 +; GCN-NEXT: v_ashrrev_i32_e32 v6, 31, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v6 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_xor_b32_e32 v7, v1, v6 +; GCN-NEXT: v_xor_b32_e32 v8, v0, v6 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, v8 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v7 +; GCN-NEXT: v_sub_i32_e32 v9, vcc, 0, v8 +; GCN-NEXT: v_subb_u32_e32 v10, vcc, 0, v7, vcc +; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-NEXT: v_mov_b32_e32 v13, 0 ; GCN-NEXT: v_mov_b32_e32 v12, 0 -; GCN-NEXT: s_mov_b32 s4, 0x8000 -; GCN-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3 -; GCN-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3 -; GCN-NEXT: v_trunc_f32_e32 v4, v4 -; GCN-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v5, v3 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v8 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v9 -; GCN-NEXT: v_mul_lo_u32 v10, v3, v7 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v8 -; GCN-NEXT: v_mul_hi_u32 v9, v3, v7 -; GCN-NEXT: v_mul_hi_u32 v14, v4, v7 -; GCN-NEXT: v_mul_lo_u32 v7, v4, v7 -; GCN-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; GCN-NEXT: v_mul_lo_u32 v11, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v8, v4, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v13, v9, vcc -; GCN-NEXT: v_add_i32_e32 v10, vcc, v10, v11 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v9, v8, vcc -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v12, vcc -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v13, v9, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v7 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v4, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v7, v5, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v5, v3 -; GCN-NEXT: v_mul_lo_u32 v6, v6, v3 -; GCN-NEXT: v_mul_lo_u32 v5, v5, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_mul_lo_u32 v9, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v8, v4, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v4, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v6 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v10, v9 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v13, v11, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v4, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v9, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v8, vcc -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v7, v12, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v13, v7, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc -; GCN-NEXT: v_lshrrev_b32_e32 v5, 17, v4 -; GCN-NEXT: v_lshlrev_b32_e32 v4, 15, v4 -; GCN-NEXT: v_lshrrev_b32_e32 v3, 17, v3 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v13, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v1, v3 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, v0, v3 -; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v4 -; GCN-NEXT: v_sub_i32_e32 v5, vcc, s4, v5 -; GCN-NEXT: v_subb_u32_e64 v6, s[4:5], v6, v1, vcc -; GCN-NEXT: v_sub_i32_e64 v7, s[4:5], v5, v0 -; GCN-NEXT: v_subbrev_u32_e64 v6, s[4:5], 0, v6, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v1 -; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v0 -; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5] -; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, v1 -; GCN-NEXT: v_cndmask_b32_e64 v6, v8, v7, s[4:5] -; GCN-NEXT: v_add_i32_e64 v7, s[4:5], 2, v3 -; GCN-NEXT: v_addc_u32_e64 v8, s[4:5], 0, v13, s[4:5] -; GCN-NEXT: v_add_i32_e64 v9, s[4:5], 1, v3 -; GCN-NEXT: v_addc_u32_e64 v10, s[4:5], 0, v13, s[4:5] -; GCN-NEXT: v_subb_u32_e32 v4, vcc, 0, v4, vcc -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6 -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v4, v1 -; GCN-NEXT: v_cndmask_b32_e64 v6, v10, v8, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v5, v0 +; GCN-NEXT: s_mov_b32 s6, 0x8000 +; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v9, v5 +; GCN-NEXT: v_mul_lo_u32 v3, v10, v4 +; GCN-NEXT: v_mul_hi_u32 v11, v4, v0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v4, v3, 0 +; GCN-NEXT: v_add_i32_e32 v11, vcc, v11, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v14, vcc, v13, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v11, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v14, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v12, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v13, v1, vcc +; GCN-NEXT: v_add_i32_e32 v11, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v14, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v11, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v9, v14 +; GCN-NEXT: v_mul_lo_u32 v5, v10, v11 +; GCN-NEXT: v_mul_hi_u32 v9, v11, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v14, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v11, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v14, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v9, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v13, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v13, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v11, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v14, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v3, 0 +; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v13, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], 0, v3, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 0, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v12, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v13, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v4, v8, v3 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v5, v7, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 +; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v7, vcc +; GCN-NEXT: v_sub_i32_e64 v5, s[4:5], v0, v8 +; GCN-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v7 +; GCN-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v8 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5] +; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v7 +; GCN-NEXT: v_cndmask_b32_e64 v4, v9, v5, s[4:5] +; GCN-NEXT: v_add_i32_e64 v5, s[4:5], 2, v2 +; GCN-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v3, s[4:5] +; GCN-NEXT: v_add_i32_e64 v10, s[4:5], 1, v2 +; GCN-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v3, s[4:5] +; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc +; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v7 +; GCN-NEXT: v_cndmask_b32_e64 v4, v11, v9, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8 ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v7 +; GCN-NEXT: v_cndmask_b32_e32 v0, v9, v0, vcc ; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 -; GCN-NEXT: v_cndmask_b32_e64 v1, v9, v7, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc -; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GCN-NEXT: v_xor_b32_e32 v3, v0, v2 -; GCN-NEXT: v_xor_b32_e32 v0, v1, v2 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc +; GCN-NEXT: v_cndmask_b32_e64 v1, v10, v5, s[4:5] +; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc +; GCN-NEXT: v_xor_b32_e32 v2, v0, v6 +; GCN-NEXT: v_xor_b32_e32 v0, v1, v6 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v2, v6, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_sdiv_pow2_k_num_i64: diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll --- a/llvm/test/CodeGen/AMDGPU/srem64.ll +++ b/llvm/test/CodeGen/AMDGPU/srem64.ll @@ -6,93 +6,81 @@ ; GCN-LABEL: s_test_srem: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0xd -; GCN-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9 -; GCN-NEXT: v_mov_b32_e32 v2, 0 -; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; GCN-NEXT: v_mov_b32_e32 v7, 0 +; GCN-NEXT: v_mov_b32_e32 v6, 0 +; GCN-NEXT: s_mov_b32 s11, 0xf000 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s12 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s13 -; GCN-NEXT: s_sub_u32 s0, 0, s12 -; GCN-NEXT: s_subb_u32 s1, 0, s13 -; GCN-NEXT: s_mov_b32 s4, s8 +; GCN-NEXT: s_sub_u32 s2, 0, s12 +; GCN-NEXT: s_subb_u32 s3, 0, s13 +; GCN-NEXT: s_mov_b32 s10, -1 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: s_mov_b32 s5, s9 +; GCN-NEXT: s_mov_b32 s8, s4 +; GCN-NEXT: s_mov_b32 s9, s5 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v7, s1, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s0, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v2, v9, vcc -; GCN-NEXT: v_mul_hi_u32 v9, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v9, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s1, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, s0, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v2, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s10, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s10, v0 -; GCN-NEXT: v_mul_hi_u32 v6, s10, v3 -; GCN-NEXT: v_mul_hi_u32 v7, s11, v3 -; GCN-NEXT: v_mul_lo_u32 v3, s11, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, s11, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s11, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v0, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v1, s12, v1 -; GCN-NEXT: v_mul_hi_u32 v2, s12, v0 -; GCN-NEXT: v_mul_lo_u32 v3, s13, v0 -; GCN-NEXT: v_mul_lo_u32 v0, s12, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s2, v5 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s3, v8 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v9, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[0:1], v8, v10, 0 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v5, v0, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v5, v10, 0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, v9, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s2, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s3, v8 +; GCN-NEXT: v_mul_hi_u32 v10, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v8, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v4, s6, v2 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s7, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s7, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, s12, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s12, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v2, s13, v2 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, s11, v1 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, s7, v1 ; GCN-NEXT: v_mov_b32_e32 v3, s13 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s10, v0 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 ; GCN-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v3, vcc ; GCN-NEXT: v_subrev_i32_e64 v4, s[0:1], s12, v0 ; GCN-NEXT: v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1] @@ -107,7 +95,7 @@ ; GCN-NEXT: v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1] ; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v6 ; GCN-NEXT: v_cndmask_b32_e64 v2, v5, v2, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v5, s11 +; GCN-NEXT: v_mov_b32_e32 v5, s7 ; GCN-NEXT: v_subb_u32_e32 v1, vcc, v5, v1, vcc ; GCN-NEXT: v_cmp_le_u32_e32 vcc, s13, v1 ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc @@ -119,7 +107,7 @@ ; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ; GCN-NEXT: v_cndmask_b32_e64 v2, v4, v3, s[0:1] ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_srem: @@ -204,19 +192,18 @@ ; GCN-IR-NEXT: v_mov_b32_e32 v0, s6 ; GCN-IR-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[14:15] ; GCN-IR-NEXT: BB0_6: ; %udiv-end -; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v2, s0, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v3, s1, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v0, s0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v3, s0, v1 +; GCN-IR-NEXT: v_mad_u64_u32 v[1:2], s[2:3], s0, v0, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, s1, v0 ; GCN-IR-NEXT: s_mov_b32 s11, 0xf000 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-IR-NEXT: v_mov_b32_e32 v2, s7 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-IR-NEXT: v_mov_b32_e32 v3, s7 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s6, v1 ; GCN-IR-NEXT: s_mov_b32 s10, -1 ; GCN-IR-NEXT: s_mov_b32 s8, s4 ; GCN-IR-NEXT: s_mov_b32 s9, s5 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GCN-IR-NEXT: s_endpgm %result = urem i64 %x, %y @@ -231,122 +218,110 @@ ; GCN-NEXT: v_ashrrev_i32_e32 v4, 31, v3 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc -; GCN-NEXT: v_xor_b32_e32 v3, v3, v4 -; GCN-NEXT: v_xor_b32_e32 v2, v2, v4 -; GCN-NEXT: v_cvt_f32_u32_e32 v4, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v5, v3 -; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v2 -; GCN-NEXT: v_subb_u32_e32 v7, vcc, 0, v3, vcc -; GCN-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 -; GCN-NEXT: v_rcp_f32_e32 v4, v4 +; GCN-NEXT: v_xor_b32_e32 v8, v3, v4 +; GCN-NEXT: v_xor_b32_e32 v9, v2, v4 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, v9 +; GCN-NEXT: v_cvt_f32_u32_e32 v3, v8 +; GCN-NEXT: v_sub_i32_e32 v10, vcc, 0, v9 +; GCN-NEXT: v_subb_u32_e32 v11, vcc, 0, v8, vcc +; GCN-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_mov_b32_e32 v14, 0 ; GCN-NEXT: v_mov_b32_e32 v13, 0 -; GCN-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 -; GCN-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 -; GCN-NEXT: v_trunc_f32_e32 v5, v5 -; GCN-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5 -; GCN-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GCN-NEXT: v_mul_hi_u32 v8, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v5 -; GCN-NEXT: v_mul_lo_u32 v10, v7, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v9 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GCN-NEXT: v_mul_lo_u32 v11, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v12, v4, v9 -; GCN-NEXT: v_mul_hi_u32 v10, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v15, v5, v8 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v8 -; GCN-NEXT: v_add_i32_e32 v11, vcc, v12, v11 -; GCN-NEXT: v_mul_lo_u32 v12, v5, v9 -; GCN-NEXT: v_mul_hi_u32 v9, v5, v9 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v14, v10, vcc -; GCN-NEXT: v_add_i32_e32 v11, vcc, v11, v12 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v10, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v15, v13, vcc -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v10, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v6, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v7, v7, v4 -; GCN-NEXT: v_mul_lo_u32 v6, v6, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_mul_lo_u32 v10, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v11, v4, v6 -; GCN-NEXT: v_mul_hi_u32 v12, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v9, v5, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v5, v6 -; GCN-NEXT: v_mul_hi_u32 v8, v5, v7 -; GCN-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; GCN-NEXT: v_addc_u32_e32 v11, vcc, v14, v12, vcc -; GCN-NEXT: v_mul_lo_u32 v7, v5, v7 +; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 +; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 +; GCN-NEXT: v_trunc_f32_e32 v3, v3 +; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 +; GCN-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v10, v7 +; GCN-NEXT: v_mul_lo_u32 v5, v11, v6 +; GCN-NEXT: v_mul_hi_u32 v12, v6, v2 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v3, v5 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GCN-NEXT: v_add_i32_e32 v12, vcc, v12, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v15, vcc, v14, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v12, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v15, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v13, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v14, v3, vcc +; GCN-NEXT: v_add_i32_e32 v12, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v15, vcc, v7, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v12, 0 +; GCN-NEXT: v_mul_lo_u32 v6, v10, v15 +; GCN-NEXT: v_mul_lo_u32 v7, v11, v12 +; GCN-NEXT: v_mul_hi_u32 v10, v12, v2 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v15, v2, 0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v7 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v12, v3, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v15, v3, 0 ; GCN-NEXT: v_add_i32_e32 v6, vcc, v10, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v8, v13, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v14, v8, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc -; GCN-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v6 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GCN-NEXT: v_xor_b32_e32 v1, v1, v6 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v14, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v9, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v10, v1, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v1, v5 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v9 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v8, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v10, v13, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v14, v7, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v2, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v2, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v7, v5 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_sub_i32_e32 v7, vcc, v1, v5 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v7, v3, vcc -; GCN-NEXT: v_sub_i32_e64 v7, s[4:5], v0, v2 -; GCN-NEXT: v_subbrev_u32_e64 v8, s[6:7], 0, v4, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v8, v3 -; GCN-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7] -; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v7, v2 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v14, v7, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v13, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v14, v3, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v12, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v15, v3, vcc +; GCN-NEXT: v_ashrrev_i32_e32 v4, 31, v1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 +; GCN-NEXT: v_xor_b32_e32 v6, v0, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v1, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v7, v6, v2 +; GCN-NEXT: v_xor_b32_e32 v5, v5, v4 +; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v0 +; GCN-NEXT: v_addc_u32_e32 v10, vcc, v14, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v7, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v13, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v14, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, v9, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v8, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, v5, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v6, v0 +; GCN-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v8, vcc +; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v9 +; GCN-NEXT: v_subbrev_u32_e64 v6, s[6:7], 0, v2, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v8 +; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[6:7] +; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v9 ; GCN-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[6:7] -; GCN-NEXT: v_cmp_eq_u32_e64 s[6:7], v8, v3 -; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v3, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GCN-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[6:7] -; GCN-NEXT: v_sub_i32_e64 v10, s[4:5], v7, v2 +; GCN-NEXT: v_cmp_eq_u32_e64 s[6:7], v6, v8 +; GCN-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v8, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v7, v7, v10, s[6:7] +; GCN-NEXT: v_sub_i32_e64 v10, s[4:5], v3, v9 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v5, v1, vcc +; GCN-NEXT: v_subbrev_u32_e64 v2, s[4:5], 0, v2, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v8 +; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7 ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GCN-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v9 -; GCN-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; GCN-NEXT: v_cndmask_b32_e64 v2, v7, v10, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v4, v8, v4, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v9 +; GCN-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v8 +; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v6, vcc +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GCN-NEXT: v_cndmask_b32_e64 v2, v3, v10, s[4:5] ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GCN-NEXT: v_xor_b32_e32 v0, v0, v6 -; GCN-NEXT: v_xor_b32_e32 v1, v1, v6 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_xor_b32_e32 v0, v0, v4 +; GCN-NEXT: v_xor_b32_e32 v1, v1, v4 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_srem: @@ -446,12 +421,11 @@ ; GCN-IR-NEXT: BB1_6: ; %Flow4 ; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN-IR-NEXT: v_mul_lo_u32 v3, v5, v10 -; GCN-IR-NEXT: v_mul_hi_u32 v7, v5, v9 -; GCN-IR-NEXT: v_mul_lo_u32 v6, v6, v9 -; GCN-IR-NEXT: v_mul_lo_u32 v5, v5, v9 -; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, v7, v3 -; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v5 +; GCN-IR-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v5, v9, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v5, v6, v9 +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, v8, v3 +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, v3, v5 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v7 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc ; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v4 ; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v2 @@ -870,103 +844,91 @@ define amdgpu_kernel void @s_test_srem33_64(i64 addrspace(1)* %out, i64 %x, i64 %y) { ; GCN-LABEL: s_test_srem33_64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9 +; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd +; GCN-NEXT: v_mov_b32_e32 v8, 0 ; GCN-NEXT: v_mov_b32_e32 v7, 0 -; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: s_mov_b32 s11, 0xf000 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i64 s[2:3], s[10:11], 31 -; GCN-NEXT: s_ashr_i64 s[4:5], s[0:1], 31 +; GCN-NEXT: s_ashr_i64 s[2:3], s[6:7], 31 +; GCN-NEXT: s_ashr_i64 s[8:9], s[0:1], 31 ; GCN-NEXT: s_ashr_i32 s0, s1, 31 -; GCN-NEXT: s_add_u32 s4, s4, s0 +; GCN-NEXT: s_add_u32 s8, s8, s0 ; GCN-NEXT: s_mov_b32 s1, s0 -; GCN-NEXT: s_addc_u32 s5, s5, s0 -; GCN-NEXT: s_xor_b64 s[12:13], s[4:5], s[0:1] +; GCN-NEXT: s_addc_u32 s9, s9, s0 +; GCN-NEXT: s_xor_b64 s[12:13], s[8:9], s[0:1] ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s12 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s13 -; GCN-NEXT: s_sub_u32 s0, 0, s12 -; GCN-NEXT: s_subb_u32 s1, 0, s13 -; GCN-NEXT: s_ashr_i32 s10, s11, 31 +; GCN-NEXT: s_sub_u32 s6, 0, s12 +; GCN-NEXT: s_subb_u32 s8, 0, s13 +; GCN-NEXT: s_mov_b32 s10, -1 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: s_mov_b32 s11, s10 -; GCN-NEXT: s_mov_b32 s4, s8 +; GCN-NEXT: s_mov_b32 s9, s5 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v2, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s5, s9 -; GCN-NEXT: v_mul_lo_u32 v3, s0, v2 -; GCN-NEXT: v_mul_hi_u32 v4, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s1, v0 -; GCN-NEXT: v_mul_lo_u32 v5, s0, v0 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v5 -; GCN-NEXT: v_mul_lo_u32 v6, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v8, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v9, v2, v3 -; GCN-NEXT: v_mul_lo_u32 v3, v2, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v7, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v5, v2, v5 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v6, v5, vcc -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v1, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v3, s0, v2 -; GCN-NEXT: v_mul_hi_u32 v4, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v5, s1, v0 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_mul_lo_u32 v4, s0, v0 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v5, v3 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v6, v2, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v5, v2, v3 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v7, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v2, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v1, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc -; GCN-NEXT: s_add_u32 s0, s2, s10 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: s_addc_u32 s1, s3, s10 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc -; GCN-NEXT: s_xor_b64 s[14:15], s[0:1], s[10:11] -; GCN-NEXT: v_mul_lo_u32 v3, s14, v2 -; GCN-NEXT: v_mul_hi_u32 v4, s14, v0 -; GCN-NEXT: v_mul_hi_u32 v5, s14, v2 -; GCN-NEXT: v_mul_hi_u32 v6, s15, v2 -; GCN-NEXT: v_mul_lo_u32 v2, s15, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v5, s15, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s15, v0 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v4, v0, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v1, s12, v1 -; GCN-NEXT: v_mul_hi_u32 v2, s12, v0 -; GCN-NEXT: v_mul_lo_u32 v3, s13, v0 -; GCN-NEXT: v_mul_lo_u32 v0, s12, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s6, v4 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v5, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s8, v5 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v6, v5, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[0:1], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v4, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v4, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v5, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v4, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s6, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s8, v6 +; GCN-NEXT: v_mul_hi_u32 v10, v6, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v6, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v7, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: s_ashr_i32 s6, s7, 31 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: s_add_u32 s0, s2, s6 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v6, v0 +; GCN-NEXT: s_mov_b32 s7, s6 +; GCN-NEXT: s_addc_u32 s1, s3, s6 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc +; GCN-NEXT: s_xor_b64 s[14:15], s[0:1], s[6:7] +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s14, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v4, s14, v2 +; GCN-NEXT: s_mov_b32 s8, s4 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s15, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s15, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v8, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, s12, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s12, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v2, s13, v2 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN-NEXT: v_sub_i32_e32 v2, vcc, s15, v1 ; GCN-NEXT: v_mov_b32_e32 v3, s13 ; GCN-NEXT: v_sub_i32_e32 v0, vcc, s14, v0 @@ -996,12 +958,12 @@ ; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ; GCN-NEXT: v_cndmask_b32_e64 v2, v4, v3, s[0:1] ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GCN-NEXT: v_xor_b32_e32 v0, s10, v0 -; GCN-NEXT: v_xor_b32_e32 v1, s10, v1 -; GCN-NEXT: v_mov_b32_e32 v2, s10 -; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s10, v0 +; GCN-NEXT: v_xor_b32_e32 v0, s6, v0 +; GCN-NEXT: v_xor_b32_e32 v1, s6, v1 +; GCN-NEXT: v_mov_b32_e32 v2, s6 +; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s6, v0 ; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc -; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_srem33_64: @@ -1098,22 +1060,21 @@ ; GCN-IR-NEXT: v_mov_b32_e32 v0, s2 ; GCN-IR-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[16:17] ; GCN-IR-NEXT: BB8_6: ; %udiv-end -; GCN-IR-NEXT: v_mul_lo_u32 v1, s8, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v2, s8, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v3, s9, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v0, s8, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v3, s8, v1 +; GCN-IR-NEXT: v_mad_u64_u32 v[1:2], s[10:11], s8, v0, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, s9, v0 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; GCN-IR-NEXT: v_mov_b32_e32 v2, s3 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc -; GCN-IR-NEXT: v_xor_b32_e32 v0, s0, v0 -; GCN-IR-NEXT: v_xor_b32_e32 v1, s1, v1 -; GCN-IR-NEXT: v_mov_b32_e32 v2, s1 -; GCN-IR-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, s2, v1 +; GCN-IR-NEXT: v_subb_u32_e32 v0, vcc, v2, v0, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v1, s0, v1 +; GCN-IR-NEXT: v_xor_b32_e32 v2, s1, v0 +; GCN-IR-NEXT: v_mov_b32_e32 v3, s1 +; GCN-IR-NEXT: v_subrev_i32_e32 v0, vcc, s0, v1 ; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v2, v3, vcc ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %1 = ashr i64 %x, 31 @@ -1260,24 +1221,23 @@ ; GCN-IR-NEXT: v_mov_b32_e32 v0, s2 ; GCN-IR-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[16:17] ; GCN-IR-NEXT: BB9_6: ; %udiv-end -; GCN-IR-NEXT: v_mul_lo_u32 v1, s6, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v2, s6, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v3, s7, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v0, s6, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v3, s6, v1 +; GCN-IR-NEXT: v_mad_u64_u32 v[1:2], s[8:9], s6, v0, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, s7, v0 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; GCN-IR-NEXT: v_mov_b32_e32 v2, s3 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc -; GCN-IR-NEXT: v_xor_b32_e32 v0, s0, v0 -; GCN-IR-NEXT: v_xor_b32_e32 v1, s1, v1 +; GCN-IR-NEXT: v_sub_i32_e32 v1, vcc, s2, v1 +; GCN-IR-NEXT: v_subb_u32_e32 v0, vcc, v2, v0, vcc +; GCN-IR-NEXT: v_xor_b32_e32 v1, s0, v1 +; GCN-IR-NEXT: v_xor_b32_e32 v0, s1, v0 ; GCN-IR-NEXT: v_mov_b32_e32 v2, s1 -; GCN-IR-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc +; GCN-IR-NEXT: v_subrev_i32_e32 v1, vcc, s0, v1 +; GCN-IR-NEXT: v_subb_u32_e32 v0, vcc, v0, v2, vcc ; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 -; GCN-IR-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; GCN-IR-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4 +; GCN-IR-NEXT: buffer_store_dword v1, off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %1 = ashr i48 %x, 24 %2 = ashr i48 %y, 24 @@ -1290,7 +1250,8 @@ ; GCN-LABEL: s_test_srem_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: v_mov_b32_e32 v7, 0 +; GCN-NEXT: v_mov_b32_e32 v6, 0 ; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) @@ -1301,74 +1262,71 @@ ; GCN-NEXT: s_xor_b64 s[8:9], s[2:3], s[4:5] ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GCN-NEXT: s_sub_u32 s2, 0, s8 -; GCN-NEXT: s_subb_u32 s3, 0, s9 -; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_sub_u32 s4, 0, s8 +; GCN-NEXT: s_subb_u32 s5, 0, s9 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s2, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s2, v0 -; GCN-NEXT: v_mul_lo_u32 v7, s3, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s2, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v2, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v10, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s2, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s2, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s3, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, s2, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v2, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v1, 24 -; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 -; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v2, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v1, s9, v0 -; GCN-NEXT: v_mul_hi_u32 v2, s8, v0 -; GCN-NEXT: v_mul_lo_u32 v0, s8, v0 -; GCN-NEXT: v_mov_b32_e32 v3, s9 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s4, v5 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s5, v8 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v9, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v8, v10, 0 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[2:3], v5, v0, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v5, v10, 0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, v9, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s4, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s5, v8 +; GCN-NEXT: v_mul_hi_u32 v10, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[2:3], v8, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], 24, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], 0, v3, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 0, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, s8, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s8, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v2, s9, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v1 +; GCN-NEXT: v_mov_b32_e32 v3, s9 ; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 ; GCN-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v3, vcc ; GCN-NEXT: v_subrev_i32_e64 v4, s[0:1], s8, v0 @@ -1473,16 +1431,15 @@ ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 ; GCN-IR-NEXT: v_cndmask_b32_e64 v0, 24, 0, s[10:11] ; GCN-IR-NEXT: BB10_6: ; %udiv-end -; GCN-IR-NEXT: v_mul_lo_u32 v1, s4, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v2, s4, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v3, s5, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v0, s4, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v3, s4, v1 +; GCN-IR-NEXT: v_mad_u64_u32 v[1:2], s[6:7], s4, v0, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, s5, v0 ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v1 ; GCN-IR-NEXT: s_mov_b32 s2, -1 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v2, vcc ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %result = srem i64 24, %x @@ -1497,102 +1454,100 @@ ; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v1 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc -; GCN-NEXT: v_xor_b32_e32 v1, v1, v2 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, v1 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 -; GCN-NEXT: v_subb_u32_e32 v5, vcc, 0, v1, vcc -; GCN-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 -; GCN-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-NEXT: v_xor_b32_e32 v6, v1, v2 +; GCN-NEXT: v_xor_b32_e32 v7, v0, v2 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, v7 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v6 +; GCN-NEXT: v_sub_i32_e32 v8, vcc, 0, v7 +; GCN-NEXT: v_subb_u32_e32 v9, vcc, 0, v6, vcc +; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-NEXT: v_mov_b32_e32 v12, 0 ; GCN-NEXT: v_mov_b32_e32 v11, 0 -; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_mul_hi_u32 v6, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v7, v4, v3 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v7 -; GCN-NEXT: v_mul_lo_u32 v7, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v8 -; GCN-NEXT: v_mul_lo_u32 v9, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v7 -; GCN-NEXT: v_mul_hi_u32 v8, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v13, v3, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v10, v9 -; GCN-NEXT: v_mul_lo_u32 v10, v3, v7 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v7 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v12, v8, vcc -; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v10 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v8, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v13, v11, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v8, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v4, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v5, v5, v2 -; GCN-NEXT: v_mul_lo_u32 v4, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GCN-NEXT: v_mul_lo_u32 v8, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v5 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v12, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 +; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v8, v5 +; GCN-NEXT: v_mul_lo_u32 v3, v9, v4 +; GCN-NEXT: v_mul_hi_u32 v10, v4, v0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v4, v3, 0 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v10, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v12, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v10, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v13, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v11, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v12, v1, vcc +; GCN-NEXT: v_add_i32_e32 v10, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v10, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v8, v13 +; GCN-NEXT: v_mul_lo_u32 v5, v9, v10 +; GCN-NEXT: v_mul_hi_u32 v8, v10, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v1, 0 ; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v11, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v12, v6, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, 24 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v12, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v11, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v12, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v10, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v13, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], 24, v3, 0 ; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 -; GCN-NEXT: v_mul_hi_u32 v3, v3, 24 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v12, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v1, v2 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v0, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v3 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, 24, v2 -; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v1, vcc -; GCN-NEXT: v_sub_i32_e64 v5, s[4:5], v2, v0 -; GCN-NEXT: v_subbrev_u32_e64 v6, s[6:7], 0, v4, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v1 -; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[6:7] -; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v12, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], 0, v3, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 0, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v11, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v12, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, v7, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v6, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +; GCN-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v6, vcc +; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v7 +; GCN-NEXT: v_subbrev_u32_e64 v4, s[6:7], 0, v2, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v4, v6 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[6:7] +; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v7 ; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7] -; GCN-NEXT: v_cmp_eq_u32_e64 s[6:7], v6, v1 -; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v1, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[6:7] -; GCN-NEXT: v_sub_i32_e64 v8, s[4:5], v5, v0 -; GCN-NEXT: v_subb_u32_e32 v3, vcc, 0, v3, vcc -; GCN-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1 -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7 +; GCN-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v6 +; GCN-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v6, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[6:7] +; GCN-NEXT: v_sub_i32_e64 v8, s[4:5], v3, v7 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc +; GCN-NEXT: v_subbrev_u32_e64 v2, s[4:5], 0, v2, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v6 +; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v5 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v7 ; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 -; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v7, v0, vcc -; GCN-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[4:5] -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 -; GCN-NEXT: v_cndmask_b32_e64 v1, v6, v4, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc -; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6 +; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc +; GCN-NEXT: v_cndmask_b32_e64 v3, v3, v8, s[4:5] +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GCN-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5] +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_srem_k_num_i64: @@ -1608,24 +1563,24 @@ ; GCN-IR-NEXT: v_ffbh_u32_e32 v3, v1 ; GCN-IR-NEXT: v_min_u32_e32 v6, v2, v3 ; GCN-IR-NEXT: s_movk_i32 s6, 0xffc5 -; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, s6, v6 -; GCN-IR-NEXT: v_addc_u32_e64 v4, s[6:7], 0, -1, vcc +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, s6, v6 +; GCN-IR-NEXT: v_addc_u32_e64 v3, s[6:7], 0, -1, vcc ; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1] -; GCN-IR-NEXT: v_cmp_lt_u64_e32 vcc, 63, v[3:4] +; GCN-IR-NEXT: v_cmp_lt_u64_e32 vcc, 63, v[2:3] ; GCN-IR-NEXT: v_mov_b32_e32 v7, 0 ; GCN-IR-NEXT: s_or_b64 s[4:5], s[4:5], vcc -; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[3:4] -; GCN-IR-NEXT: v_cndmask_b32_e64 v2, 24, 0, s[4:5] +; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3] +; GCN-IR-NEXT: v_cndmask_b32_e64 v4, 24, 0, s[4:5] ; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], -1 ; GCN-IR-NEXT: v_mov_b32_e32 v5, v7 ; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], vcc ; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ; GCN-IR-NEXT: s_cbranch_execz BB11_6 ; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 -; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 1, v3 -; GCN-IR-NEXT: v_addc_u32_e32 v9, vcc, 0, v4, vcc -; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v3 -; GCN-IR-NEXT: v_cmp_ge_u64_e32 vcc, v[8:9], v[3:4] +; GCN-IR-NEXT: v_add_i32_e32 v8, vcc, 1, v2 +; GCN-IR-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc +; GCN-IR-NEXT: v_cmp_ge_u64_e32 vcc, v[8:9], v[2:3] +; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v2 ; GCN-IR-NEXT: v_lshl_b64 v[2:3], 24, v2 ; GCN-IR-NEXT: v_mov_b32_e32 v4, 0 ; GCN-IR-NEXT: v_mov_b32_e32 v5, 0 @@ -1674,16 +1629,15 @@ ; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1 ; GCN-IR-NEXT: v_or_b32_e32 v5, v5, v3 -; GCN-IR-NEXT: v_or_b32_e32 v2, v4, v2 +; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 ; GCN-IR-NEXT: BB11_6: ; %Flow4 ; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] -; GCN-IR-NEXT: v_mul_lo_u32 v3, v0, v5 -; GCN-IR-NEXT: v_mul_hi_u32 v4, v0, v2 -; GCN-IR-NEXT: v_mul_lo_u32 v1, v1, v2 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, v2 -; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v5, v0, v5 +; GCN-IR-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v4, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v4 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v5 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v2 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %result = srem i64 24, %x @@ -1697,103 +1651,101 @@ ; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v1 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc -; GCN-NEXT: v_xor_b32_e32 v1, v1, v2 -; GCN-NEXT: v_xor_b32_e32 v0, v0, v2 -; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, v1 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 -; GCN-NEXT: v_subb_u32_e32 v5, vcc, 0, v1, vcc -; GCN-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 -; GCN-NEXT: v_rcp_f32_e32 v2, v2 +; GCN-NEXT: v_xor_b32_e32 v6, v1, v2 +; GCN-NEXT: v_xor_b32_e32 v7, v0, v2 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, v7 +; GCN-NEXT: v_cvt_f32_u32_e32 v1, v6 +; GCN-NEXT: v_sub_i32_e32 v8, vcc, 0, v7 +; GCN-NEXT: v_subb_u32_e32 v9, vcc, 0, v6, vcc +; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-NEXT: v_mov_b32_e32 v12, 0 ; GCN-NEXT: v_mov_b32_e32 v11, 0 -; GCN-NEXT: s_mov_b32 s4, 0x8000 -; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_mul_hi_u32 v6, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v7, v4, v3 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v7 -; GCN-NEXT: v_mul_lo_u32 v7, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v8 -; GCN-NEXT: v_mul_lo_u32 v9, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v7 -; GCN-NEXT: v_mul_hi_u32 v8, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v13, v3, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v9, vcc, v10, v9 -; GCN-NEXT: v_mul_lo_u32 v10, v3, v7 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v7 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v12, v8, vcc -; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v10 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v8, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v13, v11, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v8, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v4, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v5, v5, v2 -; GCN-NEXT: v_mul_lo_u32 v4, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GCN-NEXT: v_mul_lo_u32 v8, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v5 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v12, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 +; GCN-NEXT: s_mov_b32 s6, 0x8000 +; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v8, v5 +; GCN-NEXT: v_mul_lo_u32 v3, v9, v4 +; GCN-NEXT: v_mul_hi_u32 v10, v4, v0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v4, v3, 0 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v10, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v12, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v10, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v13, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v11, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v12, v1, vcc +; GCN-NEXT: v_add_i32_e32 v10, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v10, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v8, v13 +; GCN-NEXT: v_mul_lo_u32 v5, v9, v10 +; GCN-NEXT: v_mul_hi_u32 v8, v10, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v1, 0 ; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v11, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v12, v6, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_lshrrev_b32_e32 v4, 17, v3 -; GCN-NEXT: v_lshlrev_b32_e32 v3, 15, v3 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v12, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v11, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v12, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v10, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v13, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v3, 0 ; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v12, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v1, v2 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v0, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v3 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, s4, v2 -; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v1, vcc -; GCN-NEXT: v_sub_i32_e64 v5, s[4:5], v2, v0 -; GCN-NEXT: v_subbrev_u32_e64 v6, s[6:7], 0, v4, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v1 -; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[6:7] -; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v5, v0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v12, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], 0, v3, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, 0, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v11, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v12, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, v7, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v6, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 +; GCN-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v6, vcc +; GCN-NEXT: v_sub_i32_e64 v3, s[4:5], v0, v7 +; GCN-NEXT: v_subbrev_u32_e64 v4, s[6:7], 0, v2, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v4, v6 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[6:7] +; GCN-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v7 ; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7] -; GCN-NEXT: v_cmp_eq_u32_e64 s[6:7], v6, v1 -; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v1, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[6:7] -; GCN-NEXT: v_sub_i32_e64 v8, s[4:5], v5, v0 -; GCN-NEXT: v_subb_u32_e32 v3, vcc, 0, v3, vcc -; GCN-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1 -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7 +; GCN-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v6 +; GCN-NEXT: v_subb_u32_e64 v2, s[4:5], v2, v6, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[6:7] +; GCN-NEXT: v_sub_i32_e64 v8, s[4:5], v3, v7 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc +; GCN-NEXT: v_subbrev_u32_e64 v2, s[4:5], 0, v2, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v6 +; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v5 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v7 ; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 -; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v7, v0, vcc -; GCN-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[4:5] -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 -; GCN-NEXT: v_cndmask_b32_e64 v1, v6, v4, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc -; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6 +; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc +; GCN-NEXT: v_cndmask_b32_e64 v3, v3, v8, s[4:5] +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GCN-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5] +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_srem_pow2_k_num_i64: @@ -1881,13 +1833,12 @@ ; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 ; GCN-IR-NEXT: BB12_6: ; %Flow4 ; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] -; GCN-IR-NEXT: v_mul_lo_u32 v2, v0, v5 -; GCN-IR-NEXT: v_mul_hi_u32 v3, v0, v4 -; GCN-IR-NEXT: v_mul_lo_u32 v1, v1, v4 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, v4 -; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v5, v0, v5 +; GCN-IR-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v4, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v4 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v5 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v2 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %result = srem i64 32768, %x diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll --- a/llvm/test/CodeGen/AMDGPU/udiv.ll +++ b/llvm/test/CodeGen/AMDGPU/udiv.ll @@ -2486,99 +2486,87 @@ ; SI-NEXT: v_mov_b32_e32 v2, 0x4f800000 ; SI-NEXT: v_madak_f32 v2, 0, v2, 0x47c35000 ; SI-NEXT: v_rcp_f32_e32 v2, v2 -; SI-NEXT: s_mov_b32 s4, 0xfffe7960 +; SI-NEXT: s_mov_b32 s6, 0xfffe7960 ; SI-NEXT: v_mov_b32_e32 v10, 0 ; SI-NEXT: v_mov_b32_e32 v9, 0 ; SI-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 ; SI-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 ; SI-NEXT: v_trunc_f32_e32 v3, v3 ; SI-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; SI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; SI-NEXT: v_cvt_u32_f32_e32 v3, v3 -; SI-NEXT: v_mul_hi_u32 v4, v2, s4 -; SI-NEXT: v_mul_lo_u32 v5, v3, s4 -; SI-NEXT: v_mul_lo_u32 v6, v2, s4 -; SI-NEXT: v_subrev_i32_e32 v4, vcc, v2, v4 -; SI-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; SI-NEXT: v_mul_lo_u32 v7, v2, v4 -; SI-NEXT: v_mul_hi_u32 v8, v2, v6 -; SI-NEXT: v_mul_hi_u32 v5, v2, v4 -; SI-NEXT: v_mul_hi_u32 v11, v3, v4 -; SI-NEXT: v_mul_lo_u32 v4, v3, v4 -; SI-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; SI-NEXT: v_mul_lo_u32 v8, v3, v6 -; SI-NEXT: v_mul_hi_u32 v6, v3, v6 -; SI-NEXT: v_addc_u32_e32 v5, vcc, v10, v5, vcc -; SI-NEXT: v_add_i32_e32 v7, vcc, v7, v8 -; SI-NEXT: v_addc_u32_e32 v5, vcc, v5, v6, vcc -; SI-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; SI-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; SI-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; SI-NEXT: v_cvt_u32_f32_e32 v6, v2 +; SI-NEXT: v_cvt_u32_f32_e32 v7, v3 +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; SI-NEXT: v_mul_lo_u32 v4, v7, s6 +; SI-NEXT: v_subrev_i32_e32 v3, vcc, v6, v3 +; SI-NEXT: v_add_i32_e32 v5, vcc, v4, v3 +; SI-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; SI-NEXT: v_mul_hi_u32 v8, v6, v2 +; SI-NEXT: v_add_i32_e32 v8, vcc, v8, v3 +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; SI-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; SI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; SI-NEXT: v_add_i32_e32 v2, vcc, v8, v2 +; SI-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; SI-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; SI-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; SI-NEXT: v_mul_hi_u32 v4, v2, s4 -; SI-NEXT: v_mul_lo_u32 v5, v3, s4 -; SI-NEXT: v_mul_lo_u32 v6, v2, s4 -; SI-NEXT: s_mov_b32 s4, 0x186a0 -; SI-NEXT: v_subrev_i32_e32 v4, vcc, v2, v4 -; SI-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; SI-NEXT: v_mul_lo_u32 v5, v2, v4 -; SI-NEXT: v_mul_hi_u32 v7, v2, v6 -; SI-NEXT: v_mul_hi_u32 v8, v2, v4 -; SI-NEXT: v_mul_hi_u32 v11, v3, v4 -; SI-NEXT: v_mul_lo_u32 v4, v3, v4 -; SI-NEXT: v_add_i32_e32 v5, vcc, v7, v5 -; SI-NEXT: v_addc_u32_e32 v7, vcc, v10, v8, vcc -; SI-NEXT: v_mul_lo_u32 v8, v3, v6 -; SI-NEXT: v_mul_hi_u32 v6, v3, v6 -; SI-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; SI-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; SI-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; SI-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; SI-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; SI-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; SI-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; SI-NEXT: v_addc_u32_e32 v7, vcc, v7, v3, vcc +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; SI-NEXT: v_mul_lo_u32 v4, v7, s6 +; SI-NEXT: s_mov_b32 s6, 0x186a0 +; SI-NEXT: v_subrev_i32_e32 v3, vcc, v6, v3 +; SI-NEXT: v_add_i32_e32 v5, vcc, v3, v4 +; SI-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; SI-NEXT: v_mul_hi_u32 v8, v6, v2 +; SI-NEXT: v_add_i32_e32 v8, vcc, v8, v3 +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; SI-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; SI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; SI-NEXT: v_add_i32_e32 v2, vcc, v8, v2 +; SI-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; SI-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; SI-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; SI-NEXT: v_mul_lo_u32 v4, v0, v3 -; SI-NEXT: v_mul_hi_u32 v5, v0, v2 -; SI-NEXT: v_mul_hi_u32 v6, v0, v3 -; SI-NEXT: v_mul_hi_u32 v7, v1, v3 -; SI-NEXT: v_mul_lo_u32 v3, v1, v3 -; SI-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; SI-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc -; SI-NEXT: v_mul_lo_u32 v6, v1, v2 -; SI-NEXT: v_mul_hi_u32 v2, v1, v2 -; SI-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; SI-NEXT: v_addc_u32_e32 v2, vcc, v5, v2, vcc -; SI-NEXT: v_addc_u32_e32 v4, vcc, v7, v9, vcc -; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; SI-NEXT: v_addc_u32_e32 v3, vcc, v10, v4, vcc -; SI-NEXT: v_mul_lo_u32 v4, v3, s4 -; SI-NEXT: v_mul_hi_u32 v5, v2, s4 -; SI-NEXT: v_mul_lo_u32 v6, v2, s4 -; SI-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; SI-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 -; SI-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc -; SI-NEXT: v_subrev_i32_e32 v4, vcc, s4, v0 -; SI-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v1, vcc +; SI-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; SI-NEXT: v_add_i32_e32 v4, vcc, v6, v2 +; SI-NEXT: v_addc_u32_e32 v5, vcc, v7, v3, vcc +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v5, 0 +; SI-NEXT: v_mul_hi_u32 v6, v0, v4 +; SI-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; SI-NEXT: v_addc_u32_e32 v7, vcc, v10, v3, vcc +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v1, v4, 0 +; SI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v5, 0 +; SI-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; SI-NEXT: v_addc_u32_e32 v2, vcc, v7, v3, vcc +; SI-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc +; SI-NEXT: v_add_i32_e32 v4, vcc, v2, v4 +; SI-NEXT: v_addc_u32_e32 v5, vcc, v10, v3, vcc +; SI-NEXT: v_mul_lo_u32 v6, v5, s6 +; SI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, s6, 0 ; SI-NEXT: s_mov_b32 s4, 0x1869f -; SI-NEXT: v_cmp_lt_u32_e32 vcc, s4, v4 -; SI-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 -; SI-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc -; SI-NEXT: v_add_i32_e32 v5, vcc, 2, v2 -; SI-NEXT: v_addc_u32_e32 v6, vcc, 0, v3, vcc -; SI-NEXT: v_add_i32_e32 v7, vcc, 1, v2 +; SI-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; SI-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 +; SI-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc +; SI-NEXT: v_subrev_i32_e32 v2, vcc, s6, v0 +; SI-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v1, vcc +; SI-NEXT: v_cmp_lt_u32_e32 vcc, s4, v2 +; SI-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc +; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 +; SI-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc +; SI-NEXT: v_add_i32_e32 v3, vcc, 2, v4 +; SI-NEXT: v_addc_u32_e32 v6, vcc, 0, v5, vcc +; SI-NEXT: v_add_i32_e32 v7, vcc, 1, v4 ; SI-NEXT: v_cmp_lt_u32_e64 s[4:5], s4, v0 -; SI-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; SI-NEXT: v_addc_u32_e32 v8, vcc, 0, v5, vcc ; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5] ; SI-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1 -; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 ; SI-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[4:5] -; SI-NEXT: v_cndmask_b32_e32 v4, v7, v5, vcc +; SI-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; SI-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0 ; SI-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc -; SI-NEXT: v_cndmask_b32_e64 v0, v2, v4, s[4:5] -; SI-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5] +; SI-NEXT: v_cndmask_b32_e64 v0, v4, v2, s[4:5] +; SI-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] ; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_test_udiv64_mulhi_fold: @@ -2587,99 +2575,87 @@ ; VI-NEXT: v_mov_b32_e32 v2, 0x4f800000 ; VI-NEXT: v_madak_f32 v2, 0, v2, 0x47c35000 ; VI-NEXT: v_rcp_f32_e32 v2, v2 -; VI-NEXT: s_mov_b32 s4, 0xfffe7960 +; VI-NEXT: s_mov_b32 s6, 0xfffe7960 ; VI-NEXT: v_mov_b32_e32 v10, 0 ; VI-NEXT: v_mov_b32_e32 v9, 0 ; VI-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 ; VI-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 ; VI-NEXT: v_trunc_f32_e32 v3, v3 ; VI-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; VI-NEXT: v_cvt_u32_f32_e32 v2, v2 -; VI-NEXT: v_cvt_u32_f32_e32 v3, v3 -; VI-NEXT: v_mul_hi_u32 v4, v2, s4 -; VI-NEXT: v_mul_lo_u32 v5, v3, s4 -; VI-NEXT: v_mul_lo_u32 v6, v2, s4 -; VI-NEXT: v_subrev_u32_e32 v4, vcc, v2, v4 -; VI-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; VI-NEXT: v_mul_lo_u32 v7, v2, v4 -; VI-NEXT: v_mul_hi_u32 v8, v2, v6 -; VI-NEXT: v_mul_hi_u32 v5, v2, v4 -; VI-NEXT: v_mul_hi_u32 v11, v3, v4 -; VI-NEXT: v_mul_lo_u32 v4, v3, v4 -; VI-NEXT: v_add_u32_e32 v7, vcc, v8, v7 -; VI-NEXT: v_mul_lo_u32 v8, v3, v6 -; VI-NEXT: v_mul_hi_u32 v6, v3, v6 -; VI-NEXT: v_addc_u32_e32 v5, vcc, v10, v5, vcc -; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v8 -; VI-NEXT: v_addc_u32_e32 v5, vcc, v5, v6, vcc -; VI-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; VI-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; VI-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; VI-NEXT: v_cvt_u32_f32_e32 v6, v2 +; VI-NEXT: v_cvt_u32_f32_e32 v7, v3 +; VI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; VI-NEXT: v_mul_lo_u32 v4, v7, s6 +; VI-NEXT: v_subrev_u32_e32 v3, vcc, v6, v3 +; VI-NEXT: v_add_u32_e32 v5, vcc, v4, v3 +; VI-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; VI-NEXT: v_mul_hi_u32 v8, v6, v2 +; VI-NEXT: v_add_u32_e32 v8, vcc, v8, v3 +; VI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; VI-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; VI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; VI-NEXT: v_add_u32_e32 v2, vcc, v8, v2 +; VI-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; VI-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4 -; VI-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; VI-NEXT: v_mul_hi_u32 v4, v2, s4 -; VI-NEXT: v_mul_lo_u32 v5, v3, s4 -; VI-NEXT: v_mul_lo_u32 v6, v2, s4 -; VI-NEXT: s_mov_b32 s4, 0x186a0 -; VI-NEXT: v_subrev_u32_e32 v4, vcc, v2, v4 -; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; VI-NEXT: v_mul_lo_u32 v5, v2, v4 -; VI-NEXT: v_mul_hi_u32 v7, v2, v6 -; VI-NEXT: v_mul_hi_u32 v8, v2, v4 -; VI-NEXT: v_mul_hi_u32 v11, v3, v4 -; VI-NEXT: v_mul_lo_u32 v4, v3, v4 -; VI-NEXT: v_add_u32_e32 v5, vcc, v7, v5 -; VI-NEXT: v_addc_u32_e32 v7, vcc, v10, v8, vcc -; VI-NEXT: v_mul_lo_u32 v8, v3, v6 -; VI-NEXT: v_mul_hi_u32 v6, v3, v6 -; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v8 -; VI-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; VI-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; VI-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; VI-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; VI-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v2 +; VI-NEXT: v_addc_u32_e32 v7, vcc, v7, v3, vcc +; VI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; VI-NEXT: v_mul_lo_u32 v4, v7, s6 +; VI-NEXT: s_mov_b32 s6, 0x186a0 +; VI-NEXT: v_subrev_u32_e32 v3, vcc, v6, v3 +; VI-NEXT: v_add_u32_e32 v5, vcc, v3, v4 +; VI-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; VI-NEXT: v_mul_hi_u32 v8, v6, v2 +; VI-NEXT: v_add_u32_e32 v8, vcc, v8, v3 +; VI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; VI-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; VI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; VI-NEXT: v_add_u32_e32 v2, vcc, v8, v2 +; VI-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; VI-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4 -; VI-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; VI-NEXT: v_mul_lo_u32 v4, v0, v3 -; VI-NEXT: v_mul_hi_u32 v5, v0, v2 -; VI-NEXT: v_mul_hi_u32 v6, v0, v3 -; VI-NEXT: v_mul_hi_u32 v7, v1, v3 -; VI-NEXT: v_mul_lo_u32 v3, v1, v3 -; VI-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; VI-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc -; VI-NEXT: v_mul_lo_u32 v6, v1, v2 -; VI-NEXT: v_mul_hi_u32 v2, v1, v2 -; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v6 -; VI-NEXT: v_addc_u32_e32 v2, vcc, v5, v2, vcc -; VI-NEXT: v_addc_u32_e32 v4, vcc, v7, v9, vcc -; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v3 -; VI-NEXT: v_addc_u32_e32 v3, vcc, v10, v4, vcc -; VI-NEXT: v_mul_lo_u32 v4, v3, s4 -; VI-NEXT: v_mul_hi_u32 v5, v2, s4 -; VI-NEXT: v_mul_lo_u32 v6, v2, s4 -; VI-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; VI-NEXT: v_sub_u32_e32 v0, vcc, v0, v6 -; VI-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc -; VI-NEXT: v_subrev_u32_e32 v4, vcc, s4, v0 -; VI-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v1, vcc +; VI-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; VI-NEXT: v_add_u32_e32 v4, vcc, v6, v2 +; VI-NEXT: v_addc_u32_e32 v5, vcc, v7, v3, vcc +; VI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v5, 0 +; VI-NEXT: v_mul_hi_u32 v6, v0, v4 +; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v2 +; VI-NEXT: v_addc_u32_e32 v7, vcc, v10, v3, vcc +; VI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v1, v4, 0 +; VI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v5, 0 +; VI-NEXT: v_add_u32_e32 v2, vcc, v6, v2 +; VI-NEXT: v_addc_u32_e32 v2, vcc, v7, v3, vcc +; VI-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc +; VI-NEXT: v_add_u32_e32 v4, vcc, v2, v4 +; VI-NEXT: v_addc_u32_e32 v5, vcc, v10, v3, vcc +; VI-NEXT: v_mul_lo_u32 v6, v5, s6 +; VI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, s6, 0 ; VI-NEXT: s_mov_b32 s4, 0x1869f -; VI-NEXT: v_cmp_lt_u32_e32 vcc, s4, v4 -; VI-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 -; VI-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc -; VI-NEXT: v_add_u32_e32 v5, vcc, 2, v2 -; VI-NEXT: v_addc_u32_e32 v6, vcc, 0, v3, vcc -; VI-NEXT: v_add_u32_e32 v7, vcc, 1, v2 +; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v6 +; VI-NEXT: v_sub_u32_e32 v0, vcc, v0, v2 +; VI-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc +; VI-NEXT: v_subrev_u32_e32 v2, vcc, s6, v0 +; VI-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v1, vcc +; VI-NEXT: v_cmp_lt_u32_e32 vcc, s4, v2 +; VI-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc +; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 +; VI-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc +; VI-NEXT: v_add_u32_e32 v3, vcc, 2, v4 +; VI-NEXT: v_addc_u32_e32 v6, vcc, 0, v5, vcc +; VI-NEXT: v_add_u32_e32 v7, vcc, 1, v4 ; VI-NEXT: v_cmp_lt_u32_e64 s[4:5], s4, v0 -; VI-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; VI-NEXT: v_addc_u32_e32 v8, vcc, 0, v5, vcc ; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5] ; VI-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1 -; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 ; VI-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[4:5] -; VI-NEXT: v_cndmask_b32_e32 v4, v7, v5, vcc +; VI-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; VI-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0 ; VI-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc -; VI-NEXT: v_cndmask_b32_e64 v0, v2, v4, s[4:5] -; VI-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5] +; VI-NEXT: v_cndmask_b32_e64 v0, v4, v2, s[4:5] +; VI-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] ; VI-NEXT: s_setpc_b64 s[30:31] ; ; GCN-LABEL: v_test_udiv64_mulhi_fold: @@ -2688,99 +2664,87 @@ ; GCN-NEXT: v_mov_b32_e32 v2, 0x4f800000 ; GCN-NEXT: v_madak_f32 v2, 0, v2, 0x47c35000 ; GCN-NEXT: v_rcp_f32_e32 v2, v2 -; GCN-NEXT: s_mov_b32 s4, 0xfffe7960 +; GCN-NEXT: s_mov_b32 s6, 0xfffe7960 ; GCN-NEXT: v_mov_b32_e32 v10, 0 ; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 ; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 ; GCN-NEXT: v_trunc_f32_e32 v3, v3 ; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_mul_hi_u32 v4, v2, s4 -; GCN-NEXT: v_mul_lo_u32 v5, v3, s4 -; GCN-NEXT: v_mul_lo_u32 v6, v2, s4 -; GCN-NEXT: v_subrev_u32_e32 v4, vcc, v2, v4 -; GCN-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v7, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v5, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_u32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v5, vcc -; GCN-NEXT: v_add_u32_e32 v7, vcc, v7, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; GCN-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; GCN-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v7, s6 +; GCN-NEXT: v_subrev_u32_e32 v3, vcc, v6, v3 +; GCN-NEXT: v_add_u32_e32 v5, vcc, v4, v3 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v8, v6, v2 +; GCN-NEXT: v_add_u32_e32 v8, vcc, v8, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; GCN-NEXT: v_add_u32_e32 v2, vcc, v8, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; GCN-NEXT: v_add_u32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_hi_u32 v4, v2, s4 -; GCN-NEXT: v_mul_lo_u32 v5, v3, s4 -; GCN-NEXT: v_mul_lo_u32 v6, v2, s4 -; GCN-NEXT: s_mov_b32 s4, 0x186a0 -; GCN-NEXT: v_subrev_u32_e32 v4, vcc, v2, v4 -; GCN-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v8, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_u32_e32 v5, vcc, v7, v5 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v10, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_add_u32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; GCN-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; GCN-NEXT: v_add_u32_e32 v6, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v7, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v7, s6 +; GCN-NEXT: s_mov_b32 s6, 0x186a0 +; GCN-NEXT: v_subrev_u32_e32 v3, vcc, v6, v3 +; GCN-NEXT: v_add_u32_e32 v5, vcc, v3, v4 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v8, v6, v2 +; GCN-NEXT: v_add_u32_e32 v8, vcc, v8, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; GCN-NEXT: v_add_u32_e32 v2, vcc, v8, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; GCN-NEXT: v_add_u32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v6, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v1, v3 -; GCN-NEXT: v_mul_lo_u32 v3, v1, v3 -; GCN-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v1, v2 -; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 -; GCN-NEXT: v_add_u32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v2, vcc -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v9, vcc -; GCN-NEXT: v_add_u32_e32 v2, vcc, v2, v3 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v10, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, s4 -; GCN-NEXT: v_mul_hi_u32 v5, v2, s4 -; GCN-NEXT: v_mul_lo_u32 v6, v2, s4 -; GCN-NEXT: v_add_u32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_sub_u32_e32 v0, vcc, v0, v6 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc -; GCN-NEXT: v_subrev_u32_e32 v4, vcc, s4, v0 -; GCN-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; GCN-NEXT: v_add_u32_e32 v4, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v6, v0, v4 +; GCN-NEXT: v_add_u32_e32 v6, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v10, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v1, v4, 0 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v5, 0 +; GCN-NEXT: v_add_u32_e32 v2, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc +; GCN-NEXT: v_add_u32_e32 v4, vcc, v2, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v3, vcc +; GCN-NEXT: v_mul_lo_u32 v6, v5, s6 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, s6, 0 ; GCN-NEXT: s_mov_b32 s4, 0x1869f -; GCN-NEXT: v_cmp_lt_u32_e32 vcc, s4, v4 -; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 -; GCN-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc -; GCN-NEXT: v_add_u32_e32 v5, vcc, 2, v2 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, 0, v3, vcc -; GCN-NEXT: v_add_u32_e32 v7, vcc, 1, v2 +; GCN-NEXT: v_add_u32_e32 v3, vcc, v3, v6 +; GCN-NEXT: v_sub_u32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc +; GCN-NEXT: v_subrev_u32_e32 v2, vcc, s6, v0 +; GCN-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v1, vcc +; GCN-NEXT: v_cmp_lt_u32_e32 vcc, s4, v2 +; GCN-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 +; GCN-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc +; GCN-NEXT: v_add_u32_e32 v3, vcc, 2, v4 +; GCN-NEXT: v_addc_u32_e32 v6, vcc, 0, v5, vcc +; GCN-NEXT: v_add_u32_e32 v7, vcc, 1, v4 ; GCN-NEXT: v_cmp_lt_u32_e64 s[4:5], s4, v0 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v8, vcc, 0, v5, vcc ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5] ; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1 -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 ; GCN-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v4, v7, v5, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0 ; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc -; GCN-NEXT: v_cndmask_b32_e64 v0, v2, v4, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v0, v4, v2, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GFX1030-LABEL: v_test_udiv64_mulhi_fold: @@ -2813,50 +2777,46 @@ ; GFX1030-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v9, vcc_lo ; GFX1030-NEXT: v_add_co_u32 v3, vcc_lo, v4, v3 ; GFX1030-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v5, vcc_lo -; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v2, v3 -; GFX1030-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, s4, v4, vcc_lo +; GFX1030-NEXT: v_add_co_u32 v5, vcc_lo, v2, v3 +; GFX1030-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s4, v4, vcc_lo +; GFX1030-NEXT: v_mul_hi_u32 v8, v0, v5 +; GFX1030-NEXT: v_mad_u64_u32 v[4:5], s4, v1, v5, 0 +; GFX1030-NEXT: v_mad_u64_u32 v[2:3], s4, v0, v6, 0 +; GFX1030-NEXT: v_mad_u64_u32 v[6:7], s4, v1, v6, 0 ; GFX1030-NEXT: s_mov_b32 s4, 0x186a0 -; GFX1030-NEXT: v_mul_hi_u32 v4, v0, v2 -; GFX1030-NEXT: v_mul_hi_u32 v7, v1, v2 -; GFX1030-NEXT: v_mul_lo_u32 v5, v0, v3 -; GFX1030-NEXT: v_mul_hi_u32 v6, v0, v3 -; GFX1030-NEXT: v_mul_lo_u32 v2, v1, v2 -; GFX1030-NEXT: v_mul_hi_u32 v8, v1, v3 -; GFX1030-NEXT: v_mul_lo_u32 v3, v1, v3 -; GFX1030-NEXT: v_add_co_u32 v4, vcc_lo, v4, v5 -; GFX1030-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v6, vcc_lo -; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v4, v2 -; GFX1030-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v5, v7, vcc_lo -; GFX1030-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v8, vcc_lo -; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v2, v3 -; GFX1030-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v4, vcc_lo -; GFX1030-NEXT: v_mul_hi_u32 v4, v2, s4 -; GFX1030-NEXT: v_mul_lo_u32 v6, v2, s4 -; GFX1030-NEXT: v_mul_lo_u32 v5, v3, s4 -; GFX1030-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v6 -; GFX1030-NEXT: v_add_nc_u32_e32 v4, v4, v5 -; GFX1030-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo -; GFX1030-NEXT: v_sub_co_u32 v4, vcc_lo, v0, s4 +; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v8, v2 +; GFX1030-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo +; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 +; GFX1030-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v5, vcc_lo +; GFX1030-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v7, vcc_lo +; GFX1030-NEXT: v_add_co_u32 v4, vcc_lo, v2, v6 +; GFX1030-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v3, vcc_lo +; GFX1030-NEXT: v_mad_u64_u32 v[2:3], s5, v4, s4, 0 +; GFX1030-NEXT: v_mul_lo_u32 v6, v5, s4 +; GFX1030-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v2 +; GFX1030-NEXT: v_add_nc_u32_e32 v3, v3, v6 +; GFX1030-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo +; GFX1030-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s4 ; GFX1030-NEXT: s_mov_b32 s4, 0x1869f -; GFX1030-NEXT: v_subrev_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo -; GFX1030-NEXT: v_cmp_lt_u32_e32 vcc_lo, s4, v4 -; GFX1030-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc_lo -; GFX1030-NEXT: v_add_co_u32 v6, vcc_lo, v2, 2 -; GFX1030-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v3, vcc_lo +; GFX1030-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo +; GFX1030-NEXT: v_cmp_lt_u32_e32 vcc_lo, s4, v2 +; GFX1030-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc_lo +; GFX1030-NEXT: v_add_co_u32 v6, vcc_lo, v4, 2 +; GFX1030-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo ; GFX1030-NEXT: v_cmp_lt_u32_e32 vcc_lo, s4, v0 ; GFX1030-NEXT: v_cmp_eq_u32_e64 s4, 0, v1 ; GFX1030-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo -; GFX1030-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5 +; GFX1030-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3 ; GFX1030-NEXT: v_cndmask_b32_e64 v0, -1, v0, s4 -; GFX1030-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo -; GFX1030-NEXT: v_add_co_u32 v5, vcc_lo, v2, 1 -; GFX1030-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v3, vcc_lo -; GFX1030-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4 -; GFX1030-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo -; GFX1030-NEXT: v_cndmask_b32_e32 v4, v8, v7, vcc_lo +; GFX1030-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1030-NEXT: v_add_co_u32 v3, vcc_lo, v4, 1 +; GFX1030-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v5, vcc_lo +; GFX1030-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2 +; GFX1030-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo +; GFX1030-NEXT: v_cndmask_b32_e32 v2, v8, v7, vcc_lo ; GFX1030-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0 -; GFX1030-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo -; GFX1030-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo +; GFX1030-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc_lo +; GFX1030-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo ; GFX1030-NEXT: s_setpc_b64 s[30:31] ; ; EG-LABEL: v_test_udiv64_mulhi_fold: diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll --- a/llvm/test/CodeGen/AMDGPU/udiv64.ll +++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll @@ -6,95 +6,83 @@ ; GCN-LABEL: s_test_udiv_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd -; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: v_mov_b32_e32 v7, 0 +; GCN-NEXT: v_mov_b32_e32 v6, 0 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GCN-NEXT: s_sub_u32 s4, 0, s8 -; GCN-NEXT: s_subb_u32 s5, 0, s9 +; GCN-NEXT: s_sub_u32 s6, 0, s8 +; GCN-NEXT: s_subb_u32 s10, 0, s9 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s4, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v7, s5, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s4, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v2, v9, vcc -; GCN-NEXT: v_mul_hi_u32 v9, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v9, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s4, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s5, v0 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, s4, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v2, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s2, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s2, v0 -; GCN-NEXT: v_mul_hi_u32 v6, s2, v3 -; GCN-NEXT: v_mul_hi_u32 v7, s3, v3 -; GCN-NEXT: v_mul_lo_u32 v3, s3, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, s3, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 -; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v0, vcc +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s6, v5 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s10, v8 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v9, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v8, v10, 0 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v0, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v10, 0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, v9, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v2, s8, v1 -; GCN-NEXT: v_mul_hi_u32 v3, s8, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s9, v0 +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s6, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s10, v8 +; GCN-NEXT: v_mul_hi_u32 v10, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v4, s2, v2 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[10:11], s3, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[10:11], s3, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v7, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v4, s8, v3 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[10:11], s8, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v5, s9, v2 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v5 +; GCN-NEXT: v_sub_i32_e32 v4, vcc, s3, v1 ; GCN-NEXT: v_mov_b32_e32 v5, s9 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_mul_lo_u32 v3, s8, v0 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, s3, v2 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, s2, v3 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 ; GCN-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GCN-NEXT: v_subrev_i32_e64 v5, s[0:1], s8, v3 +; GCN-NEXT: v_subrev_i32_e64 v5, s[0:1], s8, v0 ; GCN-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] ; GCN-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v4 ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] @@ -102,24 +90,24 @@ ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] ; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v4 ; GCN-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] -; GCN-NEXT: v_add_i32_e64 v5, s[0:1], 2, v0 -; GCN-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1] -; GCN-NEXT: v_add_i32_e64 v7, s[0:1], 1, v0 -; GCN-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1] +; GCN-NEXT: v_add_i32_e64 v5, s[0:1], 2, v2 +; GCN-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v3, s[0:1] +; GCN-NEXT: v_add_i32_e64 v7, s[0:1], 1, v2 +; GCN-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v3, s[0:1] ; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 ; GCN-NEXT: v_cndmask_b32_e64 v4, v8, v6, s[0:1] ; GCN-NEXT: v_mov_b32_e32 v6, s3 -; GCN-NEXT: v_subb_u32_e32 v2, vcc, v6, v2, vcc -; GCN-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GCN-NEXT: v_cmp_le_u32_e32 vcc, s9, v1 ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc -; GCN-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 -; GCN-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s9, v2 -; GCN-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; GCN-NEXT: v_cndmask_b32_e64 v2, v7, v5, s[0:1] -; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GCN-NEXT: v_cmp_le_u32_e32 vcc, s8, v0 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s9, v1 +; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_cndmask_b32_e64 v0, v7, v5, s[0:1] +; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; @@ -220,8 +208,8 @@ ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v4, v2 ; GCN-NEXT: v_cvt_f32_u32_e32 v5, v3 -; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v2 -; GCN-NEXT: v_subb_u32_e32 v7, vcc, 0, v3, vcc +; GCN-NEXT: v_sub_i32_e32 v10, vcc, 0, v2 +; GCN-NEXT: v_subb_u32_e32 v11, vcc, 0, v3, vcc ; GCN-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 ; GCN-NEXT: v_rcp_f32_e32 v4, v4 ; GCN-NEXT: v_mov_b32_e32 v14, 0 @@ -230,100 +218,88 @@ ; GCN-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 ; GCN-NEXT: v_trunc_f32_e32 v5, v5 ; GCN-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GCN-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v5 -; GCN-NEXT: v_mul_hi_u32 v8, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v10, v7, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v9 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GCN-NEXT: v_mul_lo_u32 v11, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v12, v4, v9 -; GCN-NEXT: v_mul_hi_u32 v10, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v15, v5, v8 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v8 -; GCN-NEXT: v_add_i32_e32 v11, vcc, v12, v11 -; GCN-NEXT: v_mul_lo_u32 v12, v5, v9 -; GCN-NEXT: v_mul_hi_u32 v9, v5, v9 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v14, v10, vcc -; GCN-NEXT: v_add_i32_e32 v11, vcc, v11, v12 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v10, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v15, v13, vcc -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v10, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v6, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v7, v7, v4 -; GCN-NEXT: v_mul_lo_u32 v6, v6, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_mul_lo_u32 v10, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v11, v4, v6 -; GCN-NEXT: v_mul_hi_u32 v12, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v9, v5, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v5, v6 -; GCN-NEXT: v_mul_hi_u32 v8, v5, v7 -; GCN-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; GCN-NEXT: v_addc_u32_e32 v11, vcc, v14, v12, vcc -; GCN-NEXT: v_mul_lo_u32 v7, v5, v7 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v10, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v8, v13, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v14, v8, vcc +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v5 +; GCN-NEXT: v_cvt_u32_f32_e32 v9, v4 +; GCN-NEXT: v_mul_lo_u32 v6, v10, v8 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0 +; GCN-NEXT: v_mul_lo_u32 v7, v11, v9 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 +; GCN-NEXT: v_add_i32_e32 v7, vcc, v5, v7 +; GCN-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0 +; GCN-NEXT: v_mul_hi_u32 v12, v9, v4 +; GCN-NEXT: v_add_i32_e32 v12, vcc, v12, v5 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0 +; GCN-NEXT: v_addc_u32_e32 v15, vcc, v14, v6, vcc +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v12, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v15, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v13, vcc ; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v1, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v1, v5 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v14, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v8 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v9, v13, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v14, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v2, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v4 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v2, v4 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v8 -; GCN-NEXT: v_sub_i32_e32 v8, vcc, v1, v6 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v7 -; GCN-NEXT: v_subb_u32_e64 v7, s[4:5], v8, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v14, v5, vcc +; GCN-NEXT: v_add_i32_e32 v12, vcc, v9, v4 +; GCN-NEXT: v_addc_u32_e32 v15, vcc, v8, v5, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0 +; GCN-NEXT: v_mul_lo_u32 v8, v10, v15 +; GCN-NEXT: v_mul_lo_u32 v9, v11, v12 +; GCN-NEXT: v_mul_hi_u32 v10, v12, v4 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v4, 0 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v9 +; GCN-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v12, v5, 0 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v15, v5, 0 +; GCN-NEXT: v_add_i32_e32 v8, vcc, v10, v8 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v9, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v8, v6 +; GCN-NEXT: v_addc_u32_e32 v6, vcc, v9, v7, vcc +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v14, v5, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v12, v4 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v15, v5, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0 +; GCN-NEXT: v_mul_hi_u32 v8, v0, v6 +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v4 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v5, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v13, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v4, v6 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v14, v5, vcc +; GCN-NEXT: v_mul_lo_u32 v8, v2, v7 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0 +; GCN-NEXT: v_mul_lo_u32 v9, v3, v6 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v9 +; GCN-NEXT: v_sub_i32_e32 v8, vcc, v1, v5 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 +; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v8, v3, vcc ; GCN-NEXT: v_sub_i32_e64 v8, s[4:5], v0, v2 -; GCN-NEXT: v_subbrev_u32_e64 v7, s[4:5], 0, v7, s[4:5] -; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v7, v3 +; GCN-NEXT: v_subbrev_u32_e64 v4, s[4:5], 0, v4, s[4:5] +; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v3 ; GCN-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5] ; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v2 ; GCN-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5] -; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v7, v3 -; GCN-NEXT: v_cndmask_b32_e64 v7, v9, v8, s[4:5] -; GCN-NEXT: v_add_i32_e64 v8, s[4:5], 2, v4 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v6, vcc -; GCN-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v5, s[4:5] +; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v3 +; GCN-NEXT: v_cndmask_b32_e64 v4, v9, v8, s[4:5] +; GCN-NEXT: v_add_i32_e64 v8, s[4:5], 2, v6 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc +; GCN-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v7, s[4:5] ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3 -; GCN-NEXT: v_add_i32_e64 v10, s[4:5], 1, v4 -; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc +; GCN-NEXT: v_add_i32_e64 v10, s[4:5], 1, v6 +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v5, s[4:5] +; GCN-NEXT: v_addc_u32_e64 v11, s[4:5], 0, v7, s[4:5] ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc ; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v7 -; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc -; GCN-NEXT: v_cndmask_b32_e64 v7, v10, v8, s[4:5] +; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4 +; GCN-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc +; GCN-NEXT: v_cndmask_b32_e64 v4, v10, v8, s[4:5] ; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 ; GCN-NEXT: v_cndmask_b32_e64 v1, v11, v9, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc -; GCN-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v4, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_udiv_i64: @@ -689,118 +665,111 @@ ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dword s2, s[0:1], 0xd ; GCN-NEXT: s_load_dword s3, s[0:1], 0xe -; GCN-NEXT: s_mov_b32 s7, 0xff000000 -; GCN-NEXT: s_mov_b32 s6, 0xffff -; GCN-NEXT: v_cvt_f32_ubyte3_e32 v2, s6 +; GCN-NEXT: s_mov_b32 s5, 0xff000000 +; GCN-NEXT: s_mov_b32 s4, 0xffff +; GCN-NEXT: v_cvt_f32_ubyte3_e32 v1, s4 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_and_b32 s2, s2, s7 -; GCN-NEXT: s_and_b32 s3, s3, s6 +; GCN-NEXT: s_and_b32 s2, s2, s5 +; GCN-NEXT: s_and_b32 s3, s3, s4 ; GCN-NEXT: v_mov_b32_e32 v0, s2 -; GCN-NEXT: v_alignbit_b32 v0, s3, v0, 24 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GCN-NEXT: s_load_dword s8, s[0:1], 0xb -; GCN-NEXT: s_load_dword s0, s[0:1], 0xc +; GCN-NEXT: v_alignbit_b32 v6, s3, v0, 24 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, v6 +; GCN-NEXT: s_load_dword s6, s[0:1], 0xb +; GCN-NEXT: s_load_dword s7, s[0:1], 0xc +; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 24 ; GCN-NEXT: v_mov_b32_e32 v9, 0 -; GCN-NEXT: v_mov_b32_e32 v8, 0 -; GCN-NEXT: v_mac_f32_e32 v1, 0x4f800000, v2 -; GCN-NEXT: v_rcp_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 +; GCN-NEXT: v_rcp_f32_e32 v0, v0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_and_b32 s6, s0, s6 -; GCN-NEXT: s_and_b32 s8, s8, s7 -; GCN-NEXT: s_lshr_b64 s[0:1], s[2:3], 24 -; GCN-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v1 -; GCN-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_mac_f32_e32 v1, 0xcf800000, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: s_sub_u32 s0, 0, s0 -; GCN-NEXT: s_subb_u32 s1, 0, s1 -; GCN-NEXT: v_mul_hi_u32 v3, s0, v1 -; GCN-NEXT: v_mul_lo_u32 v4, s0, v2 -; GCN-NEXT: v_mul_lo_u32 v5, s1, v1 -; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, s0, v1 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v6, v1, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v5, v1, v3 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v3 -; GCN-NEXT: v_mul_lo_u32 v3, v2, v3 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v2, v4 +; GCN-NEXT: s_and_b32 s7, s7, s4 +; GCN-NEXT: s_and_b32 s6, s6, s5 +; GCN-NEXT: s_sub_u32 s8, 0, s2 +; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: s_subb_u32 s9, 0, s3 +; GCN-NEXT: v_mov_b32_e32 v8, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s8, v4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, s8, v5 +; GCN-NEXT: v_mul_lo_u32 v3, s9, v4 +; GCN-NEXT: v_mul_hi_u32 v7, v4, v0 +; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v3 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v3, 0 +; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v10, vcc, v9, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v7, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v8, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v9, v1, vcc +; GCN-NEXT: v_add_i32_e32 v7, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v10, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s8, v7, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s8, v10 +; GCN-NEXT: v_mul_lo_u32 v5, s9, v7 +; GCN-NEXT: v_mul_hi_u32 v11, v7, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v10, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v7, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v10, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v11, v4 ; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v5, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v7 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v5, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v8, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v5, vcc -; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v3, s0, v2 -; GCN-NEXT: v_mul_hi_u32 v4, s0, v1 -; GCN-NEXT: v_mul_lo_u32 v5, s1, v1 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_mul_lo_u32 v4, s0, v1 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v5, v3 -; GCN-NEXT: v_mul_lo_u32 v7, v1, v3 -; GCN-NEXT: v_mul_hi_u32 v10, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v11, v1, v3 -; GCN-NEXT: v_mul_hi_u32 v6, v2, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v5, v2, v3 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v10, v7 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v9, v11, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v2, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v7, v4 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v10, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v8, vcc -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v5, vcc -; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-NEXT: v_mov_b32_e32 v3, s8 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc -; GCN-NEXT: v_alignbit_b32 v3, s6, v3, 24 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v2 -; GCN-NEXT: v_mul_hi_u32 v1, v3, v1 -; GCN-NEXT: v_mul_hi_u32 v2, v3, v2 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v8, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v9, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v7, v0 +; GCN-NEXT: v_mov_b32_e32 v0, s6 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v10, v1, vcc +; GCN-NEXT: v_alignbit_b32 v4, s7, v0, 24 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v4, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v5, v4, v2 +; GCN-NEXT: s_mov_b32 s7, 0xf000 ; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v9, v2, vcc -; GCN-NEXT: v_add_i32_e32 v1, vcc, 0, v1 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc -; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v8, vcc -; GCN-NEXT: v_add_i32_e32 v1, vcc, 0, v1 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v9, v2, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v7, v0, v1 -; GCN-NEXT: v_add_i32_e32 v4, vcc, 2, v1 -; GCN-NEXT: v_mul_lo_u32 v10, v0, v1 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, 0, v2, vcc -; GCN-NEXT: v_add_i32_e32 v8, vcc, 1, v1 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, 0, v2, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, v3, v10 -; GCN-NEXT: v_subb_u32_e32 v6, vcc, 0, v6, vcc -; GCN-NEXT: v_sub_i32_e32 v7, vcc, v3, v0 -; GCN-NEXT: v_subbrev_u32_e32 v10, vcc, 0, v6, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v7, v0 -; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v10 -; GCN-NEXT: v_cmp_ge_u32_e64 s[0:1], v3, v0 -; GCN-NEXT: v_cndmask_b32_e32 v7, -1, v7, vcc +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v0 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], 0, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], 0, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v5, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v8, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v8, v6, v3 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v6, v2, 0 +; GCN-NEXT: v_add_i32_e32 v5, vcc, 2, v2 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc +; GCN-NEXT: v_add_i32_e32 v9, vcc, 1, v2 +; GCN-NEXT: v_addc_u32_e32 v10, vcc, 0, v3, vcc +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v8 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc +; GCN-NEXT: v_sub_i32_e32 v4, vcc, v0, v6 +; GCN-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v1, vcc +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v4, v6 +; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 +; GCN-NEXT: v_cmp_ge_u32_e64 s[0:1], v0, v6 +; GCN-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1] -; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v6 -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7 +; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1 +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 ; GCN-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[0:1] -; GCN-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc ; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v0 -; GCN-NEXT: v_cndmask_b32_e64 v0, v1, v4, s[0:1] -; GCN-NEXT: v_cndmask_b32_e32 v1, v9, v5, vcc -; GCN-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[0:1] +; GCN-NEXT: v_cndmask_b32_e32 v1, v10, v7, vcc +; GCN-NEXT: v_cndmask_b32_e32 v4, v9, v5, vcc +; GCN-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[0:1] +; GCN-NEXT: v_cndmask_b32_e64 v0, v2, v4, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: buffer_store_short v1, off, s[4:7], 0 offset:4 ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; GCN-NEXT: s_endpgm @@ -914,107 +883,97 @@ ; GCN-LABEL: s_test_udiv_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 -; GCN-NEXT: v_mov_b32_e32 v2, 0 -; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: v_mov_b32_e32 v7, 0 +; GCN-NEXT: v_mov_b32_e32 v6, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s3 -; GCN-NEXT: s_sub_u32 s4, 0, s2 -; GCN-NEXT: s_subb_u32 s5, 0, s3 +; GCN-NEXT: s_sub_u32 s6, 0, s2 +; GCN-NEXT: s_subb_u32 s7, 0, s3 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s4, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v7, s5, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s4, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v3, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v2, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v8, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v10, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s4, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s5, v0 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, s4, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v2, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v1, 24 -; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 -; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 -; GCN-NEXT: v_mov_b32_e32 v5, s3 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s6, v5 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s7, v8 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v9, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v8, v10, 0 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v0, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v10, 0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, v9, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s6, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s6, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s7, v8 +; GCN-NEXT: v_mul_hi_u32 v10, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, 24, 0 +; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 +; GCN-NEXT: v_mov_b32_e32 v4, s3 +; GCN-NEXT: s_mov_b32 s7, 0xf000 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, s3, v2 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v2, 0 ; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v2, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v1, s3, v0 -; GCN-NEXT: v_mul_hi_u32 v3, s2, v0 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-NEXT: v_mul_lo_u32 v3, s2, v0 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v1 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, 24, v3 -; GCN-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GCN-NEXT: v_subrev_i32_e64 v5, s[0:1], s2, v3 -; GCN-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] -; GCN-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v4 -; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] -; GCN-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v5 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +; GCN-NEXT: v_subb_u32_e64 v3, s[0:1], v3, v4, vcc +; GCN-NEXT: v_subrev_i32_e64 v4, s[0:1], s2, v0 +; GCN-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v3, s[0:1] +; GCN-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v3 ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] -; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v4 -; GCN-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[0:1] -; GCN-NEXT: v_add_i32_e64 v5, s[0:1], 2, v0 -; GCN-NEXT: v_addc_u32_e64 v6, s[0:1], 0, v2, s[0:1] -; GCN-NEXT: v_add_i32_e64 v7, s[0:1], 1, v0 +; GCN-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v4 +; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1] +; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v3 +; GCN-NEXT: v_cndmask_b32_e64 v3, v5, v4, s[0:1] +; GCN-NEXT: v_add_i32_e64 v4, s[0:1], 2, v2 +; GCN-NEXT: v_addc_u32_e64 v5, s[0:1], 0, v7, s[0:1] +; GCN-NEXT: v_add_i32_e64 v6, s[0:1], 1, v2 +; GCN-NEXT: v_addc_u32_e64 v7, s[0:1], 0, v7, s[0:1] ; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc -; GCN-NEXT: v_addc_u32_e64 v2, s[0:1], 0, v2, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v3 ; GCN-NEXT: v_cmp_le_u32_e32 vcc, s3, v1 -; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 -; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v3 -; GCN-NEXT: v_cndmask_b32_e64 v3, 0, -1, vcc +; GCN-NEXT: v_cndmask_b32_e64 v3, v7, v5, s[0:1] +; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GCN-NEXT: v_cmp_le_u32_e32 vcc, s2, v0 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc ; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s3, v1 -; GCN-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc -; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1] -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 -; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc -; GCN-NEXT: v_cndmask_b32_e64 v2, v7, v5, s[0:1] -; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_cndmask_b32_e64 v0, v6, v4, s[0:1] +; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; @@ -1108,71 +1067,66 @@ ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ; GCN-NEXT: v_cvt_f32_u32_e32 v3, v1 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 -; GCN-NEXT: v_subb_u32_e32 v5, vcc, 0, v1, vcc +; GCN-NEXT: v_sub_i32_e32 v8, vcc, 0, v0 +; GCN-NEXT: v_subb_u32_e32 v9, vcc, 0, v1, vcc ; GCN-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 ; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_mov_b32_e32 v12, 0 ; GCN-NEXT: v_mov_b32_e32 v11, 0 +; GCN-NEXT: s_mov_b32 s6, 0x8000 ; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 ; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 ; GCN-NEXT: v_trunc_f32_e32 v3, v3 ; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_mul_lo_u32 v6, v4, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v2 -; GCN-NEXT: v_mul_lo_u32 v9, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v8 -; GCN-NEXT: v_mul_lo_u32 v8, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v9 -; GCN-NEXT: v_mul_hi_u32 v7, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v13, v3, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v10, v8 -; GCN-NEXT: v_mul_lo_u32 v10, v3, v9 -; GCN-NEXT: v_mul_hi_u32 v9, v3, v9 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v7, vcc -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v7, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v13, v11, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v8, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v4, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v5, v5, v2 -; GCN-NEXT: v_mul_lo_u32 v4, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GCN-NEXT: v_mul_lo_u32 v8, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v5 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v12, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v11, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v12, v6, vcc +; GCN-NEXT: v_cvt_u32_f32_e32 v6, v3 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v2 +; GCN-NEXT: v_mul_lo_u32 v4, v8, v6 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v7, 0 +; GCN-NEXT: v_mul_lo_u32 v5, v9, v7 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v3, v5 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v10, v7, v2 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v10, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v12, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v5, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v10, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v13, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v11, vcc ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v3, v5, vcc -; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 -; GCN-NEXT: v_mul_lo_u32 v3, v1, v2 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GCN-NEXT: v_mul_lo_u32 v4, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v12, v3, vcc +; GCN-NEXT: v_add_i32_e32 v10, vcc, v7, v2 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v6, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v10, 0 +; GCN-NEXT: v_mul_lo_u32 v6, v8, v13 +; GCN-NEXT: v_mul_lo_u32 v7, v9, v10 +; GCN-NEXT: v_mul_hi_u32 v8, v10, v2 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v2, 0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v7 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v10, v3, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v3, 0 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v8, v6 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v7, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v11, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v12, v3, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v13, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, s6, 0 +; GCN-NEXT: v_lshrrev_b32_e32 v4, 17, v4 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v12, v3, vcc +; GCN-NEXT: v_mul_lo_u32 v5, v1, v4 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v4, 0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 ; GCN-NEXT: v_sub_i32_e32 v5, vcc, 0, v3 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0x8000, v4 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, s6, v2 ; GCN-NEXT: v_subb_u32_e64 v5, s[4:5], v5, v1, vcc -; GCN-NEXT: v_sub_i32_e64 v6, s[4:5], v4, v0 +; GCN-NEXT: v_sub_i32_e64 v6, s[4:5], v2, v0 ; GCN-NEXT: v_subbrev_u32_e64 v5, s[4:5], 0, v5, s[4:5] ; GCN-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v1 ; GCN-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5] @@ -1180,22 +1134,22 @@ ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5] ; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], v5, v1 ; GCN-NEXT: v_cndmask_b32_e64 v5, v7, v6, s[4:5] -; GCN-NEXT: v_add_i32_e64 v6, s[4:5], 2, v2 +; GCN-NEXT: v_add_i32_e64 v6, s[4:5], 2, v4 ; GCN-NEXT: v_addc_u32_e64 v7, s[4:5], 0, v12, s[4:5] -; GCN-NEXT: v_add_i32_e64 v8, s[4:5], 1, v2 +; GCN-NEXT: v_add_i32_e64 v8, s[4:5], 1, v4 ; GCN-NEXT: v_addc_u32_e64 v9, s[4:5], 0, v12, s[4:5] ; GCN-NEXT: v_subb_u32_e32 v3, vcc, 0, v3, vcc ; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v5 ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v1 ; GCN-NEXT: v_cndmask_b32_e64 v5, v8, v6, s[4:5] ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc -; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v4, v0 +; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0 ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc ; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1 ; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc ; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 ; GCN-NEXT: v_cndmask_b32_e64 v1, v9, v7, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc +; GCN-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc ; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc ; GCN-NEXT: s_setpc_b64 s[30:31] ; @@ -1387,97 +1341,85 @@ ; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GCN-NEXT: v_trunc_f32_e32 v1, v1 ; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 ; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: v_mul_hi_u32 v2, v0, s4 -; GCN-NEXT: v_mul_lo_u32 v3, v1, s4 -; GCN-NEXT: v_mul_lo_u32 v4, v0, s4 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_mul_lo_u32 v5, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v6, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v3, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v9, v1, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v1, v2 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GCN-NEXT: v_mul_lo_u32 v6, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v3, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GCN-NEXT: v_mul_hi_u32 v2, v0, s4 -; GCN-NEXT: v_mul_lo_u32 v3, v1, s4 -; GCN-NEXT: v_mul_lo_u32 v4, v0, s4 -; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GCN-NEXT: v_mul_lo_u32 v3, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v6, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v9, v1, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v1, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v5, v3 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 ; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v5, s4 +; GCN-NEXT: v_subrev_i32_e32 v1, vcc, v4, v1 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v3, v4, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v6, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v3, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v10, vcc, v8, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v5, v6, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v9, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v2, s2, v1 -; GCN-NEXT: v_mul_hi_u32 v3, s2, v0 -; GCN-NEXT: v_mul_hi_u32 v4, s2, v1 -; GCN-NEXT: v_mul_hi_u32 v5, s3, v1 -; GCN-NEXT: v_mul_lo_u32 v1, s3, v1 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s3, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v2, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v1, 24 -; GCN-NEXT: v_mul_hi_u32 v5, v0, 24 -; GCN-NEXT: v_add_i32_e32 v2, vcc, 2, v0 -; GCN-NEXT: v_mul_lo_u32 v8, v0, 24 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, 1, v0 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mov_b32_e32 v5, s3 -; GCN-NEXT: v_sub_i32_e32 v8, vcc, s2, v8 -; GCN-NEXT: v_subb_u32_e32 v4, vcc, v5, v4, vcc -; GCN-NEXT: v_subrev_i32_e32 v5, vcc, 24, v8 -; GCN-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v4, vcc -; GCN-NEXT: v_cmp_lt_u32_e32 vcc, 23, v5 -; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v5, s4 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GCN-NEXT: v_subrev_i32_e32 v1, vcc, v4, v1 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v2 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v4, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v6, v4, v0 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v4, s2, v2 +; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s3, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s3, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v6, v3, 24 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 24, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, 2, v2 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc +; GCN-NEXT: v_add_i32_e32 v7, vcc, 1, v2 +; GCN-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v6 +; GCN-NEXT: v_mov_b32_e32 v6, s3 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v6, v1, vcc +; GCN-NEXT: v_subrev_i32_e32 v6, vcc, 24, v0 +; GCN-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v1, vcc +; GCN-NEXT: v_cmp_lt_u32_e32 vcc, 23, v6 +; GCN-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc ; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 -; GCN-NEXT: v_cndmask_b32_e32 v5, -1, v5, vcc -; GCN-NEXT: v_cmp_lt_u32_e64 s[0:1], 23, v8 -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 -; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] -; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4 -; GCN-NEXT: v_cndmask_b32_e64 v4, -1, v5, s[0:1] -; GCN-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4 -; GCN-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GCN-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1] -; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1] +; GCN-NEXT: v_cmp_lt_u32_e64 s[0:1], 23, v0 +; GCN-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1] +; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1 +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 +; GCN-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[0:1] +; GCN-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v7, v4, vcc +; GCN-NEXT: v_cndmask_b32_e64 v1, v3, v5, s[0:1] +; GCN-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[0:1] ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm ; @@ -1566,97 +1508,85 @@ ; GCN-NEXT: v_mov_b32_e32 v2, 0x4f800000 ; GCN-NEXT: v_madak_f32 v2, 0, v2, 0x41c00000 ; GCN-NEXT: v_rcp_f32_e32 v2, v2 -; GCN-NEXT: s_movk_i32 s4, 0xffe8 +; GCN-NEXT: s_movk_i32 s6, 0xffe8 ; GCN-NEXT: v_mov_b32_e32 v10, 0 ; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 ; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 ; GCN-NEXT: v_trunc_f32_e32 v3, v3 ; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_mul_hi_u32 v4, v2, s4 -; GCN-NEXT: v_mul_lo_u32 v5, v3, s4 -; GCN-NEXT: v_mul_lo_u32 v6, v2, s4 -; GCN-NEXT: v_subrev_i32_e32 v4, vcc, v2, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v7, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v5, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v5, vcc -; GCN-NEXT: v_add_i32_e32 v7, vcc, v7, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; GCN-NEXT: v_cvt_u32_f32_e32 v6, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v7, s6 +; GCN-NEXT: v_subrev_i32_e32 v3, vcc, v6, v3 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v4, v3 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v8, v6, v2 +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_hi_u32 v4, v2, s4 -; GCN-NEXT: v_mul_lo_u32 v5, v3, s4 -; GCN-NEXT: v_mul_lo_u32 v6, v2, s4 -; GCN-NEXT: v_subrev_i32_e32 v4, vcc, v2, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v8, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v11, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v7, v5 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v10, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v7, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0 +; GCN-NEXT: v_mul_lo_u32 v4, v7, s6 +; GCN-NEXT: v_subrev_i32_e32 v3, vcc, v6, v3 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v3, v4 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v8, v6, v2 +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v11, vcc, v10, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v5, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v11, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v6, v0, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v1, v3 -; GCN-NEXT: v_mul_lo_u32 v3, v1, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v1, v2 -; GCN-NEXT: v_mul_hi_u32 v2, v1, v2 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v2, vcc -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v9, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v10, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, 24 -; GCN-NEXT: v_mul_hi_u32 v5, v2, 24 -; GCN-NEXT: v_mul_lo_u32 v6, v2, 24 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 -; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc -; GCN-NEXT: v_subrev_i32_e32 v4, vcc, 24, v0 -; GCN-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v1, vcc -; GCN-NEXT: v_cmp_lt_u32_e32 vcc, 23, v4 -; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 -; GCN-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, 2, v2 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, 0, v3, vcc -; GCN-NEXT: v_add_i32_e32 v7, vcc, 1, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v10, v3, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v6, v0, v4 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v10, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v1, v4, 0 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v5, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v6, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v9, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v2, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v10, v3, vcc +; GCN-NEXT: v_mul_lo_u32 v6, v5, 24 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, 24, 0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc +; GCN-NEXT: v_subrev_i32_e32 v2, vcc, 24, v0 +; GCN-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v1, vcc +; GCN-NEXT: v_cmp_lt_u32_e32 vcc, 23, v2 +; GCN-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 +; GCN-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, 2, v4 +; GCN-NEXT: v_addc_u32_e32 v6, vcc, 0, v5, vcc +; GCN-NEXT: v_add_i32_e32 v7, vcc, 1, v4 ; GCN-NEXT: v_cmp_lt_u32_e64 s[4:5], 23, v0 -; GCN-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v8, vcc, 0, v5, vcc ; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5] ; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1 -; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 ; GCN-NEXT: v_cndmask_b32_e64 v0, -1, v0, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v4, v7, v5, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc ; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0 ; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc -; GCN-NEXT: v_cndmask_b32_e64 v0, v2, v4, s[4:5] -; GCN-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v0, v4, v2, s[4:5] +; GCN-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[4:5] ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GCN-IR-LABEL: v_test_udiv_k_den_i64: diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll --- a/llvm/test/CodeGen/AMDGPU/urem64.ll +++ b/llvm/test/CodeGen/AMDGPU/urem64.ll @@ -6,93 +6,81 @@ ; GCN-LABEL: s_test_urem_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0xd -; GCN-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x9 -; GCN-NEXT: v_mov_b32_e32 v2, 0 -; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 +; GCN-NEXT: v_mov_b32_e32 v7, 0 +; GCN-NEXT: v_mov_b32_e32 v6, 0 +; GCN-NEXT: s_mov_b32 s11, 0xf000 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s12 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s13 -; GCN-NEXT: s_sub_u32 s0, 0, s12 -; GCN-NEXT: s_subb_u32 s1, 0, s13 -; GCN-NEXT: s_mov_b32 s4, s8 +; GCN-NEXT: s_sub_u32 s2, 0, s12 +; GCN-NEXT: s_subb_u32 s3, 0, s13 +; GCN-NEXT: s_mov_b32 s10, -1 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: s_mov_b32 s5, s9 +; GCN-NEXT: s_mov_b32 s8, s4 +; GCN-NEXT: s_mov_b32 s9, s5 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v7, s1, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s0, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v2, v9, vcc -; GCN-NEXT: v_mul_hi_u32 v9, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v6, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v9, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s1, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, s0, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v2, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s10, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s10, v0 -; GCN-NEXT: v_mul_hi_u32 v6, s10, v3 -; GCN-NEXT: v_mul_hi_u32 v7, s11, v3 -; GCN-NEXT: v_mul_lo_u32 v3, s11, v3 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, s11, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s11, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v0, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v1, s12, v1 -; GCN-NEXT: v_mul_hi_u32 v2, s12, v0 -; GCN-NEXT: v_mul_lo_u32 v3, s13, v0 -; GCN-NEXT: v_mul_lo_u32 v0, s12, v0 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s2, v5 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s3, v8 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v9, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[0:1], v8, v10, 0 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v5, v0, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v5, v10, 0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, v9, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s2, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s3, v8 +; GCN-NEXT: v_mul_hi_u32 v10, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v8, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v4, s6, v2 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s7, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s7, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v6, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v3, s12, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s12, v2, 0 +; GCN-NEXT: v_mul_lo_u32 v2, s13, v2 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, s11, v1 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, s7, v1 ; GCN-NEXT: v_mov_b32_e32 v3, s13 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s10, v0 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 ; GCN-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v3, vcc ; GCN-NEXT: v_subrev_i32_e64 v4, s[0:1], s12, v0 ; GCN-NEXT: v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1] @@ -107,7 +95,7 @@ ; GCN-NEXT: v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1] ; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v6 ; GCN-NEXT: v_cndmask_b32_e64 v2, v5, v2, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v5, s11 +; GCN-NEXT: v_mov_b32_e32 v5, s7 ; GCN-NEXT: v_subb_u32_e32 v1, vcc, v5, v1, vcc ; GCN-NEXT: v_cmp_le_u32_e32 vcc, s13, v1 ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc @@ -119,7 +107,7 @@ ; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc ; GCN-NEXT: v_cndmask_b32_e64 v2, v4, v3, s[0:1] ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_urem_i64: @@ -204,19 +192,18 @@ ; GCN-IR-NEXT: v_mov_b32_e32 v0, s6 ; GCN-IR-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[14:15] ; GCN-IR-NEXT: BB0_6: ; %udiv-end -; GCN-IR-NEXT: v_mul_lo_u32 v1, s0, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v2, s0, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v3, s1, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v0, s0, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v3, s0, v1 +; GCN-IR-NEXT: v_mad_u64_u32 v[1:2], s[2:3], s0, v0, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, s1, v0 ; GCN-IR-NEXT: s_mov_b32 s11, 0xf000 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-IR-NEXT: v_mov_b32_e32 v2, s7 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-IR-NEXT: v_mov_b32_e32 v3, s7 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s6, v1 ; GCN-IR-NEXT: s_mov_b32 s10, -1 ; GCN-IR-NEXT: s_mov_b32 s8, s4 ; GCN-IR-NEXT: s_mov_b32 s9, s5 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GCN-IR-NEXT: s_endpgm %result = urem i64 %x, %y @@ -230,8 +217,8 @@ ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v4, v2 ; GCN-NEXT: v_cvt_f32_u32_e32 v5, v3 -; GCN-NEXT: v_sub_i32_e32 v6, vcc, 0, v2 -; GCN-NEXT: v_subb_u32_e32 v7, vcc, 0, v3, vcc +; GCN-NEXT: v_sub_i32_e32 v10, vcc, 0, v2 +; GCN-NEXT: v_subb_u32_e32 v11, vcc, 0, v3, vcc ; GCN-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5 ; GCN-NEXT: v_rcp_f32_e32 v4, v4 ; GCN-NEXT: v_mov_b32_e32 v14, 0 @@ -240,72 +227,60 @@ ; GCN-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 ; GCN-NEXT: v_trunc_f32_e32 v5, v5 ; GCN-NEXT: v_mac_f32_e32 v4, 0xcf800000, v5 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GCN-NEXT: v_cvt_u32_f32_e32 v4, v4 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v5 -; GCN-NEXT: v_mul_hi_u32 v8, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v10, v7, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v9 -; GCN-NEXT: v_mul_lo_u32 v9, v6, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GCN-NEXT: v_mul_lo_u32 v11, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v12, v4, v9 -; GCN-NEXT: v_mul_hi_u32 v10, v4, v8 -; GCN-NEXT: v_mul_hi_u32 v15, v5, v8 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v8 -; GCN-NEXT: v_add_i32_e32 v11, vcc, v12, v11 -; GCN-NEXT: v_mul_lo_u32 v12, v5, v9 -; GCN-NEXT: v_mul_hi_u32 v9, v5, v9 -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v14, v10, vcc -; GCN-NEXT: v_add_i32_e32 v11, vcc, v11, v12 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v10, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v10, vcc, v15, v13, vcc -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v10, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v8 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v6, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v7, v7, v4 -; GCN-NEXT: v_mul_lo_u32 v6, v6, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_add_i32_e32 v7, vcc, v8, v7 -; GCN-NEXT: v_mul_lo_u32 v10, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v11, v4, v6 -; GCN-NEXT: v_mul_hi_u32 v12, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v9, v5, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v5, v6 -; GCN-NEXT: v_mul_hi_u32 v8, v5, v7 -; GCN-NEXT: v_add_i32_e32 v10, vcc, v11, v10 -; GCN-NEXT: v_addc_u32_e32 v11, vcc, v14, v12, vcc -; GCN-NEXT: v_mul_lo_u32 v7, v5, v7 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v10, v6 -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v11, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v8, v13, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v14, v8, vcc +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v5 +; GCN-NEXT: v_cvt_u32_f32_e32 v9, v4 +; GCN-NEXT: v_mul_lo_u32 v6, v10, v8 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v9, 0 +; GCN-NEXT: v_mul_lo_u32 v7, v11, v9 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 +; GCN-NEXT: v_add_i32_e32 v7, vcc, v5, v7 +; GCN-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v7, 0 +; GCN-NEXT: v_mul_hi_u32 v12, v9, v4 +; GCN-NEXT: v_add_i32_e32 v12, vcc, v12, v5 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v4, 0 +; GCN-NEXT: v_addc_u32_e32 v15, vcc, v14, v6, vcc +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v7, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v12, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v15, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v13, vcc ; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v6 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v7, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v1, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v1, v5 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v14, v8, vcc -; GCN-NEXT: v_mul_lo_u32 v8, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v8 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v9, v13, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v14, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v2, v4 -; GCN-NEXT: v_mul_lo_u32 v7, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v2, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v6, v5 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v14, v5, vcc +; GCN-NEXT: v_add_i32_e32 v12, vcc, v9, v4 +; GCN-NEXT: v_addc_u32_e32 v15, vcc, v8, v5, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v12, 0 +; GCN-NEXT: v_mul_lo_u32 v8, v10, v15 +; GCN-NEXT: v_mul_lo_u32 v9, v11, v12 +; GCN-NEXT: v_mul_hi_u32 v10, v12, v4 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v4, 0 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v8 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v9 +; GCN-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v12, v5, 0 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v15, v5, 0 +; GCN-NEXT: v_add_i32_e32 v8, vcc, v10, v8 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v9, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v8, v6 +; GCN-NEXT: v_addc_u32_e32 v6, vcc, v9, v7, vcc +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v13, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v14, v5, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v12, v4 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v15, v5, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v7, 0 +; GCN-NEXT: v_mul_hi_u32 v8, v0, v6 +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v4 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v14, v5, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v6, 0 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v7, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v13, vcc +; GCN-NEXT: v_add_i32_e32 v6, vcc, v4, v6 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v14, v5, vcc +; GCN-NEXT: v_mul_lo_u32 v7, v2, v4 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0 +; GCN-NEXT: v_mul_lo_u32 v6, v3, v6 ; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 ; GCN-NEXT: v_sub_i32_e32 v6, vcc, v1, v5 ; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 ; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v6, v3, vcc @@ -349,24 +324,24 @@ ; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 32, v4 ; GCN-IR-NEXT: v_ffbh_u32_e32 v5, v1 ; GCN-IR-NEXT: v_min_u32_e32 v10, v4, v5 -; GCN-IR-NEXT: v_sub_i32_e32 v5, vcc, v8, v10 -; GCN-IR-NEXT: v_subb_u32_e64 v6, s[6:7], 0, 0, vcc -; GCN-IR-NEXT: v_cmp_lt_u64_e32 vcc, 63, v[5:6] +; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v8, v10 +; GCN-IR-NEXT: v_subb_u32_e64 v5, s[6:7], 0, 0, vcc +; GCN-IR-NEXT: v_cmp_lt_u64_e32 vcc, 63, v[4:5] ; GCN-IR-NEXT: v_mov_b32_e32 v9, 0 ; GCN-IR-NEXT: s_or_b64 s[4:5], s[4:5], vcc -; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[5:6] +; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[4:5] ; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1 ; GCN-IR-NEXT: v_mov_b32_e32 v11, v9 ; GCN-IR-NEXT: v_cndmask_b32_e64 v7, v1, 0, s[4:5] -; GCN-IR-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5] +; GCN-IR-NEXT: v_cndmask_b32_e64 v6, v0, 0, s[4:5] ; GCN-IR-NEXT: s_and_b64 s[4:5], s[6:7], vcc ; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] ; GCN-IR-NEXT: s_cbranch_execz BB1_6 ; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1 -; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 1, v5 -; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, 0, v6, vcc -; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], 63, v5 -; GCN-IR-NEXT: v_cmp_ge_u64_e32 vcc, v[12:13], v[5:6] +; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 1, v4 +; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, 0, v5, vcc +; GCN-IR-NEXT: v_cmp_ge_u64_e32 vcc, v[12:13], v[4:5] +; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], 63, v4 ; GCN-IR-NEXT: v_lshl_b64 v[4:5], v[0:1], v4 ; GCN-IR-NEXT: v_mov_b32_e32 v6, 0 ; GCN-IR-NEXT: v_mov_b32_e32 v7, 0 @@ -417,17 +392,16 @@ ; GCN-IR-NEXT: s_or_b64 exec, exec, s[8:9] ; GCN-IR-NEXT: v_lshl_b64 v[4:5], v[4:5], 1 ; GCN-IR-NEXT: v_or_b32_e32 v7, v7, v5 -; GCN-IR-NEXT: v_or_b32_e32 v4, v6, v4 +; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4 ; GCN-IR-NEXT: BB1_6: ; %Flow4 ; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] -; GCN-IR-NEXT: v_mul_lo_u32 v5, v2, v7 -; GCN-IR-NEXT: v_mul_hi_u32 v6, v2, v4 -; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v4 -; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, v4 -; GCN-IR-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, v5, v3 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc +; GCN-IR-NEXT: v_mul_lo_u32 v7, v2, v7 +; GCN-IR-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v3, v6 +; GCN-IR-NEXT: v_add_i32_e32 v3, vcc, v5, v7 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v3, v2 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %result = urem i64 %x, %y ret i64 %result @@ -737,79 +711,69 @@ ; GCN-LABEL: s_test_urem_k_num_i64: ; GCN: ; %bb.0: ; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 -; GCN-NEXT: v_mov_b32_e32 v2, 0 +; GCN-NEXT: v_mov_b32_e32 v7, 0 +; GCN-NEXT: v_mov_b32_e32 v6, 0 ; GCN-NEXT: s_mov_b32 s11, 0xf000 ; GCN-NEXT: s_mov_b32 s10, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s6 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s7 -; GCN-NEXT: s_sub_u32 s0, 0, s6 -; GCN-NEXT: s_subb_u32 s1, 0, s7 +; GCN-NEXT: s_sub_u32 s2, 0, s6 +; GCN-NEXT: s_subb_u32 s3, 0, s7 ; GCN-NEXT: s_mov_b32 s8, s4 ; GCN-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_mov_b32 s9, s5 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v0 -; GCN-NEXT: v_trunc_f32_e32 v3, v3 -; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v7, s1, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s0, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v7 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v6 -; GCN-NEXT: v_mul_lo_u32 v7, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v8, v3, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v3, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v7 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v2, v9, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v8, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v10, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v2, v6, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s0, v3 -; GCN-NEXT: v_mul_hi_u32 v5, s0, v0 -; GCN-NEXT: v_mul_lo_u32 v6, s1, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_mul_lo_u32 v5, s0, v0 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 -; GCN-NEXT: v_mul_lo_u32 v8, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v9, v0, v5 -; GCN-NEXT: v_mul_hi_u32 v10, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v5 -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v4 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v2, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v8, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v6, v1, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v5, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v4 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v3, v1, 24 -; GCN-NEXT: v_mul_hi_u32 v0, v0, 24 -; GCN-NEXT: v_mul_hi_u32 v1, v1, 24 -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v3 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v2, v1, vcc -; GCN-NEXT: v_mul_lo_u32 v1, s7, v0 -; GCN-NEXT: v_mul_hi_u32 v2, s6, v0 -; GCN-NEXT: v_mul_lo_u32 v0, s6, v0 -; GCN-NEXT: v_mov_b32_e32 v3, s7 +; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 +; GCN-NEXT: v_trunc_f32_e32 v1, v1 +; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 +; GCN-NEXT: v_cvt_u32_f32_e32 v8, v0 +; GCN-NEXT: v_mul_lo_u32 v2, s2, v5 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v3, s3, v8 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v1, v3 +; GCN-NEXT: v_mul_hi_u32 v9, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[0:1], v8, v10, 0 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[0:1], v5, v0, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v9, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v5, v10, 0 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v7, v2, vcc +; GCN-NEXT: v_add_i32_e32 v3, vcc, v9, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v8, 0 +; GCN-NEXT: v_mul_lo_u32 v4, s2, v9 +; GCN-NEXT: v_mul_lo_u32 v5, s3, v8 +; GCN-NEXT: v_mul_hi_u32 v10, v8, v0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v9, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v4 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v5, v1 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v8, v1, 0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v9, v1, 0 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v4 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v7, v5, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v7, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v8, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 24, 0 +; GCN-NEXT: v_mul_hi_u32 v2, v2, 24 +; GCN-NEXT: v_mov_b32_e32 v3, s7 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v2, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v7, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v2, s7, v0 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v0, 0 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0, v1 ; GCN-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 ; GCN-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v3, vcc @@ -910,18 +874,17 @@ ; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 ; GCN-IR-NEXT: v_cndmask_b32_e64 v0, 24, 0, s[10:11] ; GCN-IR-NEXT: BB6_6: ; %udiv-end -; GCN-IR-NEXT: v_mul_lo_u32 v1, s2, v1 -; GCN-IR-NEXT: v_mul_hi_u32 v2, s2, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v3, s3, v0 -; GCN-IR-NEXT: v_mul_lo_u32 v0, s2, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v3, s2, v1 +; GCN-IR-NEXT: v_mad_u64_u32 v[1:2], s[8:9], s2, v0, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, s3, v0 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v0 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v3 +; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 24, v1 ; GCN-IR-NEXT: s_mov_b32 s6, -1 ; GCN-IR-NEXT: s_mov_b32 s4, s0 ; GCN-IR-NEXT: s_mov_b32 s5, s1 -; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc +; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v2, vcc ; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-IR-NEXT: s_endpgm %result = urem i64 24, %x @@ -942,74 +905,62 @@ ; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GCN-NEXT: v_trunc_f32_e32 v1, v1 ; GCN-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1 -; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v0 +; GCN-NEXT: v_cvt_u32_f32_e32 v5, v1 ; GCN-NEXT: s_mov_b32 s7, 0xf000 -; GCN-NEXT: v_mul_hi_u32 v2, v0, s4 -; GCN-NEXT: v_mul_lo_u32 v3, v1, s4 -; GCN-NEXT: v_mul_lo_u32 v4, v0, s4 +; GCN-NEXT: s_mov_b32 s6, -1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v5, s4 +; GCN-NEXT: v_subrev_i32_e32 v1, vcc, v4, v1 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v2, v1 +; GCN-NEXT: v_mul_hi_u32 v3, v4, v0 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v6, 0 +; GCN-NEXT: v_add_i32_e32 v9, vcc, v3, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v10, vcc, v8, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v5, v6, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v9, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v10, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v5, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v4, s4, 0 +; GCN-NEXT: v_mul_lo_u32 v2, v5, s4 +; GCN-NEXT: v_subrev_i32_e32 v1, vcc, v4, v1 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v1, v2 +; GCN-NEXT: v_mad_u64_u32 v[1:2], s[2:3], v4, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v6, v4, v0 +; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v1 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v0, 0 +; GCN-NEXT: v_addc_u32_e32 v9, vcc, v8, v2, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v9, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc +; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v1, vcc ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_mul_lo_u32 v5, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v6, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v3, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v9, v1, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v1, v2 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GCN-NEXT: v_mul_lo_u32 v6, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v3, vcc -; GCN-NEXT: v_add_i32_e32 v5, vcc, v5, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v3, 0 +; GCN-NEXT: v_mul_hi_u32 v4, s2, v2 +; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v1, vcc +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s3, v2, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s3, v3, 0 +; GCN-NEXT: v_add_i32_e32 v0, vcc, v4, v0 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v1, vcc +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v3, v7, vcc ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GCN-NEXT: v_mul_hi_u32 v2, v0, s4 -; GCN-NEXT: v_mul_lo_u32 v3, v1, s4 -; GCN-NEXT: v_mul_lo_u32 v4, v0, s4 +; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v1, vcc +; GCN-NEXT: v_mul_lo_u32 v2, v1, 24 +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, 24, 0 ; GCN-NEXT: s_mov_b32 s4, s0 -; GCN-NEXT: v_subrev_i32_e32 v2, vcc, v0, v2 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GCN-NEXT: v_mul_lo_u32 v3, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v5, v0, v4 -; GCN-NEXT: v_mul_hi_u32 v6, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v9, v1, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v1, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v5, v3 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v8, v6, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v1, v4 -; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 -; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v4, vcc -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v2, s2, v1 -; GCN-NEXT: v_mul_hi_u32 v3, s2, v0 -; GCN-NEXT: v_mul_hi_u32 v4, s2, v1 -; GCN-NEXT: v_mul_hi_u32 v5, s3, v1 -; GCN-NEXT: v_mul_lo_u32 v1, s3, v1 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v8, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s3, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s3, v0 -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v5, v7, vcc -; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GCN-NEXT: v_addc_u32_e32 v1, vcc, v8, v2, vcc -; GCN-NEXT: v_mul_lo_u32 v1, v1, 24 -; GCN-NEXT: v_mul_hi_u32 v2, v0, 24 -; GCN-NEXT: v_mul_lo_u32 v0, v0, 24 -; GCN-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-NEXT: s_mov_b32 s5, s1 +; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN-NEXT: v_mov_b32_e32 v2, s3 ; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 ; GCN-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc @@ -1103,12 +1054,11 @@ ; GCN-IR-NEXT: v_mov_b32_e32 v0, s2 ; GCN-IR-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[10:11] ; GCN-IR-NEXT: BB7_6: ; %udiv-end -; GCN-IR-NEXT: v_mul_lo_u32 v1, v1, 24 -; GCN-IR-NEXT: v_mul_hi_u32 v2, v0, 24 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, 24 +; GCN-IR-NEXT: v_mul_lo_u32 v2, v1, 24 +; GCN-IR-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, 24, 0 ; GCN-IR-NEXT: s_mov_b32 s7, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s6, -1 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN-IR-NEXT: v_mov_b32_e32 v2, s3 ; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s2, v0 ; GCN-IR-NEXT: s_mov_b32 s4, s0 @@ -1133,69 +1083,64 @@ ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ; GCN-NEXT: v_cvt_f32_u32_e32 v3, v1 -; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v0 -; GCN-NEXT: v_subb_u32_e32 v5, vcc, 0, v1, vcc +; GCN-NEXT: v_sub_i32_e32 v8, vcc, 0, v0 +; GCN-NEXT: v_subb_u32_e32 v9, vcc, 0, v1, vcc ; GCN-NEXT: v_mac_f32_e32 v2, 0x4f800000, v3 ; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_mov_b32_e32 v12, 0 ; GCN-NEXT: v_mov_b32_e32 v11, 0 +; GCN-NEXT: s_mov_b32 s6, 0x8000 ; GCN-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2 ; GCN-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2 ; GCN-NEXT: v_trunc_f32_e32 v3, v3 ; GCN-NEXT: v_mac_f32_e32 v2, 0xcf800000, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GCN-NEXT: v_mul_lo_u32 v6, v4, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v8, v5, v2 -; GCN-NEXT: v_mul_lo_u32 v9, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v6, v8 -; GCN-NEXT: v_mul_lo_u32 v8, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v9 -; GCN-NEXT: v_mul_hi_u32 v7, v2, v6 -; GCN-NEXT: v_mul_hi_u32 v13, v3, v6 -; GCN-NEXT: v_mul_lo_u32 v6, v3, v6 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v10, v8 -; GCN-NEXT: v_mul_lo_u32 v10, v3, v9 -; GCN-NEXT: v_mul_hi_u32 v9, v3, v9 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v7, vcc -; GCN-NEXT: v_add_i32_e32 v8, vcc, v8, v10 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v7, v9, vcc -; GCN-NEXT: v_addc_u32_e32 v8, vcc, v13, v11, vcc -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v8, vcc -; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v6 -; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc -; GCN-NEXT: v_mul_lo_u32 v6, v4, v3 -; GCN-NEXT: v_mul_hi_u32 v7, v4, v2 -; GCN-NEXT: v_mul_lo_u32 v5, v5, v2 -; GCN-NEXT: v_mul_lo_u32 v4, v4, v2 -; GCN-NEXT: v_add_i32_e32 v6, vcc, v7, v6 -; GCN-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GCN-NEXT: v_mul_lo_u32 v8, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v9, v2, v4 -; GCN-NEXT: v_mul_hi_u32 v10, v2, v5 -; GCN-NEXT: v_mul_hi_u32 v7, v3, v4 -; GCN-NEXT: v_mul_lo_u32 v4, v3, v4 -; GCN-NEXT: v_mul_hi_u32 v6, v3, v5 -; GCN-NEXT: v_add_i32_e32 v8, vcc, v9, v8 -; GCN-NEXT: v_addc_u32_e32 v9, vcc, v12, v10, vcc -; GCN-NEXT: v_mul_lo_u32 v5, v3, v5 -; GCN-NEXT: v_add_i32_e32 v4, vcc, v8, v4 -; GCN-NEXT: v_addc_u32_e32 v4, vcc, v9, v7, vcc -; GCN-NEXT: v_addc_u32_e32 v6, vcc, v6, v11, vcc -; GCN-NEXT: v_add_i32_e32 v4, vcc, v4, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, v12, v6, vcc +; GCN-NEXT: v_cvt_u32_f32_e32 v6, v3 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v2 +; GCN-NEXT: v_mul_lo_u32 v4, v8, v6 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v7, 0 +; GCN-NEXT: v_mul_lo_u32 v5, v9, v7 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 +; GCN-NEXT: v_add_i32_e32 v5, vcc, v3, v5 +; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v5, 0 +; GCN-NEXT: v_mul_hi_u32 v10, v7, v2 +; GCN-NEXT: v_add_i32_e32 v10, vcc, v10, v3 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v2, 0 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v12, v4, vcc +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v5, 0 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v10, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v13, v3, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v11, vcc ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, v3, v5, vcc -; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 -; GCN-NEXT: v_mul_lo_u32 v3, v1, v2 -; GCN-NEXT: v_mul_hi_u32 v4, v0, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v0, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v4, v3 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v12, v3, vcc +; GCN-NEXT: v_add_i32_e32 v10, vcc, v7, v2 +; GCN-NEXT: v_addc_u32_e32 v13, vcc, v6, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v10, 0 +; GCN-NEXT: v_mul_lo_u32 v6, v8, v13 +; GCN-NEXT: v_mul_lo_u32 v7, v9, v10 +; GCN-NEXT: v_mul_hi_u32 v8, v10, v2 +; GCN-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v2, 0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v7 +; GCN-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v10, v3, 0 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v3, 0 +; GCN-NEXT: v_add_i32_e32 v6, vcc, v8, v6 +; GCN-NEXT: v_addc_u32_e32 v7, vcc, v12, v7, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v6, v4 +; GCN-NEXT: v_addc_u32_e32 v4, vcc, v7, v5, vcc +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v3, v11, vcc +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v3, vcc, v12, v3, vcc +; GCN-NEXT: v_add_i32_e32 v4, vcc, v10, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v13, v3, vcc +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, s6, 0 +; GCN-NEXT: v_lshrrev_b32_e32 v4, 17, v4 +; GCN-NEXT: v_add_i32_e32 v2, vcc, v4, v2 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, v12, v3, vcc +; GCN-NEXT: v_mul_lo_u32 v4, v1, v2 +; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v2, 0 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v4 ; GCN-NEXT: v_sub_i32_e32 v4, vcc, 0, v3 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, 0x8000, v2 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, s6, v2 ; GCN-NEXT: v_subb_u32_e64 v4, s[4:5], v4, v1, vcc ; GCN-NEXT: v_sub_i32_e64 v5, s[4:5], v2, v0 ; GCN-NEXT: v_subbrev_u32_e64 v6, s[6:7], 0, v4, s[4:5] @@ -1302,13 +1247,12 @@ ; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v2 ; GCN-IR-NEXT: BB8_6: ; %Flow4 ; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7] -; GCN-IR-NEXT: v_mul_lo_u32 v2, v0, v5 -; GCN-IR-NEXT: v_mul_hi_u32 v3, v0, v4 -; GCN-IR-NEXT: v_mul_lo_u32 v1, v1, v4 -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, v4 -; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v3, v2 -; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v0 +; GCN-IR-NEXT: v_mul_lo_u32 v5, v0, v5 +; GCN-IR-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v0, v4, 0 +; GCN-IR-NEXT: v_mul_lo_u32 v0, v1, v4 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v3, v5 +; GCN-IR-NEXT: v_add_i32_e32 v1, vcc, v1, v0 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, 0x8000, v2 ; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc ; GCN-IR-NEXT: s_setpc_b64 s[30:31] %result = urem i64 32768, %x diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll @@ -435,39 +435,40 @@ ; GFX9-O0-LABEL: strict_wwm_called_i64: ; GFX9-O0: ; %bb.0: ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-O0-NEXT: v_mov_b32_e32 v6, v0 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v7 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: v_add_co_u32_e64 v4, s[34:35], v2, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr2_vgpr3 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v3 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-O0-NEXT: v_add_co_u32_e64 v4, s[34:35], v4, v5 ; GFX9-O0-NEXT: v_addc_co_u32_e64 v0, s[34:35], v0, v1, s[34:35] ; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v0 ; GFX9-O0-NEXT: s_mov_b32 s34, 32 -; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 -; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2 +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 ; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s34, v[0:1] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v4 -; GFX9-O0-NEXT: v_mul_lo_u32 v2, v0, v1 -; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec -; GFX9-O0-NEXT: v_mul_hi_u32 v1, v0, v6 -; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s34, v[4:5] -; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7 -; GFX9-O0-NEXT: v_mul_lo_u32 v3, v3, v6 -; GFX9-O0-NEXT: v_add3_u32 v1, v1, v2, v3 +; GFX9-O0-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 +; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s34, v[4:5] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v6 +; GFX9-O0-NEXT: v_mul_lo_u32 v2, v2, v3 +; GFX9-O0-NEXT: v_mad_u64_u32 v[6:7], s[36:37], v0, v3, 0 +; GFX9-O0-NEXT: v_mov_b32_e32 v0, v7 +; GFX9-O0-NEXT: v_add3_u32 v0, v0, v1, v2 ; GFX9-O0-NEXT: ; implicit-def: $sgpr35 ; GFX9-O0-NEXT: ; implicit-def: $sgpr36 -; GFX9-O0-NEXT: v_mov_b32_e32 v3, s35 -; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3 -; GFX9-O0-NEXT: v_lshlrev_b64 v[1:2], s34, v[1:2] +; GFX9-O0-NEXT: v_mov_b32_e32 v2, s35 +; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec +; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2 +; GFX9-O0-NEXT: v_lshlrev_b64 v[1:2], s34, v[0:1] ; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-O0-NEXT: v_mul_lo_u32 v6, v0, v6 +; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec ; GFX9-O0-NEXT: s_mov_b32 s35, 0 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec @@ -498,10 +499,9 @@ ; GFX9-O3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v0 ; GFX9-O3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v1, vcc ; GFX9-O3-NEXT: v_mul_lo_u32 v4, v3, v0 -; GFX9-O3-NEXT: v_mul_lo_u32 v1, v2, v1 -; GFX9-O3-NEXT: v_mul_hi_u32 v5, v2, v0 -; GFX9-O3-NEXT: v_mul_lo_u32 v0, v2, v0 -; GFX9-O3-NEXT: v_add3_u32 v1, v5, v1, v4 +; GFX9-O3-NEXT: v_mul_lo_u32 v5, v2, v1 +; GFX9-O3-NEXT: v_mad_u64_u32 v[0:1], s[34:35], v2, v0, 0 +; GFX9-O3-NEXT: v_add3_u32 v1, v1, v5, v4 ; GFX9-O3-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2 ; GFX9-O3-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc ; GFX9-O3-NEXT: s_setpc_b64 s[30:31] @@ -516,10 +516,10 @@ ; GFX9-O0: ; %bb.0: ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1 -; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill -; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) -; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill ; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill ; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_waitcnt vmcnt(0) @@ -531,40 +531,40 @@ ; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill ; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill ; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35] -; GFX9-O0-NEXT: v_writelane_b32 v11, s33, 8 +; GFX9-O0-NEXT: v_writelane_b32 v10, s33, 8 ; GFX9-O0-NEXT: s_mov_b32 s33, s32 ; GFX9-O0-NEXT: s_add_i32 s32, s32, 0xc00 -; GFX9-O0-NEXT: v_writelane_b32 v11, s30, 0 -; GFX9-O0-NEXT: v_writelane_b32 v11, s31, 1 +; GFX9-O0-NEXT: v_writelane_b32 v10, s30, 0 +; GFX9-O0-NEXT: v_writelane_b32 v10, s31, 1 ; GFX9-O0-NEXT: s_mov_b32 s34, s8 ; GFX9-O0-NEXT: s_mov_b32 s36, s4 ; GFX9-O0-NEXT: ; kill: def $sgpr36 killed $sgpr36 def $sgpr36_sgpr37_sgpr38_sgpr39 ; GFX9-O0-NEXT: s_mov_b32 s37, s5 ; GFX9-O0-NEXT: s_mov_b32 s38, s6 ; GFX9-O0-NEXT: s_mov_b32 s39, s7 -; GFX9-O0-NEXT: v_writelane_b32 v11, s36, 2 -; GFX9-O0-NEXT: v_writelane_b32 v11, s37, 3 -; GFX9-O0-NEXT: v_writelane_b32 v11, s38, 4 -; GFX9-O0-NEXT: v_writelane_b32 v11, s39, 5 +; GFX9-O0-NEXT: v_writelane_b32 v10, s36, 2 +; GFX9-O0-NEXT: v_writelane_b32 v10, s37, 3 +; GFX9-O0-NEXT: v_writelane_b32 v10, s38, 4 +; GFX9-O0-NEXT: v_writelane_b32 v10, s39, 5 ; GFX9-O0-NEXT: ; kill: def $sgpr34 killed $sgpr34 def $sgpr34_sgpr35 ; GFX9-O0-NEXT: s_mov_b32 s35, s9 ; GFX9-O0-NEXT: ; kill: def $sgpr30_sgpr31 killed $sgpr34_sgpr35 ; GFX9-O0-NEXT: s_mov_b64 s[30:31], 0 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, s34 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, s35 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v9, v0 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, v1 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, v0 ; GFX9-O0-NEXT: s_not_b64 exec, exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s30 -; GFX9-O0-NEXT: v_mov_b32_e32 v10, s31 +; GFX9-O0-NEXT: v_mov_b32_e32 v8, s30 +; GFX9-O0-NEXT: v_mov_b32_e32 v9, s31 ; GFX9-O0-NEXT: s_not_b64 exec, exec ; GFX9-O0-NEXT: s_or_saveexec_b64 s[30:31], -1 -; GFX9-O0-NEXT: v_writelane_b32 v11, s30, 6 -; GFX9-O0-NEXT: v_writelane_b32 v11, s31, 7 -; GFX9-O0-NEXT: v_mov_b32_e32 v2, v9 +; GFX9-O0-NEXT: v_writelane_b32 v10, s30, 6 +; GFX9-O0-NEXT: v_writelane_b32 v10, s31, 7 +; GFX9-O0-NEXT: v_mov_b32_e32 v2, v8 ; GFX9-O0-NEXT: s_mov_b32 s30, 32 ; GFX9-O0-NEXT: ; implicit-def: $sgpr34_sgpr35 -; GFX9-O0-NEXT: v_lshrrev_b64 v[3:4], s30, v[9:10] +; GFX9-O0-NEXT: v_lshrrev_b64 v[3:4], s30, v[8:9] ; GFX9-O0-NEXT: s_getpc_b64 s[30:31] ; GFX9-O0-NEXT: s_add_u32 s30, s30, strict_wwm_called_i64@gotpcrel32@lo+4 ; GFX9-O0-NEXT: s_addc_u32 s31, s31, strict_wwm_called_i64@gotpcrel32@hi+12 @@ -577,18 +577,18 @@ ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[30:31] -; GFX9-O0-NEXT: v_readlane_b32 s34, v11, 6 -; GFX9-O0-NEXT: v_readlane_b32 s35, v11, 7 -; GFX9-O0-NEXT: v_readlane_b32 s36, v11, 2 -; GFX9-O0-NEXT: v_readlane_b32 s37, v11, 3 -; GFX9-O0-NEXT: v_readlane_b32 s38, v11, 4 -; GFX9-O0-NEXT: v_readlane_b32 s39, v11, 5 -; GFX9-O0-NEXT: v_readlane_b32 s30, v11, 0 -; GFX9-O0-NEXT: v_readlane_b32 s31, v11, 1 +; GFX9-O0-NEXT: v_readlane_b32 s34, v10, 6 +; GFX9-O0-NEXT: v_readlane_b32 s35, v10, 7 +; GFX9-O0-NEXT: v_readlane_b32 s36, v10, 2 +; GFX9-O0-NEXT: v_readlane_b32 s37, v10, 3 +; GFX9-O0-NEXT: v_readlane_b32 s38, v10, 4 +; GFX9-O0-NEXT: v_readlane_b32 s39, v10, 5 +; GFX9-O0-NEXT: v_readlane_b32 s30, v10, 0 +; GFX9-O0-NEXT: v_readlane_b32 s31, v10, 1 ; GFX9-O0-NEXT: v_mov_b32_e32 v2, v0 ; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-O0-NEXT: v_mov_b32_e32 v4, v9 -; GFX9-O0-NEXT: v_mov_b32_e32 v5, v10 +; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8 +; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9 ; GFX9-O0-NEXT: v_add_co_u32_e64 v2, s[40:41], v2, v4 ; GFX9-O0-NEXT: v_addc_co_u32_e64 v3, s[40:41], v3, v5, s[40:41] ; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35] @@ -597,13 +597,13 @@ ; GFX9-O0-NEXT: s_mov_b32 s34, 0 ; GFX9-O0-NEXT: buffer_store_dwordx2 v[0:1], off, s[36:39], s34 offset:4 ; GFX9-O0-NEXT: s_add_i32 s32, s32, 0xfffff400 -; GFX9-O0-NEXT: v_readlane_b32 s33, v11, 8 +; GFX9-O0-NEXT: v_readlane_b32 s33, v10, 8 ; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1 -; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_nop 0 -; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_nop 0 -; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_nop 0 ; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload ; GFX9-O0-NEXT: s_nop 0 @@ -636,7 +636,7 @@ ; GFX9-O3-NEXT: s_waitcnt vmcnt(0) ; GFX9-O3-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill ; GFX9-O3-NEXT: s_mov_b64 exec, s[34:35] -; GFX9-O3-NEXT: s_mov_b32 s38, s33 +; GFX9-O3-NEXT: s_mov_b32 s40, s33 ; GFX9-O3-NEXT: s_mov_b32 s33, s32 ; GFX9-O3-NEXT: s_addk_i32 s32, 0x800 ; GFX9-O3-NEXT: s_mov_b64 s[36:37], s[30:31] @@ -646,7 +646,7 @@ ; GFX9-O3-NEXT: v_mov_b32_e32 v6, 0 ; GFX9-O3-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-O3-NEXT: s_not_b64 exec, exec -; GFX9-O3-NEXT: s_or_saveexec_b64 s[34:35], -1 +; GFX9-O3-NEXT: s_or_saveexec_b64 s[38:39], -1 ; GFX9-O3-NEXT: s_getpc_b64 s[30:31] ; GFX9-O3-NEXT: s_add_u32 s30, s30, strict_wwm_called_i64@gotpcrel32@lo+4 ; GFX9-O3-NEXT: s_addc_u32 s31, s31, strict_wwm_called_i64@gotpcrel32@hi+12 @@ -659,12 +659,12 @@ ; GFX9-O3-NEXT: v_mov_b32_e32 v3, v1 ; GFX9-O3-NEXT: v_add_co_u32_e32 v2, vcc, v2, v6 ; GFX9-O3-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v7, vcc -; GFX9-O3-NEXT: s_mov_b64 exec, s[34:35] +; GFX9-O3-NEXT: s_mov_b64 exec, s[38:39] ; GFX9-O3-NEXT: v_mov_b32_e32 v0, v2 ; GFX9-O3-NEXT: v_mov_b32_e32 v1, v3 ; GFX9-O3-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 offset:4 ; GFX9-O3-NEXT: s_addk_i32 s32, 0xf800 -; GFX9-O3-NEXT: s_mov_b32 s33, s38 +; GFX9-O3-NEXT: s_mov_b32 s33, s40 ; GFX9-O3-NEXT: s_or_saveexec_b64 s[30:31], -1 ; GFX9-O3-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload ; GFX9-O3-NEXT: s_nop 0 diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll @@ -144,15 +144,15 @@ ; GFX9-O0: s_mov_b64 s{{\[}}[[ZERO_LO:[0-9]+]]:[[ZERO_HI:[0-9]+]]{{\]}}, 0{{$}} ; GFX9-O0: v_mov_b32_e32 v0, s[[ARG_LO]] ; GFX9-O0: v_mov_b32_e32 v1, s[[ARG_HI]] -; GFX9-O0-DAG: v_mov_b32_e32 v10, v1 -; GFX9-O0-DAG: v_mov_b32_e32 v9, v0 +; GFX9-O0-DAG: v_mov_b32_e32 v9, v1 +; GFX9-O0-DAG: v_mov_b32_e32 v8, v0 ; GFX9-O3-DAG: v_mov_b32_e32 v7, s[[ARG_HI]] ; GFX9-O3-DAG: v_mov_b32_e32 v6, s[[ARG_LO]] ; GFX9: s_not_b64 exec, exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_LO]] -; GFX9-O0-NEXT: v_mov_b32_e32 v10, s[[ZERO_HI]] +; GFX9-O0-NEXT: v_mov_b32_e32 v8, s[[ZERO_LO]] +; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_HI]] ; GFX9-O3-NEXT: v_mov_b32_e32 v6, 0 ; GFX9-O3-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-NEXT: s_not_b64 exec, exec @@ -338,15 +338,15 @@ ; GFX9-O0: s_mov_b64 s{{\[}}[[ZERO_LO:[0-9]+]]:[[ZERO_HI:[0-9]+]]{{\]}}, 0{{$}} ; GFX9-O0: v_mov_b32_e32 v0, s[[ARG_LO]] ; GFX9-O0: v_mov_b32_e32 v1, s[[ARG_HI]] -; GFX9-O0-DAG: v_mov_b32_e32 v10, v1 -; GFX9-O0-DAG: v_mov_b32_e32 v9, v0 +; GFX9-O0-DAG: v_mov_b32_e32 v9, v1 +; GFX9-O0-DAG: v_mov_b32_e32 v8, v0 ; GFX9-O3-DAG: v_mov_b32_e32 v7, s[[ARG_HI]] ; GFX9-O3-DAG: v_mov_b32_e32 v6, s[[ARG_LO]] ; GFX9: s_not_b64 exec, exec -; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_LO]] -; GFX9-O0-NEXT: v_mov_b32_e32 v10, s[[ZERO_HI]] +; GFX9-O0-NEXT: v_mov_b32_e32 v8, s[[ZERO_LO]] +; GFX9-O0-NEXT: v_mov_b32_e32 v9, s[[ZERO_HI]] ; GFX9-O3-NEXT: v_mov_b32_e32 v6, 0 ; GFX9-O3-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-NEXT: s_not_b64 exec, exec