Index: llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
===================================================================
--- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -3515,6 +3515,16 @@
return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi);
}
+/// If \p V is an add of a constant 1, returns the other operand. Otherwise
+/// return SDValue().
+static SDValue getAddOneOp(const SDNode *V) {
+ if (V->getOpcode() != ISD::ADD)
+ return SDValue();
+
+ auto *C = dyn_cast<ConstantSDNode>(V->getOperand(1));
+ return C && C->isOne() ? V->getOperand(0) : SDValue();
+}
+
SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
EVT VT = N->getValueType(0);
@@ -3530,16 +3540,49 @@
if (VT.isVector() || Size > 64)
return SDValue();
- // There are i16 integer mul/mad.
- if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
- return SDValue();
-
SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
+ // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
+ // matching.
+
+ // mul x, (add y, 1) -> add (mul x, y), x
+ auto IsFoldableAdd = [](SDValue V) -> SDValue {
+ SDValue AddOp = getAddOneOp(V.getNode());
+ if (!AddOp)
+ return SDValue();
+
+ if (V.hasOneUse() || all_of(V->uses(), [](const SDNode *U) -> bool {
+ return U->getOpcode() == ISD::MUL;
+ }))
+ return AddOp;
+
+ return SDValue();
+ };
+
+ // FIXME: The selection pattern is not properly checking for commuted
+ // operands, so we have to place the mul in the LHS
+ if (SDValue MulOper = IsFoldableAdd(N0)) {
+ SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper);
+ return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1);
+ }
+
+ if (SDValue MulOper = IsFoldableAdd(N1)) {
+ SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper);
+ return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0);
+ }
+
+ // Skip if already mul24.
+ if (N->getOpcode() != ISD::MUL)
+ return SDValue();
+
+ // There are i16 integer mul/mad.
+ if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
+ return SDValue();
+
// SimplifyDemandedBits has the annoying habit of turning useful zero_extends
// in the source into any_extends if the result of the mul is truncated. Since
// we can assume the high bits are whatever we want, use the underlying value
@@ -4347,6 +4390,15 @@
return performTruncateCombine(N, DCI);
case ISD::MUL:
return performMulCombine(N, DCI);
+ case AMDGPUISD::MUL_U24:
+ case AMDGPUISD::MUL_I24: {
+ if (SDValue Simplified = simplifyMul24(N, DCI))
+ return Simplified;
+ return performMulCombine(N, DCI);
+ }
+ case AMDGPUISD::MULHI_I24:
+ case AMDGPUISD::MULHI_U24:
+ return simplifyMul24(N, DCI);
case ISD::SMUL_LOHI:
case ISD::UMUL_LOHI:
return performMulLoHiCombine(N, DCI);
@@ -4354,11 +4406,6 @@
return performMulhsCombine(N, DCI);
case ISD::MULHU:
return performMulhuCombine(N, DCI);
- case AMDGPUISD::MUL_I24:
- case AMDGPUISD::MUL_U24:
- case AMDGPUISD::MULHI_I24:
- case AMDGPUISD::MULHI_U24:
- return simplifyMul24(N, DCI);
case ISD::SELECT:
return performSelectCombine(N, DCI);
case ISD::FNEG:
Index: llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
===================================================================
--- llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -14,30 +14,28 @@
; GFX67-LABEL: v_mul_add_1_i32:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i32 %y, 1
%mul = mul i32 %x, %add
@@ -48,30 +46,28 @@
; GFX67-LABEL: v_mul_add_1_i32_commute:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_lo_u32 v0, v1, v0
+; GFX67-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i32_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_lo_u32 v0, v1, v0
+; GFX8-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i32_commute:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v1, v0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i32_commute:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: v_mul_lo_u32 v0, v1, v0
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i32 %y, 1
%mul = mul i32 %add, %x
@@ -352,30 +348,26 @@
; GFX67-LABEL: v_mul_add_1_i24_zext:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i24_zext:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i24_zext:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX9-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i24_zext:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX10-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i24 %y, 1
%mul = mul i24 %x, %add
@@ -450,30 +442,26 @@
; GFX67-LABEL: v_mul_add_1_i24_sext:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i24_sext:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i24_sext:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX9-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i24_sext:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX10-NEXT: v_mad_u32_u24 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i24 %y, 1
%mul = mul i24 %x, %add
@@ -548,30 +536,28 @@
; GFX67-LABEL: v_mul_add_1_i25_zext:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i25_zext:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i25_zext:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i25_zext:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i25 %y, 1
%mul = mul i25 %x, %add
@@ -616,30 +602,28 @@
; GFX67-LABEL: v_mul_add_1_i25_sext:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i25_sext:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i25_sext:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i25_sext:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i25 %y, 1
%mul = mul i25 %x, %add
@@ -693,23 +677,20 @@
; GFX8-LABEL: v_mul_add_1_i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i16 %y, 1
%mul = mul i16 %x, %add
@@ -730,23 +711,20 @@
; GFX8-LABEL: v_mul_add_1_i16_zext_result:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i16_zext_result:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i16_zext_result:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i16 %y, 1
@@ -768,23 +746,20 @@
; GFX8-LABEL: v_mul_add_1_i16_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i16_commute:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i16_commute:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT: v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i16 %y, 1
%mul = mul i16 %add, %x
@@ -1007,63 +982,57 @@
; GFX6-LABEL: v_mul_add_1_i64:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
-; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX6-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX6-NEXT: v_mul_hi_u32 v4, v0, v2
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, v2
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v3
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT: v_mul_lo_u32 v5, v1, v2
+; GFX6-NEXT: v_mul_lo_u32 v2, v0, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_mul_add_1_i64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, 1, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_mul_lo_u32 v3, v0, v1
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX7-NEXT: v_mul_lo_u32 v2, v4, v2
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX7-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX7-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v5
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v0, v1
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_mul_lo_u32 v3, v0, v1
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX8-NEXT: v_mul_lo_u32 v2, v4, v2
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX8-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX8-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, 1, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_mul_lo_u32 v4, v1, v2
-; GFX9-NEXT: v_mul_lo_u32 v3, v0, v3
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX9-NEXT: v_add3_u32 v1, v1, v3, v4
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX9-NEXT: v_add3_u32 v1, v1, v5, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: v_mul_lo_u32 v4, v1, v2
-; GFX10-NEXT: v_mul_lo_u32 v3, v0, v3
-; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v0, v2, 0
-; GFX10-NEXT: v_add3_u32 v1, v1, v3, v4
+; GFX10-NEXT: v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX10-NEXT: v_add3_u32 v1, v1, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i64 %y, 1
%mul = mul i64 %x, %add
@@ -1074,63 +1043,57 @@
; GFX6-LABEL: v_mul_add_1_i64_commute:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v2
-; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v2, v1
-; GFX6-NEXT: v_mul_hi_u32 v4, v2, v0
-; GFX6-NEXT: v_mul_lo_u32 v3, v3, v0
-; GFX6-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, v4, v1
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX6-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX6-NEXT: v_mul_lo_u32 v5, v1, v2
+; GFX6-NEXT: v_mul_lo_u32 v2, v0, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_mul_add_1_i64_commute:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX7-NEXT: v_mul_lo_u32 v3, v0, v1
-; GFX7-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
-; GFX7-NEXT: v_mul_lo_u32 v2, v2, v4
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX7-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX7-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX7-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v1, v5
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v0, v1
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i64_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX8-NEXT: v_mul_lo_u32 v3, v0, v1
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
-; GFX8-NEXT: v_mul_lo_u32 v2, v2, v4
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX8-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX8-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i64_commute:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, 1, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v0
-; GFX9-NEXT: v_mul_lo_u32 v4, v2, v1
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v2, v0, 0
-; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX9-NEXT: v_add3_u32 v1, v1, v5, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i64_commute:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: v_mul_lo_u32 v4, v2, v1
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v0
-; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v2, v0, 0
-; GFX10-NEXT: v_add3_u32 v1, v1, v4, v3
+; GFX10-NEXT: v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT: v_mul_lo_u32 v0, v0, v3
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX10-NEXT: v_add3_u32 v1, v1, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i64 %y, 1
%mul = mul i64 %add, %x
@@ -1535,34 +1498,45 @@
; GFX67-LABEL: v_mul_add_1_i32_multiple:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT: v_mul_lo_u32 v3, v0, v1
; GFX67-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v3, v0
+; GFX67-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i32_multiple:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT: v_mul_lo_u32 v3, v0, v1
; GFX8-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v3, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_mul_add_1_i32_multiple:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, v1
-; GFX9-NEXT: v_mul_lo_u32 v1, v2, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_i32_multiple:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v3, v1
+; GFX900-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v3, v[0:1]
+; GFX900-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v2, v3, v[2:3]
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_i32_multiple:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v3, v[0:1]
+; GFX90A-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, v3, v[2:3]
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i32_multiple:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, v1
-; GFX10-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v1
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v0, v3, v[0:1]
+; GFX10-NEXT: v_mad_u64_u32 v[1:2], null, v2, v3, v[2:3]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i32 %y, 1
%mul0 = mul i32 %x, %add
@@ -1617,8 +1591,8 @@
; GFX67-NEXT: v_mul_lo_u32 v1, v2, v1
; GFX67-NEXT: v_add_i32_e32 v2, vcc, v1, v2
; GFX67-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_i32_chain:
@@ -1628,8 +1602,8 @@
; GFX8-NEXT: v_mul_lo_u32 v1, v2, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v2
; GFX8-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT: v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i32_chain:
@@ -1639,8 +1613,7 @@
; GFX9-NEXT: v_mul_lo_u32 v1, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, v1, v2
; GFX9-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i32_chain:
@@ -1650,9 +1623,8 @@
; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0
; GFX10-NEXT: v_mul_lo_u32 v1, v2, v1
; GFX10-NEXT: v_add_nc_u32_e32 v2, v1, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX10-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%i2 = add i32 %arg0, 1
%i3 = mul i32 %i2, %arg1
@@ -1680,12 +1652,12 @@
; GFX8-LABEL: v_mul_add_1_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 1
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v1
-; GFX8-NEXT: v_add_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_lo_u16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT: v_mad_u16 v2, v3, v2, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_v2i16:
@@ -1725,12 +1697,12 @@
; GFX8-LABEL: v_mul_add_1_v2i16_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 1
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v1
-; GFX8-NEXT: v_add_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_lo_u16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT: v_mad_u16 v2, v3, v2, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_v2i16_commute:
@@ -2030,38 +2002,45 @@
; GFX67-LABEL: v_mul_add_1_v2i32:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT: v_mul_lo_u32 v0, v0, v2
-; GFX67-NEXT: v_mul_lo_u32 v1, v1, v3
+; GFX67-NEXT: v_mul_lo_u32 v2, v0, v2
+; GFX67-NEXT: v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GFX67-NEXT: v_add_i32_e32 v1, vcc, v3, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_v2i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, v2
-; GFX8-NEXT: v_mul_lo_u32 v1, v1, v3
+; GFX8-NEXT: v_mul_lo_u32 v2, v0, v2
+; GFX8-NEXT: v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_mul_add_1_v2i32:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, v2
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, v3
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_v2i32:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX900-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v1, v3, v[1:2]
+; GFX900-NEXT: v_mov_b32_e32 v0, v4
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_v2i32:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[0:1]
+; GFX90A-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, v[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_v2i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, v2
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v3
+; GFX10-NEXT: v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT: v_mad_u64_u32 v[1:2], null, v1, v3, v[1:2]
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i32> %y, <i32 1, i32 1>
%mul = mul <2 x i32> %x, %add
@@ -2072,38 +2051,45 @@
; GFX67-LABEL: v_mul_add_1_v2i32_commute:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX67-NEXT: v_mul_lo_u32 v1, v3, v1
+; GFX67-NEXT: v_mul_lo_u32 v2, v0, v2
+; GFX67-NEXT: v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GFX67-NEXT: v_add_i32_e32 v1, vcc, v3, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_v2i32_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX8-NEXT: v_mul_lo_u32 v1, v3, v1
+; GFX8-NEXT: v_mul_lo_u32 v2, v0, v2
+; GFX8-NEXT: v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: v_mul_add_1_v2i32_commute:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX9-NEXT: v_mul_lo_u32 v1, v3, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_v2i32_commute:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX900-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v1, v3, v[1:2]
+; GFX900-NEXT: v_mov_b32_e32 v0, v4
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_v2i32_commute:
+; GFX90A: ; %bb.0:
+; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[0:1]
+; GFX90A-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v3, v[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_v2i32_commute:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_mul_lo_u32 v0, v2, v0
-; GFX10-NEXT: v_mul_lo_u32 v1, v3, v1
+; GFX10-NEXT: v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT: v_mad_u64_u32 v[1:2], null, v1, v3, v[1:2]
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i32> %y, <i32 1, i32 1>
%mul = mul <2 x i32> %add, %x
@@ -2382,38 +2368,30 @@
; GFX67-LABEL: v_mul_add_1_v2i24:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX67-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX67-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_v2i24:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT: v_mul_u32_u24_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_v2i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT: v_mul_u32_u24_e32 v0, v0, v2
-; GFX9-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX9-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX9-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_mul_u32_u24_e32 v0, v0, v2
-; GFX10-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX10-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX10-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i24> %y, <i24 1, i24 1>
%mul = mul <2 x i24> %x, %add
@@ -2424,38 +2402,30 @@
; GFX67-LABEL: v_mul_add_1_v2i24_commute:
; GFX67: ; %bb.0:
; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v2, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX67-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_mul_add_1_v2i24_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT: v_mul_u32_u24_e32 v0, v2, v0
-; GFX8-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX8-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_v2i24_commute:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT: v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT: v_mul_u32_u24_e32 v0, v2, v0
-; GFX9-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX9-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX9-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_v2i24_commute:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_mul_u32_u24_e32 v0, v2, v0
-; GFX10-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX10-NEXT: v_mad_u32_u24 v0, v0, v2, v0
+; GFX10-NEXT: v_mad_u32_u24 v1, v1, v3, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i24> %y, <i24 1, i24 1>
%mul = mul <2 x i24> %add, %x
@@ -3318,36 +3288,36 @@
; GFX67: ; %bb.0: ; %bb
; GFX67-NEXT: s_load_dword s3, s[0:1], 0x6
; GFX67-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX67-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x4
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-NEXT: s_load_dword s6, s[6:7], 0x1
+; GFX67-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GFX67-NEXT: s_add_i32 s3, s3, 1
; GFX67-NEXT: v_mul_lo_u32 v1, s3, v0
+; GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX67-NEXT: s_mul_i32 s2, s2, s6
; GFX67-NEXT: v_add_i32_e32 v2, vcc, s3, v1
; GFX67-NEXT: v_mul_lo_u32 v2, v2, v0
; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x4
-; GFX67-NEXT: s_load_dword s3, s[6:7], 0x1
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; GFX67-NEXT: v_mul_lo_u32 v3, v2, v1
-; GFX67-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GFX67-NEXT: s_mov_b32 s2, 0
; GFX67-NEXT: v_add_i32_e32 v1, vcc, v3, v1
; GFX67-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX67-NEXT: v_add_i32_e32 v2, vcc, 1, v3
-; GFX67-NEXT: s_mul_i32 s2, s2, s3
; GFX67-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, s2, v0
-; GFX67-NEXT: v_mov_b32_e32 v4, s5
; GFX67-NEXT: v_add_i32_e32 v2, vcc, v3, v2
; GFX67-NEXT: v_mul_lo_u32 v1, v2, v1
-; GFX67-NEXT: v_add_i32_e32 v2, vcc, 1, v3
-; GFX67-NEXT: s_mov_b32 s3, 0xf000
-; GFX67-NEXT: v_mul_lo_u32 v2, v1, v2
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-NEXT: v_mov_b32_e32 v2, s5
+; GFX67-NEXT: v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT: v_add_i32_e32 v3, vcc, v3, v1
+; GFX67-NEXT: v_mul_lo_u32 v4, v3, v1
; GFX67-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GFX67-NEXT: v_mul_lo_u32 v2, v2, v1
-; GFX67-NEXT: v_addc_u32_e32 v1, vcc, 0, v4, vcc
+; GFX67-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
; GFX67-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; GFX67-NEXT: s_mov_b32 s2, 0
+; GFX67-NEXT: v_add_i32_e32 v2, vcc, v4, v3
; GFX67-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX67-NEXT: s_endpgm
;
@@ -3355,37 +3325,37 @@
; GFX8: ; %bb.0: ; %bb
; GFX8-NEXT: s_load_dword s3, s[0:1], 0x18
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x10
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GFX8-NEXT: s_add_i32 s3, s3, 1
; GFX8-NEXT: v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s3, v1
; GFX8-NEXT: v_mul_lo_u32 v2, v2, v0
; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x10
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x4
; GFX8-NEXT: v_mul_lo_u32 v3, v2, v1
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX8-NEXT: s_and_b32 s1, s3, 0xffff
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1
; GFX8-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v3
-; GFX8-NEXT: s_mul_i32 s2, s2, s3
+; GFX8-NEXT: s_mul_i32 s2, s2, s1
; GFX8-NEXT: v_mul_lo_u32 v3, v1, v2
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
-; GFX8-NEXT: v_mov_b32_e32 v4, s5
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
; GFX8-NEXT: v_mul_lo_u32 v1, v2, v1
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v3
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mul_lo_u32 v2, v1, v2
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s5
+; GFX8-NEXT: v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_mul_lo_u32 v5, v3, v1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v0
-; GFX8-NEXT: v_mul_lo_u32 v2, v2, v1
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v4, vcc
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
; GFX8-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v3
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v4, v1, vcc
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -3397,7 +3367,7 @@
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_add_i32 s3, s3, 1
; GFX900-NEXT: v_mul_lo_u32 v1, s3, v0
-; GFX900-NEXT: v_mov_b32_e32 v4, s9
+; GFX900-NEXT: v_mov_b32_e32 v5, s9
; GFX900-NEXT: v_add_u32_e32 v2, s3, v1
; GFX900-NEXT: v_mul_lo_u32 v2, v2, v0
; GFX900-NEXT: v_add_u32_e32 v1, 1, v1
@@ -3411,20 +3381,18 @@
; GFX900-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX900-NEXT: v_add_u32_e32 v2, 1, v3
; GFX900-NEXT: v_add_u32_e32 v0, s2, v0
-; GFX900-NEXT: v_mov_b32_e32 v5, s1
+; GFX900-NEXT: v_mov_b32_e32 v4, s1
; GFX900-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX900-NEXT: v_add_u32_e32 v2, v3, v2
-; GFX900-NEXT: v_mul_lo_u32 v2, v2, v1
-; GFX900-NEXT: v_add_u32_e32 v1, 1, v3
-; GFX900-NEXT: v_mul_lo_u32 v3, v2, v1
-; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
-; GFX900-NEXT: v_add_u32_e32 v2, 1, v2
-; GFX900-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX900-NEXT: v_mul_lo_u32 v2, v3, v2
-; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, s8, v0
-; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
-; GFX900-NEXT: global_store_dword v[0:1], v2, off
+; GFX900-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX900-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v1, v3, v[1:2]
+; GFX900-NEXT: v_add_co_u32_e32 v3, vcc, s0, v0
+; GFX900-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; GFX900-NEXT: v_lshlrev_b64 v[3:4], 2, v[3:4]
+; GFX900-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v1, v[2:3]
+; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, s8, v3
+; GFX900-NEXT: v_addc_co_u32_e32 v2, vcc, v5, v4, vcc
+; GFX900-NEXT: global_store_dword v[1:2], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX90A-LABEL: compute_mad:
@@ -3438,23 +3406,21 @@
; GFX90A-NEXT: v_add_u32_e32 v2, s3, v1
; GFX90A-NEXT: v_mul_lo_u32 v2, v2, v0
; GFX90A-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX90A-NEXT: s_load_dword s3, s[6:7], 0x4
; GFX90A-NEXT: v_mul_lo_u32 v3, v2, v1
; GFX90A-NEXT: v_add_u32_e32 v1, v3, v1
; GFX90A-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX90A-NEXT: v_add_u32_e32 v2, 1, v3
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX90A-NEXT: v_mul_lo_u32 v3, v1, v2
; GFX90A-NEXT: v_add_u32_e32 v2, v3, v2
+; GFX90A-NEXT: s_load_dword s3, s[6:7], 0x4
+; GFX90A-NEXT: v_mul_lo_u32 v2, v2, v1
+; GFX90A-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v2, v3, v[2:3]
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX90A-NEXT: v_mul_lo_u32 v1, v2, v1
-; GFX90A-NEXT: v_add_u32_e32 v2, 1, v3
; GFX90A-NEXT: s_mul_i32 s2, s2, s3
-; GFX90A-NEXT: v_add_u32_e32 v3, 1, v1
-; GFX90A-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX90A-NEXT: v_add_u32_e32 v0, s2, v0
-; GFX90A-NEXT: v_mul_lo_u32 v2, v1, v3
+; GFX90A-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v4, v2, v[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, s1
; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
@@ -3485,20 +3451,18 @@
; GFX10-NEXT: s_and_b32 s3, s3, 0xffff
; GFX10-NEXT: v_mul_lo_u32 v2, v1, v2
; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v3
-; GFX10-NEXT: v_mul_lo_u32 v3, v2, v1
-; GFX10-NEXT: v_add_nc_u32_e32 v4, v3, v1
+; GFX10-NEXT: v_mul_lo_u32 v4, v2, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v3, v4, v1
; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, s2, s3, v[0:1]
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_mul_lo_u32 v2, v4, v2
-; GFX10-NEXT: v_add_co_u32 v0, s2, s4, v0
-; GFX10-NEXT: v_add_co_ci_u32_e64 v1, null, s5, 0, s2
-; GFX10-NEXT: v_mul_lo_u32 v3, v2, v3
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX10-NEXT: v_mul_lo_u32 v2, v3, v2
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT: global_store_dword v[0:1], v2, off
+; GFX10-NEXT: v_mul_lo_u32 v1, v3, v2
+; GFX10-NEXT: v_add_co_u32 v2, s2, s4, v0
+; GFX10-NEXT: v_add_co_ci_u32_e64 v3, null, s5, 0, s2
+; GFX10-NEXT: v_mad_u64_u32 v[4:5], null, v1, v4, v[1:2]
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, v4, v1, v[4:5]
+; GFX10-NEXT: v_add_co_u32 v1, vcc_lo, s0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, s1, v3, vcc_lo
+; GFX10-NEXT: global_store_dword v[1:2], v0, off
; GFX10-NEXT: s_endpgm
bb:
%i = tail call i32 @llvm.amdgcn.workitem.id.x(), !range !0
@@ -3603,23 +3567,20 @@
; GFX8-LABEL: v_mul_add_1_i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i8 %y, 1
%mul = mul i8 %x, %add
@@ -3639,23 +3600,20 @@
; GFX8-LABEL: v_mul_add_1_i8_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i8_commute:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i8_commute:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT: v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i8 %y, 1
%mul = mul i8 %add, %x
@@ -3674,23 +3632,20 @@
; GFX8-LABEL: v_mul_add_1_i8_zext:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i8_zext:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i8_zext:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i8 %y, 1
%mul = mul i8 %x, %add
@@ -3709,23 +3664,20 @@
; GFX8-LABEL: v_mul_add_1_i8_zext_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_add_1_i8_zext_commute:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_1_i8_zext_commute:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT: v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT: v_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add i8 %y, 1
%mul = mul i8 %add, %x
@@ -3752,11 +3704,9 @@
; GFX8-LABEL: v_mul_add_1_v2i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v3, 1, v3
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-NEXT: v_mad_u16 v1, v1, v3, v1
; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-NEXT: v_mad_u16 v0, v0, v2, v0
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -3764,11 +3714,9 @@
; GFX9-LABEL: v_mul_add_1_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v3, 1, v3
-; GFX9-NEXT: v_add_u16_e32 v2, 1, v2
-; GFX9-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v3, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v2, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3777,10 +3725,8 @@
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v3, v3, 1
-; GFX10-NEXT: v_add_nc_u16 v2, v2, 1
-; GFX10-NEXT: v_mul_lo_u16 v1, v1, v3
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT: v_mad_u16 v1, v1, v3, v1
+; GFX10-NEXT: v_mad_u16 v0, v0, v2, v0
; GFX10-NEXT: v_lshlrev_b16 v2, 8, v1
; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -3810,11 +3756,9 @@
; GFX8-LABEL: v_mul_add_1_v2i8_commute:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v3, 1, v3
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v3, v1
+; GFX8-NEXT: v_mad_u16 v1, v1, v3, v1
; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v2, v0
+; GFX8-NEXT: v_mad_u16 v0, v0, v2, v0
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -3822,11 +3766,9 @@
; GFX9-LABEL: v_mul_add_1_v2i8_commute:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v3, 1, v3
-; GFX9-NEXT: v_add_u16_e32 v2, 1, v2
-; GFX9-NEXT: v_mul_lo_u16_e32 v1, v3, v1
+; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v3, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT: v_mul_lo_u16_e32 v0, v2, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v2, v0
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3835,10 +3777,8 @@
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_nc_u16 v3, v3, 1
-; GFX10-NEXT: v_add_nc_u16 v2, v2, 1
-; GFX10-NEXT: v_mul_lo_u16 v1, v3, v1
-; GFX10-NEXT: v_mul_lo_u16 v0, v2, v0
+; GFX10-NEXT: v_mad_u16 v1, v1, v3, v1
+; GFX10-NEXT: v_mad_u16 v0, v0, v2, v0
; GFX10-NEXT: v_lshlrev_b16 v2, 8, v1
; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD