Index: llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
===================================================================
--- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -3515,6 +3515,16 @@
   return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi);
 }
 
+/// If \p V is an add of a constant 1, returns the other operand. Otherwise
+/// return SDValue().
+static SDValue getAddOneOp(const SDNode *V) {
+  if (V->getOpcode() != ISD::ADD)
+    return SDValue();
+
+  auto *C = dyn_cast<ConstantSDNode>(V->getOperand(1));
+  return C && C->isOne() ? V->getOperand(0) : SDValue();
+}
+
 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
@@ -3530,16 +3540,49 @@
   if (VT.isVector() || Size > 64)
     return SDValue();
 
-  // There are i16 integer mul/mad.
-  if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
-    return SDValue();
-
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
+  // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
+  // matching.
+
+  // mul x, (add y, 1) -> add (mul x, y), x
+  auto IsFoldableAdd = [](SDValue V) -> SDValue {
+    SDValue AddOp = getAddOneOp(V.getNode());
+    if (!AddOp)
+      return SDValue();
+
+    if (V.hasOneUse() || all_of(V->uses(), [](const SDNode *U) -> bool {
+          return U->getOpcode() == ISD::MUL;
+        }))
+      return AddOp;
+
+    return SDValue();
+  };
+
+  // FIXME: The selection pattern is not properly checking for commuted
+  // operands, so we have to place the mul in the LHS
+  if (SDValue MulOper = IsFoldableAdd(N0)) {
+    SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper);
+    return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1);
+  }
+
+  if (SDValue MulOper = IsFoldableAdd(N1)) {
+    SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper);
+    return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0);
+  }
+
+  // Skip if already mul24.
+  if (N->getOpcode() != ISD::MUL)
+    return SDValue();
+
+  // There are i16 integer mul/mad.
+  if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
+    return SDValue();
+
   // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
   // in the source into any_extends if the result of the mul is truncated. Since
   // we can assume the high bits are whatever we want, use the underlying value
@@ -4347,6 +4390,15 @@
     return performTruncateCombine(N, DCI);
   case ISD::MUL:
     return performMulCombine(N, DCI);
+  case AMDGPUISD::MUL_U24:
+  case AMDGPUISD::MUL_I24: {
+    if (SDValue Simplified = simplifyMul24(N, DCI))
+      return Simplified;
+    return performMulCombine(N, DCI);
+  }
+  case AMDGPUISD::MULHI_I24:
+  case AMDGPUISD::MULHI_U24:
+    return simplifyMul24(N, DCI);
   case ISD::SMUL_LOHI:
   case ISD::UMUL_LOHI:
     return performMulLoHiCombine(N, DCI);
@@ -4354,11 +4406,6 @@
     return performMulhsCombine(N, DCI);
   case ISD::MULHU:
     return performMulhuCombine(N, DCI);
-  case AMDGPUISD::MUL_I24:
-  case AMDGPUISD::MUL_U24:
-  case AMDGPUISD::MULHI_I24:
-  case AMDGPUISD::MULHI_U24:
-    return simplifyMul24(N, DCI);
   case ISD::SELECT:
     return performSelectCombine(N, DCI);
   case ISD::FNEG:
Index: llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
===================================================================
--- llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -14,30 +14,28 @@
 ; GFX67-LABEL: v_mul_add_1_i32:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i32 %y, 1
   %mul = mul i32 %x, %add
@@ -48,30 +46,28 @@
 ; GFX67-LABEL: v_mul_add_1_i32_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i32_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i32 %y, 1
   %mul = mul i32 %add, %x
@@ -352,30 +348,26 @@
 ; GFX67-LABEL: v_mul_add_1_i24_zext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i24_zext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i24_zext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i24_zext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i24 %y, 1
   %mul = mul i24 %x, %add
@@ -450,30 +442,26 @@
 ; GFX67-LABEL: v_mul_add_1_i24_sext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i24_sext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i24_sext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i24_sext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i24 %y, 1
   %mul = mul i24 %x, %add
@@ -548,30 +536,28 @@
 ; GFX67-LABEL: v_mul_add_1_i25_zext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i25_zext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i25_zext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i25_zext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i25 %y, 1
   %mul = mul i25 %x, %add
@@ -616,30 +602,28 @@
 ; GFX67-LABEL: v_mul_add_1_i25_sext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i25_sext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i25_sext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i25_sext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i25 %y, 1
   %mul = mul i25 %x, %add
@@ -693,23 +677,20 @@
 ; GFX8-LABEL: v_mul_add_1_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i16 %y, 1
   %mul = mul i16 %x, %add
@@ -730,23 +711,20 @@
 ; GFX8-LABEL: v_mul_add_1_i16_zext_result:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i16_zext_result:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i16_zext_result:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i16 %y, 1
@@ -768,23 +746,20 @@
 ; GFX8-LABEL: v_mul_add_1_i16_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i16_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i16_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i16 %y, 1
   %mul = mul i16 %add, %x
@@ -1007,63 +982,57 @@
 ; GFX6-LABEL: v_mul_add_1_i64:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v3
 ; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v4, v3
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT:    v_mul_lo_u32 v5, v1, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: v_mul_add_1_i64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX7-NEXT:    v_mov_b32_e32 v4, v1
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v4, v2
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX7-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX7-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mov_b32_e32 v4, v1
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX8-NEXT:    v_mul_lo_u32 v2, v4, v2
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 1, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v4, v1, v2
-; GFX9-NEXT:    v_mul_lo_u32 v3, v0, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX9-NEXT:    v_add3_u32 v1, v1, v3, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 1
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT:    v_mul_lo_u32 v4, v1, v2
-; GFX10-NEXT:    v_mul_lo_u32 v3, v0, v3
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v2, 0
-; GFX10-NEXT:    v_add3_u32 v1, v1, v3, v4
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i64 %y, 1
   %mul = mul i64 %x, %add
@@ -1074,63 +1043,57 @@
 ; GFX6-LABEL: v_mul_add_1_i64_commute:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX6-NEXT:    v_mul_hi_u32 v4, v2, v0
-; GFX6-NEXT:    v_mul_lo_u32 v3, v3, v0
-; GFX6-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v3
+; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v2
+; GFX6-NEXT:    v_mul_lo_u32 v5, v1, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: v_mul_add_1_i64_commute:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX7-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX7-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i64_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v4, v0
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
-; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i64_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 1, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v0
-; GFX9-NEXT:    v_mul_lo_u32 v4, v2, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v2, v0, 0
-; GFX9-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i64_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 1
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v1
-; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v0
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v2, v0, 0
-; GFX10-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i64 %y, 1
   %mul = mul i64 %add, %x
@@ -1535,34 +1498,45 @@
 ; GFX67-LABEL: v_mul_add_1_i32_multiple:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v3, v0, v1
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32_multiple:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v1
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v3, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: v_mul_add_1_i32_multiple:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
-; GFX9-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_i32_multiple:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v3, v1
+; GFX900-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v3, v[0:1]
+; GFX900-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v2, v3, v[2:3]
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_i32_multiple:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v3, v[0:1]
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v2, v3, v[2:3]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32_multiple:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
-; GFX10-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX10-NEXT:    v_mov_b32_e32 v3, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v3, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v2, v3, v[2:3]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i32 %y, 1
   %mul0 = mul i32 %x, %add
@@ -1617,8 +1591,8 @@
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, v1, v2
 ; GFX67-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32_chain:
@@ -1628,8 +1602,8 @@
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v1, v2
 ; GFX8-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i32_chain:
@@ -1639,8 +1613,7 @@
 ; GFX9-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX9-NEXT:    v_add_u32_e32 v2, v1, v2
 ; GFX9-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32_chain:
@@ -1650,9 +1623,8 @@
 ; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v2, v1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %i2 = add i32 %arg0, 1
   %i3 = mul i32 %i2, %arg1
@@ -1680,12 +1652,12 @@
 ; GFX8-LABEL: v_mul_add_1_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v3, 1
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v1
-; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_mul_lo_u16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT:    v_mad_u16 v2, v3, v2, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i16:
@@ -1725,12 +1697,12 @@
 ; GFX8-LABEL: v_mul_add_1_v2i16_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v3, 1
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v1
-; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_mul_lo_u16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v2, v0
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT:    v_mad_u16 v2, v3, v2, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i16_commute:
@@ -2030,38 +2002,45 @@
 ; GFX67-LABEL: v_mul_add_1_v2i32:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX67-NEXT:    v_mul_lo_u32 v1, v1, v3
+; GFX67-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v3
+; GFX8-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v2, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: v_mul_add_1_v2i32:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v3
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_v2i32:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX900-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v1, v3, v[1:2]
+; GFX900-NEXT:    v_mov_b32_e32 v0, v4
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_v2i32:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[0:1]
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, v[4:5]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v3
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v1, v3, v[1:2]
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i32> %y, <i32 1, i32 1>
   %mul = mul <2 x i32> %x, %add
@@ -2072,38 +2051,45 @@
 ; GFX67-LABEL: v_mul_add_1_v2i32_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX67-NEXT:    v_mul_lo_u32 v1, v3, v1
+; GFX67-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i32_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX8-NEXT:    v_mul_lo_u32 v1, v3, v1
+; GFX8-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v2, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: v_mul_add_1_v2i32_commute:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v3, v1
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_v2i32_commute:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX900-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v1, v3, v[1:2]
+; GFX900-NEXT:    v_mov_b32_e32 v0, v4
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_v2i32_commute:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[0:1]
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, v[4:5]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i32_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX10-NEXT:    v_mul_lo_u32 v1, v3, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v1, v3, v[1:2]
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i32> %y, <i32 1, i32 1>
   %mul = mul <2 x i32> %add, %x
@@ -2382,38 +2368,30 @@
 ; GFX67-LABEL: v_mul_add_1_v2i24:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX67-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i24:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX8-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX8-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i24:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX9-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX9-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i24:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX10-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX10-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i24> %y, <i24 1, i24 1>
   %mul = mul <2 x i24> %x, %add
@@ -2424,38 +2402,30 @@
 ; GFX67-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX67-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX8-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX8-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX9-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX9-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX10-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX10-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i24> %y, <i24 1, i24 1>
   %mul = mul <2 x i24> %add, %x
@@ -3318,36 +3288,36 @@
 ; GFX67:       ; %bb.0: ; %bb
 ; GFX67-NEXT:    s_load_dword s3, s[0:1], 0x6
 ; GFX67-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX67-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4
 ; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX67-NEXT:    s_load_dword s6, s[6:7], 0x1
+; GFX67-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX67-NEXT:    s_add_i32 s3, s3, 1
 ; GFX67-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX67-NEXT:    s_and_b32 s6, s6, 0xffff
+; GFX67-NEXT:    s_mul_i32 s2, s2, s6
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, s3, v1
 ; GFX67-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4
-; GFX67-NEXT:    s_load_dword s3, s[6:7], 0x1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
 ; GFX67-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX67-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX67-NEXT:    s_and_b32 s3, s3, 0xffff
+; GFX67-NEXT:    s_mov_b32 s3, 0xf000
+; GFX67-NEXT:    s_mov_b32 s2, 0
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
-; GFX67-NEXT:    s_mul_i32 s2, s2, s3
 ; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX67-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
-; GFX67-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
-; GFX67-NEXT:    s_mov_b32 s3, 0xf000
-; GFX67-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-NEXT:    v_mov_b32_e32 v2, s5
+; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT:    v_add_i32_e32 v3, vcc, v3, v1
+; GFX67-NEXT:    v_mul_lo_u32 v4, v3, v1
 ; GFX67-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
-; GFX67-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX67-NEXT:    v_addc_u32_e32 v1, vcc, 0, v4, vcc
+; GFX67-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX67-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
-; GFX67-NEXT:    s_mov_b32 s2, 0
+; GFX67-NEXT:    v_add_i32_e32 v2, vcc, v4, v3
 ; GFX67-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX67-NEXT:    s_endpgm
 ;
@@ -3355,37 +3325,37 @@
 ; GFX8:       ; %bb.0: ; %bb
 ; GFX8-NEXT:    s_load_dword s3, s[0:1], 0x18
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x10
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX8-NEXT:    s_add_i32 s3, s3, 1
 ; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s3, v1
 ; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x10
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x4
 ; GFX8-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s3, s3, 0xffff
+; GFX8-NEXT:    s_and_b32 s1, s3, 0xffff
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
-; GFX8-NEXT:    s_mul_i32 s2, s2, s3
+; GFX8-NEXT:    s_mul_i32 s2, s2, s1
 ; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT:    v_mul_lo_u32 v5, v3, v1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v0
-; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v4, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v3
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -3397,7 +3367,7 @@
 ; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-NEXT:    s_add_i32 s3, s3, 1
 ; GFX900-NEXT:    v_mul_lo_u32 v1, s3, v0
-; GFX900-NEXT:    v_mov_b32_e32 v4, s9
+; GFX900-NEXT:    v_mov_b32_e32 v5, s9
 ; GFX900-NEXT:    v_add_u32_e32 v2, s3, v1
 ; GFX900-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX900-NEXT:    v_add_u32_e32 v1, 1, v1
@@ -3411,20 +3381,18 @@
 ; GFX900-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX900-NEXT:    v_add_u32_e32 v2, 1, v3
 ; GFX900-NEXT:    v_add_u32_e32 v0, s2, v0
-; GFX900-NEXT:    v_mov_b32_e32 v5, s1
+; GFX900-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX900-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX900-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX900-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX900-NEXT:    v_add_u32_e32 v1, 1, v3
-; GFX900-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX900-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
-; GFX900-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX900-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX900-NEXT:    v_mul_lo_u32 v2, v3, v2
-; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s8, v0
-; GFX900-NEXT:    v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
-; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX900-NEXT:    v_mad_u64_u32 v[2:3], s[2:3], v1, v3, v[1:2]
+; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, s0, v0
+; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; GFX900-NEXT:    v_lshlrev_b64 v[3:4], 2, v[3:4]
+; GFX900-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v2, v1, v[2:3]
+; GFX900-NEXT:    v_add_co_u32_e32 v1, vcc, s8, v3
+; GFX900-NEXT:    v_addc_co_u32_e32 v2, vcc, v5, v4, vcc
+; GFX900-NEXT:    global_store_dword v[1:2], v0, off
 ; GFX900-NEXT:    s_endpgm
 ;
 ; GFX90A-LABEL: compute_mad:
@@ -3438,23 +3406,21 @@
 ; GFX90A-NEXT:    v_add_u32_e32 v2, s3, v1
 ; GFX90A-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX90A-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX90A-NEXT:    s_load_dword s3, s[6:7], 0x4
 ; GFX90A-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v1, v3, v1
 ; GFX90A-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX90A-NEXT:    v_add_u32_e32 v2, 1, v3
-; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; GFX90A-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GFX90A-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX90A-NEXT:    s_load_dword s3, s[6:7], 0x4
+; GFX90A-NEXT:    v_mul_lo_u32 v2, v2, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], v2, v3, v[2:3]
+; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_and_b32 s3, s3, 0xffff
-; GFX90A-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX90A-NEXT:    v_add_u32_e32 v2, 1, v3
 ; GFX90A-NEXT:    s_mul_i32 s2, s2, s3
-; GFX90A-NEXT:    v_add_u32_e32 v3, 1, v1
-; GFX90A-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX90A-NEXT:    v_add_u32_e32 v0, s2, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v2, v1, v3
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v2, v[4:5]
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
@@ -3485,20 +3451,18 @@
 ; GFX10-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX10-NEXT:    v_mul_lo_u32 v2, v1, v2
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, v3, v1
+; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v4, v1
 ; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s2, s3, v[0:1]
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v2, v4, v2
-; GFX10-NEXT:    v_add_co_u32 v0, s2, s4, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s2
-; GFX10-NEXT:    v_mul_lo_u32 v3, v2, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX10-NEXT:    v_mul_lo_u32 v2, v3, v2
-; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    v_mul_lo_u32 v1, v3, v2
+; GFX10-NEXT:    v_add_co_u32 v2, s2, s4, v0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, 0, s2
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v1, v4, v[1:2]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v1, v[4:5]
+; GFX10-NEXT:    v_add_co_u32 v1, vcc_lo, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, s1, v3, vcc_lo
+; GFX10-NEXT:    global_store_dword v[1:2], v0, off
 ; GFX10-NEXT:    s_endpgm
 bb:
   %i = tail call i32 @llvm.amdgcn.workitem.id.x(), !range !0
@@ -3603,23 +3567,20 @@
 ; GFX8-LABEL: v_mul_add_1_i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %x, %add
@@ -3639,23 +3600,20 @@
 ; GFX8-LABEL: v_mul_add_1_i8_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %add, %x
@@ -3674,23 +3632,20 @@
 ; GFX8-LABEL: v_mul_add_1_i8_zext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8_zext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8_zext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %x, %add
@@ -3709,23 +3664,20 @@
 ; GFX8-LABEL: v_mul_add_1_i8_zext_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8_zext_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8_zext_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %add, %x
@@ -3752,11 +3704,9 @@
 ; GFX8-LABEL: v_mul_add_1_v2i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX8-NEXT:    v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-NEXT:    v_mad_u16 v1, v1, v3, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -3764,11 +3714,9 @@
 ; GFX9-LABEL: v_mul_add_1_v2i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u16_e32 v1, v1, v3
+; GFX9-NEXT:    v_mad_legacy_u16 v1, v1, v3, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v2
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v2, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3777,10 +3725,8 @@
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v3, v3, 1
-; GFX10-NEXT:    v_add_nc_u16 v2, v2, 1
-; GFX10-NEXT:    v_mul_lo_u16 v1, v1, v3
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT:    v_mad_u16 v1, v1, v3, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX10-NEXT:    v_lshlrev_b16 v2, 8, v1
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
@@ -3810,11 +3756,9 @@
 ; GFX8-LABEL: v_mul_add_1_v2i8_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX8-NEXT:    v_mul_lo_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_mad_u16 v1, v1, v3, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v2, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -3822,11 +3766,9 @@
 ; GFX9-LABEL: v_mul_add_1_v2i8_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u16_e32 v1, v3, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v1, v1, v3, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v2, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v2, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3835,10 +3777,8 @@
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v3, v3, 1
-; GFX10-NEXT:    v_add_nc_u16 v2, v2, 1
-; GFX10-NEXT:    v_mul_lo_u16 v1, v3, v1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v2, v0
+; GFX10-NEXT:    v_mad_u16 v1, v1, v3, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX10-NEXT:    v_lshlrev_b16 v2, 8, v1
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD