AMDGPU: Fix regressions in integer mad matching

[arsenm]

Summary

Undo the canonicalize done in
0cfc6510323fbb5a56a5de23cbc65f7cc30fd34c. Restores some regressed
matching of integer mad. The selection patterns fo the actual mads
don't seem to be properly commuting, so some of the commuted cases are
still missed.

Fixes: SWDEV-363009

[rampitec]

LGTM. I assume the baseline test is not committed yet.

[arsenm]

c01f284fbbf4c93c65b1f92fb8b3d263afb796c0

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
   unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
 
   SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
   SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
 
   return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi);
 }
 
+/// If \p V is an add of a constant 1, returns the other operand. Otherwise
+/// return SDValue().
+static SDValue getAddOneOp(const SDNode *V) {
+  if (V->getOpcode() != ISD::ADD)
+    return SDValue();
+
+  auto *C = dyn_cast<ConstantSDNode>(V->getOperand(1));
+  return C && C->isOne() ? V->getOperand(0) : SDValue();
+}
+
 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
 
   // Don't generate 24-bit multiplies on values that are in SGPRs, since
   // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
   // unnecessarily). isDivergent() is used as an approximation of whether the
   // value is in an SGPR.
   if (!N->isDivergent())
     return SDValue();
 
   unsigned Size = VT.getSizeInBits();
   if (VT.isVector() || Size > 64)
     return SDValue();
 
-  // There are i16 integer mul/mad.
-  if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
-    return SDValue();
-
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
+  // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
+  // matching.
+
+  // mul x, (add y, 1) -> add (mul x, y), x
+  auto IsFoldableAdd = [](SDValue V) -> SDValue {
+    SDValue AddOp = getAddOneOp(V.getNode());
+    if (!AddOp)
+      return SDValue();
+
+    if (V.hasOneUse() || all_of(V->uses(), [](const SDNode *U) -> bool {
+          return U->getOpcode() == ISD::MUL;
+        }))
+      return AddOp;
+
+    return SDValue();
+  };
+
+  // FIXME: The selection pattern is not properly checking for commuted
+  // operands, so we have to place the mul in the LHS
+  if (SDValue MulOper = IsFoldableAdd(N0)) {
+    SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper);
+    return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1);
+  }
+
+  if (SDValue MulOper = IsFoldableAdd(N1)) {
+    SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper);
+    return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0);
+  }
+
+  // Skip if already mul24.
+  if (N->getOpcode() != ISD::MUL)
+    return SDValue();
+
+  // There are i16 integer mul/mad.
+  if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
+    return SDValue();
+
   // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
   // in the source into any_extends if the result of the mul is truncated. Since
   // we can assume the high bits are whatever we want, use the underlying value
   // to avoid the unknown high bits from interfering.
   if (N0.getOpcode() == ISD::ANY_EXTEND)
     N0 = N0.getOperand(0);
 
   if (N1.getOpcode() == ISD::ANY_EXTEND)
if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
       break;
 
     return performSraCombine(N, DCI);
   }
   case ISD::TRUNCATE:
     return performTruncateCombine(N, DCI);
   case ISD::MUL:
     return performMulCombine(N, DCI);
+  case AMDGPUISD::MUL_U24:
+  case AMDGPUISD::MUL_I24: {
+    if (SDValue Simplified = simplifyMul24(N, DCI))
+      return Simplified;
+    return performMulCombine(N, DCI);
+  }
+  case AMDGPUISD::MULHI_I24:
+  case AMDGPUISD::MULHI_U24:
+    return simplifyMul24(N, DCI);
   case ISD::SMUL_LOHI:
   case ISD::UMUL_LOHI:
     return performMulLoHiCombine(N, DCI);
   case ISD::MULHS:
     return performMulhsCombine(N, DCI);
   case ISD::MULHU:
     return performMulhuCombine(N, DCI);
-  case AMDGPUISD::MUL_I24:
-  case AMDGPUISD::MUL_U24:
-  case AMDGPUISD::MULHI_I24:
-  case AMDGPUISD::MULHI_U24:
-    return simplifyMul24(N, DCI);
   case ISD::SELECT:
     return performSelectCombine(N, DCI);
   case ISD::FNEG:
     return performFNegCombine(N, DCI);
   case ISD::FABS:
     return performFAbsCombine(N, DCI);
   case AMDGPUISD::BFE_I32:
   case AMDGPUISD::BFE_U32: {

llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx600 < %s | FileCheck -check-prefixes=GFX67,GFX6 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX67,GFX7 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX9,GFX90A %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10 %s
 
 ; We want to undo these canonicalizations to enable mad matching:
 ; (x * y) + x --> x * (y + 1)
 ; (x * y) - x --> x * (y - 1)
 
 define i32 @v_mul_add_1_i32(i32 %x, i32 %y) {
 ; GFX67-LABEL: v_mul_add_1_i32:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i32 %y, 1
   %mul = mul i32 %x, %add
   ret i32 %mul
 }
 
 define i32 @v_mul_add_1_i32_commute(i32 %x, i32 %y) {
 ; GFX67-LABEL: v_mul_add_1_i32_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i32_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i32 %y, 1
   %mul = mul i32 %add, %x
   ret i32 %mul
 }
 
 define i32 @v_mul_add_x_i32(i32 %x, i32 %y) {
 ; GFX67-LABEL: v_mul_add_x_i32:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i32 %x, %sub
   ret i32 %mul
 }
 
 define i24 @v_mul_add_1_i24_zext(i24 zeroext %x, i24 zeroext %y) {
 ; GFX67-LABEL: v_mul_add_1_i24_zext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i24_zext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i24_zext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i24_zext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i24 %y, 1
   %mul = mul i24 %x, %add
   ret i24 %mul
 }
 
 define i24 @v_mul_sub_1_i24_zext(i24 zeroext %x, i24 zeroext %y) {
 ; GFX67-LABEL: v_mul_sub_1_i24_zext:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i24 %mul, %x
   ret i24 %add
 }
 
 define i24 @v_mul_add_1_i24_sext(i24 signext %x, i24 signext %y) {
 ; GFX67-LABEL: v_mul_add_1_i24_sext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i24_sext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i24_sext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i24_sext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i24 %y, 1
   %mul = mul i24 %x, %add
   ret i24 %mul
 }
 
 define i24 @v_add_mul_i24_sext_1(i24 signext %x, i24 signext %y) {
 ; GFX67-LABEL: v_add_mul_i24_sext_1:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i24 %x, %sub
   ret i24 %mul
 }
 
 define i25 @v_mul_add_1_i25_zext(i25 zeroext %x, i25 zeroext %y) {
 ; GFX67-LABEL: v_mul_add_1_i25_zext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i25_zext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i25_zext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i25_zext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i25 %y, 1
   %mul = mul i25 %x, %add
   ret i25 %mul
 }
 
 define i25 @v_mul_sub_1_i25_zext(i25 zeroext %x, i25 zeroext %y) {
 ; GFX67-LABEL: v_mul_sub_1_i25_zext:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i25 %x, %sub
   ret i25 %mul
 }
 
 define i25 @v_mul_add_1_i25_sext(i25 signext %x, i25 signext %y) {
 ; GFX67-LABEL: v_mul_add_1_i25_sext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i25_sext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i25_sext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i25_sext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i25 %y, 1
   %mul = mul i25 %x, %add
   ret i25 %mul
 }
 
 define i25 @v_mul_sub_1_i25_sext(i25 signext %x, i25 signext %y) {
 ; GFX67-LABEL: v_mul_sub_1_i25_sext:

 ; GFX67-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i16 %y, 1
   %mul = mul i16 %x, %add
   ret i16 %mul
 }
 
 define i32 @v_mul_add_1_i16_zext_result(i16 %x, i16 %y) {
 ; GFX67-LABEL: v_mul_add_1_i16_zext_result:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
 ; GFX67-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
 ; GFX67-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i16_zext_result:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i16_zext_result:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i16_zext_result:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i16 %y, 1
   %mul = mul i16 %x, %add
   %zext = zext i16 %mul to i32
   ret i32 %zext
 }
 
 define i16 @v_mul_add_1_i16_commute(i16 %x, i16 %y) {
 ; GFX67-LABEL: v_mul_add_1_i16_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX67-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i16_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i16_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i16_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i16 %y, 1
   %mul = mul i16 %add, %x
   ret i16 %mul
 }
 
 define i16 @v_mul_add_x_i16(i16 %x, i16 %y) {
 ; GFX67-LABEL: v_mul_add_x_i16:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i16 %x, %sub
   ret i16 %mul
 }
 
 define i64 @v_mul_add_1_i64(i64 %x, i64 %y) {
 ; GFX6-LABEL: v_mul_add_1_i64:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v3
 ; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v2
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v4, v3
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT:    v_mul_lo_u32 v5, v1, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: v_mul_add_1_i64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX7-NEXT:    v_mov_b32_e32 v4, v1
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v4, v2
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX7-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX7-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mov_b32_e32 v4, v1
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX8-NEXT:    v_mul_lo_u32 v2, v4, v2
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 1, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v4, v1, v2
-; GFX9-NEXT:    v_mul_lo_u32 v3, v0, v3
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
-; GFX9-NEXT:    v_add3_u32 v1, v1, v3, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 1
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT:    v_mul_lo_u32 v4, v1, v2
-; GFX10-NEXT:    v_mul_lo_u32 v3, v0, v3
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v2, 0
-; GFX10-NEXT:    v_add3_u32 v1, v1, v3, v4
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i64 %y, 1
   %mul = mul i64 %x, %add
   ret i64 %mul
 }
 
 define i64 @v_mul_add_1_i64_commute(i64 %x, i64 %y) {
 ; GFX6-LABEL: v_mul_add_1_i64_commute:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX6-NEXT:    v_mul_hi_u32 v4, v2, v0
-; GFX6-NEXT:    v_mul_lo_u32 v3, v3, v0
-; GFX6-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v3
+; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v2
+; GFX6-NEXT:    v_mul_lo_u32 v5, v1, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: v_mul_add_1_i64_commute:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v4, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX7-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
-; GFX7-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX7-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX7-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i64_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v4, v0
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v1
-; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
-; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v4
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i64_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, 1, v2
-; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v0
-; GFX9-NEXT:    v_mul_lo_u32 v4, v2, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v2, v0, 0
-; GFX9-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i64_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 1
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v1
-; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v0
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v2, v0, 0
-; GFX10-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v3
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX10-NEXT:    v_add3_u32 v1, v1, v5, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i64 %y, 1
   %mul = mul i64 %add, %x
   ret i64 %mul
 }
 
 define i64 @v_mul_add_x_i64(i64 %x, i64 %y) {
 ; GFX6-LABEL: v_mul_add_x_i64:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul i64 %x, %sub
   ret i64 %mul
 }
 
 define <2 x i32> @v_mul_add_1_i32_multiple(i32 %x, i32 %y, i32 %z) {
 ; GFX67-LABEL: v_mul_add_1_i32_multiple:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v3, v0, v1
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32_multiple:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v1
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v3, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: v_mul_add_1_i32_multiple:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
-; GFX9-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_i32_multiple:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v3, v1
+; GFX900-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v3, v[0:1]
+; GFX900-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v2, v3, v[2:3]
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_i32_multiple:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v3, v[0:1]
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v2, v3, v[2:3]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32_multiple:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
-; GFX10-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX10-NEXT:    v_mov_b32_e32 v3, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v3, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v2, v3, v[2:3]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i32 %y, 1
   %mul0 = mul i32 %x, %add
   %mul1 = mul i32 %z, %add
   %insert.0 = insertelement <2 x i32> poison, i32 %mul0, i32 0
   %insert.1 = insertelement <2 x i32> %insert.0, i32 %mul1, i32 1
   ret <2 x i32> %insert.1
 }

 define i32 @v_mul_add_1_i32_chain(i32 %arg0, i32 %arg1, i32 %arg2) {
 ; GFX67-LABEL: v_mul_add_1_i32_chain:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, v1, v2
 ; GFX67-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX67-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i32_chain:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v0
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v1, v2
 ; GFX8-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT:    v_mul_lo_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i32_chain:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_add_u32_e32 v2, 1, v0
 ; GFX9-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX9-NEXT:    v_add_u32_e32 v2, v1, v2
 ; GFX9-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX9-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[0:1]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i32_chain:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v1, v2, v1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v2, v1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
 ; GFX10-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v0, v1, v[0:1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %i2 = add i32 %arg0, 1
   %i3 = mul i32 %i2, %arg1
   %i4 = add i32 %i3, %i2
   %i5 = mul i32 %i4, %arg0
   %i6 = add i32 %i3, 1
   %i7 = mul i32 %i5, %i6
   ret i32 %i7

 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v3, 1
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v1
-; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_mul_lo_u16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT:    v_mad_u16 v2, v3, v2, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_mul_lo_u16 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]

 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i16_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v3, 1
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v1
-; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_mul_lo_u16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v2, v0
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-NEXT:    v_mad_u16 v2, v3, v2, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i16_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0]
 ; GFX9-NEXT:    v_pk_mul_lo_u16 v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul <2 x i16> %x, %sub
   ret <2 x i16> %mul
 }
 
 define <2 x i32> @v_mul_add_1_v2i32(<2 x i32> %x, <2 x i32> %y) {
 ; GFX67-LABEL: v_mul_add_1_v2i32:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX67-NEXT:    v_mul_lo_u32 v1, v1, v3
+; GFX67-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v3
+; GFX8-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v2, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: v_mul_add_1_v2i32:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v3
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_v2i32:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX900-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v1, v3, v[1:2]
+; GFX900-NEXT:    v_mov_b32_e32 v0, v4
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_v2i32:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[0:1]
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, v[4:5]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
-; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v3
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v1, v3, v[1:2]
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i32> %y, <i32 1, i32 1>
   %mul = mul <2 x i32> %x, %add
   ret <2 x i32> %mul
 }
 
 define <2 x i32> @v_mul_add_1_v2i32_commute(<2 x i32> %x, <2 x i32> %y) {
 ; GFX67-LABEL: v_mul_add_1_v2i32_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX67-NEXT:    v_mul_lo_u32 v1, v3, v1
+; GFX67-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i32_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX8-NEXT:    v_mul_lo_u32 v1, v3, v1
+; GFX8-NEXT:    v_mul_lo_u32 v2, v0, v2
+; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v2, v0
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX9-LABEL: v_mul_add_1_v2i32_commute:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v3, v1
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
+; GFX900-LABEL: v_mul_add_1_v2i32_commute:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v0, v2, v[0:1]
+; GFX900-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v1, v3, v[1:2]
+; GFX900-NEXT:    v_mov_b32_e32 v0, v4
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: v_mul_add_1_v2i32_commute:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[0:1]
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, v[4:5]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v2
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i32_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v0, v2, v0
-; GFX10-NEXT:    v_mul_lo_u32 v1, v3, v1
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v0, v2, v[0:1]
+; GFX10-NEXT:    v_mad_u64_u32 v[1:2], null, v1, v3, v[1:2]
+; GFX10-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i32> %y, <i32 1, i32 1>
   %mul = mul <2 x i32> %add, %x
   ret <2 x i32> %mul
 }
 
 define <2 x i32> @v_mul_add_x_v2i32(<2 x i32> %x, <2 x i32> %y) {
 ; GFX67-LABEL: v_mul_add_x_v2i32:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %mul = mul <2 x i32> %x, %sub
   ret <2 x i32> %mul
 }
 
 define <2 x i24> @v_mul_add_1_v2i24(<2 x i24> %x, <2 x i24> %y) {
 ; GFX67-LABEL: v_mul_add_1_v2i24:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX67-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i24:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX8-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX8-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i24:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX9-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX9-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i24:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
-; GFX10-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX10-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i24> %y, <i24 1, i24 1>
   %mul = mul <2 x i24> %x, %add
   ret <2 x i24> %mul
 }
 
 define <2 x i24> @v_mul_add_1_v2i24_commute(<2 x i24> %x, <2 x i24> %y) {
 ; GFX67-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT:    v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v2
-; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX67-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 1, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v2
-; GFX8-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX8-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX8-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX8-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX9-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX9-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i24_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
-; GFX10-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
+; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v2, v0
+; GFX10-NEXT:    v_mad_u32_u24 v1, v1, v3, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i24> %y, <i24 1, i24 1>
   %mul = mul <2 x i24> %add, %x
   ret <2 x i24> %mul
 }
 
 define <2 x i24> @v_mul_add_x_v2i24(<2 x i24> %x, <2 x i24> %y) {
 ; GFX67-LABEL: v_mul_add_x_v2i24:
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   ret i64 %add
 }
 
 define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) %i21, ptr addrspace(1) nocapture noundef writeonly align 4 %arg, i32 noundef %arg1) #1 {
 ; GFX67-LABEL: compute_mad:
 ; GFX67:       ; %bb.0: ; %bb
 ; GFX67-NEXT:    s_load_dword s3, s[0:1], 0x6
 ; GFX67-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX67-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4
 ; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX67-NEXT:    s_load_dword s6, s[6:7], 0x1
+; GFX67-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX67-NEXT:    s_add_i32 s3, s3, 1
 ; GFX67-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX67-NEXT:    s_and_b32 s6, s6, 0xffff
+; GFX67-NEXT:    s_mul_i32 s2, s2, s6
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, s3, v1
 ; GFX67-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4
-; GFX67-NEXT:    s_load_dword s3, s[6:7], 0x1
+; GFX67-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
 ; GFX67-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX67-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX67-NEXT:    s_and_b32 s3, s3, 0xffff
+; GFX67-NEXT:    s_mov_b32 s3, 0xf000
+; GFX67-NEXT:    s_mov_b32 s2, 0
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
-; GFX67-NEXT:    s_mul_i32 s2, s2, s3
 ; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX67-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
-; GFX67-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX67-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GFX67-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX67-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
-; GFX67-NEXT:    s_mov_b32 s3, 0xf000
-; GFX67-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-NEXT:    v_mov_b32_e32 v2, s5
+; GFX67-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX67-NEXT:    v_add_i32_e32 v3, vcc, v3, v1
+; GFX67-NEXT:    v_mul_lo_u32 v4, v3, v1
 ; GFX67-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
-; GFX67-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX67-NEXT:    v_addc_u32_e32 v1, vcc, 0, v4, vcc
+; GFX67-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX67-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
-; GFX67-NEXT:    s_mov_b32 s2, 0
+; GFX67-NEXT:    v_add_i32_e32 v2, vcc, v4, v3
 ; GFX67-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX67-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: compute_mad:
 ; GFX8:       ; %bb.0: ; %bb
 ; GFX8-NEXT:    s_load_dword s3, s[0:1], 0x18
 ; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x10
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX8-NEXT:    s_add_i32 s3, s3, 1
 ; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s3, v1
 ; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x10
 ; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x4
 ; GFX8-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_and_b32 s3, s3, 0xffff
+; GFX8-NEXT:    s_and_b32 s1, s3, 0xffff
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
-; GFX8-NEXT:    s_mul_i32 s2, s2, s3
+; GFX8-NEXT:    s_mul_i32 s2, s2, s1
 ; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
 ; GFX8-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 1, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT:    v_mul_lo_u32 v5, v3, v1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v0
-; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v4, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v3
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; GFX8-NEXT:    flat_store_dword v[0:1], v2
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX900-LABEL: compute_mad:
 ; GFX900:       ; %bb.0: ; %bb
 ; GFX900-NEXT:    s_load_dword s3, s[0:1], 0x18
 ; GFX900-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
 ; GFX900-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x10
 ; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-NEXT:    s_add_i32 s3, s3, 1
 ; GFX900-NEXT:    v_mul_lo_u32 v1, s3, v0
-; GFX900-NEXT:    v_mov_b32_e32 v4, s9
+; GFX900-NEXT:    v_mov_b32_e32 v5, s9
 ; GFX900-NEXT:    v_add_u32_e32 v2, s3, v1
 ; GFX900-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX900-NEXT:    v_add_u32_e32 v1, 1, v1
 ; GFX900-NEXT:    s_load_dword s3, s[6:7], 0x4
 ; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; GFX900-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX900-NEXT:    s_mul_i32 s2, s2, s3
 ; GFX900-NEXT:    v_add_u32_e32 v1, v3, v1
 ; GFX900-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX900-NEXT:    v_add_u32_e32 v2, 1, v3
 ; GFX900-NEXT:    v_add_u32_e32 v0, s2, v0
-; GFX900-NEXT:    v_mov_b32_e32 v5, s1
+; GFX900-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX900-NEXT:    v_mul_lo_u32 v3, v1, v2
-; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX900-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX900-NEXT:    v_mul_lo_u32 v2, v2, v1
-; GFX900-NEXT:    v_add_u32_e32 v1, 1, v3
-; GFX900-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX900-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
-; GFX900-NEXT:    v_add_u32_e32 v2, 1, v2
-; GFX900-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX900-NEXT:    v_mul_lo_u32 v2, v3, v2
-; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s8, v0
-; GFX900-NEXT:    v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
-; GFX900-NEXT:    global_store_dword v[0:1], v2, off
+; GFX900-NEXT:    v_mul_lo_u32 v1, v2, v1
+; GFX900-NEXT:    v_mad_u64_u32 v[2:3], s[2:3], v1, v3, v[1:2]
+; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, s0, v0
+; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; GFX900-NEXT:    v_lshlrev_b64 v[3:4], 2, v[3:4]
+; GFX900-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v2, v1, v[2:3]
+; GFX900-NEXT:    v_add_co_u32_e32 v1, vcc, s8, v3
+; GFX900-NEXT:    v_addc_co_u32_e32 v2, vcc, v5, v4, vcc
+; GFX900-NEXT:    global_store_dword v[1:2], v0, off
 ; GFX900-NEXT:    s_endpgm
 ;
 ; GFX90A-LABEL: compute_mad:
 ; GFX90A:       ; %bb.0: ; %bb
 ; GFX90A-NEXT:    s_load_dword s3, s[0:1], 0x18
 ; GFX90A-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
 ; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x10
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_add_i32 s3, s3, 1
 ; GFX90A-NEXT:    v_mul_lo_u32 v1, s3, v0
 ; GFX90A-NEXT:    v_add_u32_e32 v2, s3, v1
 ; GFX90A-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX90A-NEXT:    v_add_u32_e32 v1, 1, v1
-; GFX90A-NEXT:    s_load_dword s3, s[6:7], 0x4
 ; GFX90A-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GFX90A-NEXT:    v_add_u32_e32 v1, v3, v1
 ; GFX90A-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX90A-NEXT:    v_add_u32_e32 v2, 1, v3
-; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; GFX90A-NEXT:    v_mul_lo_u32 v3, v1, v2
 ; GFX90A-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX90A-NEXT:    s_load_dword s3, s[6:7], 0x4
+; GFX90A-NEXT:    v_mul_lo_u32 v2, v2, v1
+; GFX90A-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], v2, v3, v[2:3]
+; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_and_b32 s3, s3, 0xffff
-; GFX90A-NEXT:    v_mul_lo_u32 v1, v2, v1
-; GFX90A-NEXT:    v_add_u32_e32 v2, 1, v3
 ; GFX90A-NEXT:    s_mul_i32 s2, s2, s3
-; GFX90A-NEXT:    v_add_u32_e32 v3, 1, v1
-; GFX90A-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; GFX90A-NEXT:    v_add_u32_e32 v0, s2, v0
-; GFX90A-NEXT:    v_mul_lo_u32 v2, v1, v3
+; GFX90A-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v2, v[4:5]
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX90A-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, s9
 ; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, s8, v0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
 ; GFX90A-NEXT:    global_store_dword v[0:1], v2, off

 ; GFX10-NEXT:    s_load_dword s3, s[6:7], 0x4
 ; GFX10-NEXT:    v_mul_lo_u32 v2, v2, v0
 ; GFX10-NEXT:    v_mul_lo_u32 v3, v2, v1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v3, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX10-NEXT:    v_mul_lo_u32 v2, v1, v2
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v3, v2, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, v3, v1
+; GFX10-NEXT:    v_mul_lo_u32 v4, v2, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v4, v1
 ; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, s2, s3, v[0:1]
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_mul_lo_u32 v2, v4, v2
-; GFX10-NEXT:    v_add_co_u32 v0, s2, s4, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, s5, 0, s2
-; GFX10-NEXT:    v_mul_lo_u32 v3, v2, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX10-NEXT:    v_mul_lo_u32 v2, v3, v2
-; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
-; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    v_mul_lo_u32 v1, v3, v2
+; GFX10-NEXT:    v_add_co_u32 v2, s2, s4, v0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, 0, s2
+; GFX10-NEXT:    v_mad_u64_u32 v[4:5], null, v1, v4, v[1:2]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v1, v[4:5]
+; GFX10-NEXT:    v_add_co_u32 v1, vcc_lo, s0, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, s1, v3, vcc_lo
+; GFX10-NEXT:    global_store_dword v[1:2], v0, off
 ; GFX10-NEXT:    s_endpgm
 bb:
   %i = tail call i32 @llvm.amdgcn.workitem.id.x(), !range !0
   %i2 = add i32 %arg1, 1
   %i3 = mul i32 %i2, %i
   %i4 = add i32 %i3, %i2
   %i5 = mul i32 %i4, %i
   %i6 = add i32 %i3, 1

 ; GFX67-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %x, %add
   ret i8 %mul
 }
 
 define i8 @v_mul_add_1_i8_commute(i8 %x, i8 %y) {
 ; GFX67-LABEL: v_mul_add_1_i8_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX67-NEXT:    v_and_b32_e32 v0, 0xff, v0
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i8_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %add, %x
   ret i8 %mul
 }
 
 define i8 @v_mul_add_1_i8_zext(i8 zeroext %x, i8 zeroext %y) {
 ; GFX67-LABEL: v_mul_add_1_i8_zext:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i8_zext:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8_zext:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8_zext:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %x, %add
   ret i8 %mul
 }
 
 define i8 @v_mul_add_1_i8_zext_commute(i8 zeroext %x, i8 zeroext %y) {
 ; GFX67-LABEL: v_mul_add_1_i8_zext_commute:
 ; GFX67:       ; %bb.0:
 ; GFX67-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX67-NEXT:    v_add_i32_e32 v1, vcc, 1, v1
 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v1, v0
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_i8_zext_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_i8_zext_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v1, 1, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v1, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_i8_zext_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v1, v1, 1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v1, v0
+; GFX10-NEXT:    v_mad_u16 v0, v0, v1, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add i8 %y, 1
   %mul = mul i8 %add, %x
   ret i8 %mul
 }
 
 define <2 x i8> @v_mul_add_1_v2i8(<2 x i8> %x, <2 x i8> %y) {
 ; GFX67-LABEL: v_mul_add_1_v2i8:

 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v0, v2
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v1, v3
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i8:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX8-NEXT:    v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-NEXT:    v_mad_u16 v1, v1, v3, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u16_e32 v1, v1, v3
+; GFX9-NEXT:    v_mad_legacy_u16 v1, v1, v3, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v2
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v2, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v3, v3, 1
-; GFX10-NEXT:    v_add_nc_u16 v2, v2, 1
-; GFX10-NEXT:    v_mul_lo_u16 v1, v1, v3
-; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT:    v_mad_u16 v1, v1, v3, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX10-NEXT:    v_lshlrev_b16 v2, 8, v1
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i8> %y, <i8 1, i8 1>
   %mul = mul <2 x i8> %x, %add
   ret <2 x i8> %mul
 }

 ; GFX67-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v0, v2, v0
 ; GFX67-NEXT:    v_mul_u32_u24_e32 v1, v3, v1
 ; GFX67-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_mul_add_1_v2i8_commute:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX8-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX8-NEXT:    v_mul_lo_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_mad_u16 v1, v1, v3, v1
 ; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v2, v0
+; GFX8-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_mul_add_1_v2i8_commute:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_add_u16_e32 v3, 1, v3
-; GFX9-NEXT:    v_add_u16_e32 v2, 1, v2
-; GFX9-NEXT:    v_mul_lo_u16_e32 v1, v3, v1
+; GFX9-NEXT:    v_mad_legacy_u16 v1, v1, v3, v1
 ; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v2, v0
+; GFX9-NEXT:    v_mad_legacy_u16 v0, v0, v2, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_mul_add_1_v2i8_commute:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_add_nc_u16 v3, v3, 1
-; GFX10-NEXT:    v_add_nc_u16 v2, v2, 1
-; GFX10-NEXT:    v_mul_lo_u16 v1, v3, v1
-; GFX10-NEXT:    v_mul_lo_u16 v0, v2, v0
+; GFX10-NEXT:    v_mad_u16 v1, v1, v3, v1
+; GFX10-NEXT:    v_mad_u16 v0, v0, v2, v0
 ; GFX10-NEXT:    v_lshlrev_b16 v2, 8, v1
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
 ; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = add <2 x i8> %y, <i8 1, i8 1>
   %mul = mul <2 x i8> %add, %x
   ret <2 x i8> %mul
 }