[AMDGPU] gfx1010: prefer V_MUL_LO_U32 over V_MUL_LO_I32

[rampitec]

Summary

GFX10 deprecates v_mul_lo_i32 instruction, so choose u32 form for
all targets.

[arsenm]

What does the signedness of the multiply even mean?

[rampitec]

In D61525#1490186, @arsenm wrote:

What does the signedness of the multiply even mean?

It does not. There is problem with the instruction itself.

[nhaehnle]

In D61525#1490187, @rampitec wrote:

In D61525#1490186, @arsenm wrote:

What does the signedness of the multiply even mean?

It does not. There is problem with the instruction itself.

I believe the goal is to generally use the u32 suffix for arithmetic instructions where the signedness doesn't matter, but the ISA has generally been inconsistent about that and the i32 vs. u32 confusion here is simply because they simultaneously (1) changed some encodings around and (2) changed the canonical mnemonic of the instruction. So the _i32 and _u32 variants are really the same instruction semantically.

(The sign does matter for mul_hi)

[arsenm]

Can this just use the u32 version on all targets then?

[rampitec]

In D61525#1490811, @arsenm wrote:

Can this just use the u32 version on all targets then?

It's sp3 compatibility. I am bot sure u32 is also available in all ASICs.

[rampitec]

In D61525#1490852, @rampitec wrote:

In D61525#1490811, @arsenm wrote:

Can this just use the u32 version on all targets then?

It's sp3 compatibility. I am bot sure u32 is also available in all ASICs.

OK, I have checked. All ASICs support u32 version. I will change it.

[rampitec]

Unconditionally use V_MUL_LO_U32.

[kzhuravl]

LGTM

lib/Target/AMDGPU/SIInstrInfo.cpp

case AMDGPU::S_SUB_I32:
     return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
     // FIXME: These are not consistently handled, and selected when the carry is
     // used.
   case AMDGPU::S_ADD_U32:
     return AMDGPU::V_ADD_I32_e32;
   case AMDGPU::S_SUB_U32:
     return AMDGPU::V_SUB_I32_e32;
   case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
-  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32;
+  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
   case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
   case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
   case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
   case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
   case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
   case AMDGPU::S_XNOR_B32:
     return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
   case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;

test/CodeGen/AMDGPU/add3.ll

; GFX9-NEXT:    ; return to shader part epilog
   ret float %bc
 }
 
 define amdgpu_ps <2 x float> @add3_multiuse_outer(i32 %a, i32 %b, i32 %c, i32 %x) {
 ; VI-LABEL: add3_multiuse_outer:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
-; VI-NEXT:    v_mul_lo_i32 v1, v0, v3
+; VI-NEXT:    v_mul_lo_u32 v1, v0, v3
 ; VI-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: add3_multiuse_outer:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    v_add3_u32 v0, v0, v1, v2
-; GFX9-NEXT:    v_mul_lo_i32 v1, v0, v3
+; GFX9-NEXT:    v_mul_lo_u32 v1, v0, v3
 ; GFX9-NEXT:    ; return to shader part epilog
   %inner = add i32 %a, %b
   %outer = add i32 %inner, %c
   %x1 = mul i32 %outer, %x
   %r1 = insertelement <2 x i32> undef, i32 %outer, i32 0
   %r0 = insertelement <2 x i32> %r1, i32 %x1, i32 1
   %bc = bitcast <2 x i32> %r0 to <2 x float>
   ret <2 x float> %bc

test/CodeGen/AMDGPU/frame-index-elimination.ll

 ; GCN: s_sub_u32 s6, s5, s4
 
 ; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
 ; CI-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
 
 ; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
 ; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
 
-; GCN-NEXT: v_mul_lo_i32 v0, v0, 9
+; GCN-NEXT: v_mul_lo_u32 v0, v0, 9
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @func_other_fi_user_i32() #0 {
   %alloca = alloca [2 x i32], align 4, addrspace(5)
   %ptrtoint = ptrtoint [2 x i32] addrspace(5)* %alloca to i32
   %mul = mul i32 %ptrtoint, 9
   store volatile i32 %mul, i32 addrspace(3)* undef
   ret void

 ; GCN-DAG: s_movk_i32 s6, 0x204
 
 ; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
 ; CI: v_add_i32_e64 [[VZ:v[0-9]+]], s[6:7], s6, [[SCALED]]
 
 ; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
 ; GFX9: v_add_u32_e32 [[VZ:v[0-9]+]], s6, [[SCALED]]
 
-; GCN: v_mul_lo_i32 [[VZ]], [[VZ]], 9
+; GCN: v_mul_lo_u32 [[VZ]], [[VZ]], 9
 ; GCN: ds_write_b32 v0, [[VZ]]
 define void @func_other_fi_user_non_inline_imm_offset_i32() #0 {
   %alloca0 = alloca [128 x i32], align 4, addrspace(5)
   %alloca1 = alloca [8 x i32], align 4, addrspace(5)
   %gep0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca0, i32 0, i32 65
   %gep1 = getelementptr inbounds [8 x i32], [8 x i32] addrspace(5)* %alloca1, i32 0, i32 0
   store volatile i32 7, i32 addrspace(5)* %gep0
   %ptrtoint = ptrtoint i32 addrspace(5)* %gep1 to i32
   %mul = mul i32 %ptrtoint, 9
   store volatile i32 %mul, i32 addrspace(3)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32_vcc_live:
 ; GCN: s_sub_u32 [[DIFF:s[0-9]+]], s5, s4
 ; GCN-DAG: s_movk_i32 [[OFFSET:s[0-9]+]], 0x204
 
 ; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[DIFF]], 6
 ; CI: v_add_i32_e64 [[VZ:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, [[OFFSET]], [[SCALED]]
 
 ; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, [[DIFF]]
 ; GFX9: v_add_u32_e32 [[VZ:v[0-9]+]], [[OFFSET]], [[SCALED]]
 
-; GCN: v_mul_lo_i32 [[VZ]], [[VZ]], 9
+; GCN: v_mul_lo_u32 [[VZ]], [[VZ]], 9
 ; GCN: ds_write_b32 v0, [[VZ]]
 define void @func_other_fi_user_non_inline_imm_offset_i32_vcc_live() #0 {
   %alloca0 = alloca [128 x i32], align 4, addrspace(5)
   %alloca1 = alloca [8 x i32], align 4, addrspace(5)
   %vcc = call i64 asm sideeffect "; def $0", "={VCC}"()
   %gep0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca0, i32 0, i32 65
   %gep1 = getelementptr inbounds [8 x i32], [8 x i32] addrspace(5)* %alloca1, i32 0, i32 0
   store volatile i32 7, i32 addrspace(5)* %gep0

test/CodeGen/AMDGPU/idiv-licm.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}udiv32_invariant_denom:
 ; GCN:     v_cvt_f32_u32
 ; GCN:     v_rcp_iflag_f32
 ; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
 ; GCN:     v_cvt_u32_f32_e32
 ; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_i32
+; GCN-DAG: v_mul_lo_u32
 ; GCN-DAG: v_sub_i32_e32
 ; GCN-DAG: v_cmp_eq_u32_e64
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN-DAG: v_mul_hi_u32
 ; GCN-DAG: v_add_i32_e32
 ; GCN-DAG: v_subrev_i32_e32
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN:     [[LOOP:BB[0-9_]+]]:

 }
 
 ; GCN-LABEL: {{^}}urem32_invariant_denom:
 ; GCN:     v_cvt_f32_u32
 ; GCN:     v_rcp_iflag_f32
 ; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
 ; GCN:     v_cvt_u32_f32_e32
 ; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_i32
+; GCN-DAG: v_mul_lo_u32
 ; GCN-DAG: v_sub_i32_e32
 ; GCN-DAG: v_cmp_eq_u32_e64
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN-DAG: v_mul_hi_u32
 ; GCN-DAG: v_add_i32_e32
 ; GCN-DAG: v_subrev_i32_e32
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN:     [[LOOP:BB[0-9_]+]]:

 }
 
 ; GCN-LABEL: {{^}}sdiv32_invariant_denom:
 ; GCN:     v_cvt_f32_u32
 ; GCN:     v_rcp_iflag_f32
 ; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
 ; GCN:     v_cvt_u32_f32_e32
 ; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_i32
+; GCN-DAG: v_mul_lo_u32
 ; GCN-DAG: v_sub_i32_e32
 ; GCN-DAG: v_cmp_eq_u32_e64
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN-DAG: v_mul_hi_u32
 ; GCN-DAG: v_add_i32_e32
 ; GCN-DAG: v_subrev_i32_e32
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN:     [[LOOP:BB[0-9_]+]]:

 }
 
 ; GCN-LABEL: {{^}}srem32_invariant_denom:
 ; GCN:     v_cvt_f32_u32
 ; GCN:     v_rcp_iflag_f32
 ; GCN:     v_mul_f32_e32 v{{[0-9]+}}, 0x4f800000,
 ; GCN:     v_cvt_u32_f32_e32
 ; GCN-DAG: v_mul_hi_u32
-; GCN-DAG: v_mul_lo_i32
+; GCN-DAG: v_mul_lo_u32
 ; GCN-DAG: v_sub_i32_e32
 ; GCN-DAG: v_cmp_eq_u32_e64
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN-DAG: v_mul_hi_u32
 ; GCN-DAG: v_add_i32_e32
 ; GCN-DAG: v_subrev_i32_e32
 ; GCN-DAG: v_cndmask_b32_e64
 ; GCN:     [[LOOP:BB[0-9_]+]]:

test/CodeGen/AMDGPU/mad_64_32.ll

 ; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI %s
 
 ; GCN-LABEL: {{^}}mad_i64_i32_sextops:
 ; CI: v_mad_i64_i32 v[0:1], s[6:7], v0, v1, v[2:3]
 
-; SI: v_mul_lo_i32
+; SI: v_mul_lo_u32
 ; SI: v_mul_hi_i32
 ; SI: v_add_i32
 ; SI: v_addc_u32
 define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %mul, %arg2
   ret i64 %mad
 }
 
 ; GCN-LABEL: {{^}}mad_i64_i32_sextops_commute:
 ; CI: v_mad_i64_i32 v[0:1], s[6:7], v0, v1, v[2:3]
 
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_hi_i32
 ; SI: v_add_i32
 ; SI: v_addc_u32
 define i64 @mad_i64_i32_sextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %arg2, %mul
   ret i64 %mad
 }
 
 ; GCN-LABEL: {{^}}mad_u64_u32_zextops:
 ; CI: v_mad_u64_u32 v[0:1], s[6:7], v0, v1, v[2:3]
 
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_hi_u32
 ; SI: v_add_i32
 ; SI: v_addc_u32
 define i64 @mad_u64_u32_zextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
   %sext0 = zext i32 %arg0 to i64
   %sext1 = zext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %mul, %arg2
   ret i64 %mad
 }
 
 ; GCN-LABEL: {{^}}mad_u64_u32_zextops_commute:
 ; CI: v_mad_u64_u32 v[0:1], s[6:7], v0, v1, v[2:3]
 
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_hi_u32
 ; SI: v_add_i32
 ; SI: v_addc_u32
 define i64 @mad_u64_u32_zextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
   %sext0 = zext i32 %arg0 to i64
   %sext1 = zext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
   %mad = add i64 %arg2, %mul

test/CodeGen/AMDGPU/mul.i16.ll

define amdgpu_kernel void @s_mul_i16(i16 %a, i16 %b) {
   %r.val = mul i16 %a, %b
   store volatile i16 %r.val, i16 addrspace(1)* null
   ret void
 }
 
 ; FIXME: Should emit u16 mul here. Instead it's worse than SI
 ; GCN-LABEL: {{^}}v_mul_i16_uniform_load:
 ; SI: v_mul_u32_u24
-; GFX89: v_mul_lo_i32
+; GFX89: v_mul_lo_u32
 define amdgpu_kernel void @v_mul_i16_uniform_load(
     i16 addrspace(1)* %r,
     i16 addrspace(1)* %a,
     i16 addrspace(1)* %b) {
 entry:
   %a.val = load i16, i16 addrspace(1)* %a
   %b.val = load i16, i16 addrspace(1)* %b
   %r.val = mul i16 %a.val, %b.val
   store i16 %r.val, i16 addrspace(1)* %r
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_mul_v2i16:
-; SI: v_mul_lo_i32
-; SI: v_mul_lo_i32
+; SI: v_mul_lo_u32
+; SI: v_mul_lo_u32
 
 ; VI: v_mul_lo_u16_sdwa
 ; VI: v_mul_lo_u16_e32
 ; VI: v_or_b32_e32
 
 
 ; GFX9: s_waitcnt
 ; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
 ; GFX9-NEXT: s_setpc_b64
 define <2 x i16> @v_mul_v2i16(<2 x i16> %a, <2 x i16> %b) {
   %r.val = mul <2 x i16> %a, %b
   ret <2 x i16> %r.val
 }
 
 ; FIXME: Unpack garbage on gfx9
 ; GCN-LABEL: {{^}}v_mul_v3i16:
-; SI: v_mul_lo_i32
-; SI: v_mul_lo_i32
-; SI: v_mul_lo_i32
+; SI: v_mul_lo_u32
+; SI: v_mul_lo_u32
+; SI: v_mul_lo_u32
 
 ; VI: v_mul_lo_u16
 ; VI: v_mul_lo_u16
 ; VI: v_mul_lo_u16
 
 ; GFX9: s_waitcnt
 ; GFX9-NEXT: v_pk_mul_lo_u16
 ; GFX9-NEXT: v_pk_mul_lo_u16
 ; GFX9-NEXT: s_setpc_b64
 define <3 x i16> @v_mul_v3i16(<3 x i16> %a, <3 x i16> %b) {
   %r.val = mul <3 x i16> %a, %b
   ret <3 x i16> %r.val
 }
 
 ; GCN-LABEL: {{^}}v_mul_v4i16:
-; SI: v_mul_lo_i32
-; SI: v_mul_lo_i32
-; SI: v_mul_lo_i32
-; SI: v_mul_lo_i32
+; SI: v_mul_lo_u32
+; SI: v_mul_lo_u32
+; SI: v_mul_lo_u32
+; SI: v_mul_lo_u32
 
 ; VI: v_mul_lo_u16_sdwa
 ; VI: v_mul_lo_u16_e32
 ; VI: v_mul_lo_u16_sdwa
 ; VI: v_mul_lo_u16_e32
 ; VI: v_or_b32_e32
 ; VI: v_or_b32_e32
 
 ; GFX9: s_waitcnt
 ; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
 ; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v3
 ; GFX9-NEXT: s_setpc_b64
 define <4 x i16> @v_mul_v4i16(<4 x i16> %a, <4 x i16> %b) {
   %r.val = mul <4 x i16> %a, %b
   ret <4 x i16> %r.val
 }

test/CodeGen/AMDGPU/mul.ll

 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SI,FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,VI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9_10 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=FUNC,GFX9_10 %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=EG,FUNC %s
 
 ; mul24 and mad24 are affected
 
 ; FUNC-LABEL: {{^}}test_mul_v2i32:
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 
 define amdgpu_kernel void @test_mul_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
   %a = load <2 x i32>, <2 x i32> addrspace(1) * %in
   %b = load <2 x i32>, <2 x i32> addrspace(1) * %b_ptr
   %result = mul <2 x i32> %a, %b
   store <2 x i32> %result, <2 x i32> addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul_v4i32:
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_u32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 
 define amdgpu_kernel void @v_mul_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
   %a = load <4 x i32>, <4 x i32> addrspace(1) * %in
   %b = load <4 x i32>, <4 x i32> addrspace(1) * %b_ptr
   %result = mul <4 x i32> %a, %b
   store <4 x i32> %result, <4 x i32> addrspace(1)* %out
   ret void
define amdgpu_kernel void @s_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 %a, i64 %b) {
   %trunc = trunc i64 %mul to i32
   store i32 %trunc, i32 addrspace(1)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_trunc_i64_mul_to_i32:
 ; GCN: s_load_dword
 ; GCN: s_load_dword
-; GCN: v_mul_lo_i32
+; GCN: v_mul_lo_u32
 ; GCN: buffer_store_dword
 define amdgpu_kernel void @v_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %b = load i64, i64 addrspace(1)* %bptr, align 8
   %mul = mul i64 %b, %a
   %trunc = trunc i64 %mul to i32
   store i32 %trunc, i32 addrspace(1)* %out, align 8
   ret void
entry:
   %1 = mul i64 %0, 80
   store i64 %1, i64 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul64_sext_c:
 ; EG-DAG: MULLO_INT
 ; EG-DAG: MULHI_INT
-; GCN-DAG: v_mul_lo_i32
+; GCN-DAG: v_mul_lo_u32
 ; GCN-DAG: v_mul_hi_i32
 ; GCN: s_endpgm
 define amdgpu_kernel void @v_mul64_sext_c(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %val = load i32, i32 addrspace(1)* %in, align 4
   %ext = sext i32 %val to i64
   %mul = mul i64 %ext, 80
   store i64 %mul, i64 addrspace(1)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul64_sext_inline_imm:
-; GCN-DAG: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+; GCN-DAG: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, 9
 ; GCN-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
 ; GCN: s_endpgm
 define amdgpu_kernel void @v_mul64_sext_inline_imm(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %val = load i32, i32 addrspace(1)* %in, align 4
   %ext = sext i32 %val to i64
   %mul = mul i64 %ext, 9
   store i64 %mul, i64 addrspace(1)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}s_mul_i32:
 ; GCN: s_load_dword [[SRC0:s[0-9]+]],
 ; GCN: s_load_dword [[SRC1:s[0-9]+]],
 ; GCN: s_mul_i32 [[SRESULT:s[0-9]+]], [[SRC0]], [[SRC1]]
 ; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]]
 ; GCN: buffer_store_dword [[VRESULT]],
 ; GCN: s_endpgm
 define amdgpu_kernel void @s_mul_i32(i32 addrspace(1)* %out, [8 x i32], i32 %a, [8 x i32], i32 %b) nounwind {
   %mul = mul i32 %a, %b
   store i32 %mul, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul_i32:
-; GCN: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @v_mul_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %a = load i32, i32 addrspace(1)* %in
   %b = load i32, i32 addrspace(1)* %b_ptr
   %result = mul i32 %a, %b
   store i32 %result, i32 addrspace(1)* %out
   ret void
 }
 
 ; A standard 64-bit multiply.  The expansion should be around 6 instructions.
 ; It would be difficult to match the expansion correctly without writing
 ; a really complicated list of FileCheck expressions.  I don't want
 ; to confuse people who may 'break' this test with a correct optimization,
 ; so this test just uses FUNC-LABEL to make sure the compiler does not
 ; crash with a 'failed to select' error.
 
 ; FUNC-LABEL: {{^}}s_mul_i64:
-; GFX9-DAG: s_mul_i32
-; GFX9-DAG: s_mul_hi_u32
-; GFX9-DAG: s_mul_i32
-; GFX9-DAG: s_mul_i32
-; GFX9: s_endpgm
+; GFX9_10-DAG: s_mul_i32
+; GFX9_10-DAG: s_mul_hi_u32
+; GFX9_10-DAG: s_mul_i32
+; GFX9_10-DAG: s_mul_i32
+; GFX9_10: s_endpgm
 define amdgpu_kernel void @s_mul_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
   %mul = mul i64 %a, %b
   store i64 %mul, i64 addrspace(1)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul_i64:
-; GCN: v_mul_lo_i32
+; GCN: v_mul_lo_u32
 define amdgpu_kernel void @v_mul_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) {
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %b = load i64, i64 addrspace(1)* %bptr, align 8
   %mul = mul i64 %a, %b
   store i64 %mul, i64 addrspace(1)* %out, align 8
   ret void
 }
 
define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, [8 x i32], i128 %a, [8 x i32], i128 %b) nounwind #0 {
   store i128 %mul, i128 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}v_mul_i128:
 ; GCN: {{buffer|flat}}_load_dwordx4
 ; GCN: {{buffer|flat}}_load_dwordx4
 
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_add_i32_e32
 
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
+; SI-DAG: v_mul_lo_u32
+; SI-DAG: v_mul_lo_u32
+; SI-DAG: v_mul_lo_u32
+; SI-DAG: v_mul_lo_u32
 
-; VI-DAG: v_mul_lo_i32
+; VI-DAG: v_mul_lo_u32
 ; VI-DAG: v_mul_hi_u32
 ; VI: v_mad_u64_u32
 ; VI: v_mad_u64_u32
 ; VI: v_mad_u64_u32
 
 ; GCN: {{buffer|flat}}_store_dwordx4
 define amdgpu_kernel void @v_mul_i128(i128 addrspace(1)* %out, i128 addrspace(1)* %aptr, i128 addrspace(1)* %bptr) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()

test/CodeGen/AMDGPU/sdwa-peephole.ll

 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole=0 -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=NOSDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,SDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,SDWA,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,GFX89,SDWA,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_10,SDWA,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX9_10,SDWA,GCN %s
 
 ; GCN-LABEL: {{^}}add_shr_i32:
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST:[0-9]+]], 16, v{{[0-9]+}}
 ; NOSDWA: v_add_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
 ; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa
 
 ; VI: v_add_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9: v_add_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10: v_add_nc_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 
 define amdgpu_kernel void @add_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %a = load i32, i32 addrspace(1)* %in, align 4
   %shr = lshr i32 %a, 16
   %add = add i32 %a, %shr
   store i32 %add, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}sub_shr_i32:
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST:[0-9]+]], 16, v{{[0-9]+}}
 ; NOSDWA: v_subrev_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
 ; NOSDWA-NOT: v_subrev_{{(_co)?}}_u32_sdwa
 
 ; VI: v_subrev_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9: v_sub_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10: v_sub_nc_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 define amdgpu_kernel void @sub_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %a = load i32, i32 addrspace(1)* %in, align 4
   %shr = lshr i32 %a, 16
   %sub = sub i32 %shr, %a
   store i32 %sub, i32 addrspace(1)* %out, align 4
   ret void
 }
 
define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in1, i32 addrspace(1)* %in2) {
   %shra = lshr i32 %a, 16
   %shrb = lshr i32 %b, 16
   %mul = mul i32 %shra, %shrb
   store i32 %mul, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}mul_i16:
-; NOSDWA: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; SDWA: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX89: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; SDWA-NOT: v_mul_u32_u24_sdwa
 
 define amdgpu_kernel void @mul_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %ina, i16 addrspace(1)* %inb) {
 entry:
   %a = load i16, i16 addrspace(1)* %ina, align 4
   %b = load i16, i16 addrspace(1)* %inb, align 4
   %mul = mul i16 %a, %b
   store i16 %mul, i16 addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}mul_v2i16:
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
 ; NOSDWA: v_mul_u32_u24_e32 v[[DST_MUL:[0-9]+]], v[[DST0]], v[[DST1]]
 ; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
 ; NOSDWA-NOT: v_mul_u32_u24_sdwa
 
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 
-; GFX9: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
   %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
   %mul = mul <2 x i16> %a, %b
   store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4
   ret void

 
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 
-; GFX9-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @mul_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %ina, <4 x i16> addrspace(1)* %inb) {
 entry:
   %a = load <4 x i16>, <4 x i16> addrspace(1)* %ina, align 4
   %b = load <4 x i16>, <4 x i16> addrspace(1)* %inb, align 4
   %mul = mul <4 x i16> %a, %b
   store <4 x i16> %mul, <4 x i16> addrspace(1)* %out, align 4
   ret void

 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL5:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL6:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
 ; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL7:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL6]], v[[DST_MUL7]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL4]], v[[DST_MUL5]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 
-; GFX9-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @mul_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(1)* %ina, <8 x i16> addrspace(1)* %inb) {
 entry:
   %a = load <8 x i16>, <8 x i16> addrspace(1)* %ina, align 4
   %b = load <8 x i16>, <8 x i16> addrspace(1)* %inb, align 4
   %mul = mul <8 x i16> %a, %b
   store <8 x i16> %mul, <8 x i16> addrspace(1)* %out, align 4
   ret void

 ; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
 ; NOSDWA-NOT: v_mul_f16_sdwa
 
 ; VI-DAG: v_mul_f16_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_mul_f16_e32 v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; VI: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]]
 
-; GFX9: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @mul_v2half(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %ina, <2 x half> addrspace(1)* %inb) {
 entry:
   %a = load <2 x half>, <2 x half> addrspace(1)* %ina, align 4
   %b = load <2 x half>, <2 x half> addrspace(1)* %inb, align 4
   %mul = fmul <2 x half> %a, %b
   store <2 x half> %mul, <2 x half> addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}mul_v4half:
 ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; NOSDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_mul_f16_sdwa
 
 ; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-; GFX9-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @mul_v4half(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %ina, <4 x half> addrspace(1)* %inb) {
 entry:
   %a = load <4 x half>, <4 x half> addrspace(1)* %ina, align 4
   %b = load <4 x half>, <4 x half> addrspace(1)* %inb, align 4
   %mul = fmul <4 x half> %a, %b
   store <4 x half> %mul, <4 x half> addrspace(1)* %out, align 4
   ret void

 ; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_mul_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-; GFX9-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @mul_v8half(<8 x half> addrspace(1)* %out, <8 x half> addrspace(1)* %ina, <8 x half> addrspace(1)* %inb) {
 entry:
   %a = load <8 x half>, <8 x half> addrspace(1)* %ina, align 4
   %b = load <8 x half>, <8 x half> addrspace(1)* %inb, align 4
   %mul = fmul <8 x half> %a, %b
   store <8 x half> %mul, <8 x half> addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}mul_i8:
-; NOSDWA: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; NOSDWA: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_mul_u32_u24_sdwa
-; SDWA: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX89: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; SDWA-NOT: v_mul_u32_u24_sdwa
 
 define amdgpu_kernel void @mul_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %ina, i8 addrspace(1)* %inb) {
 entry:
   %a = load i8, i8 addrspace(1)* %ina, align 4
   %b = load i8, i8 addrspace(1)* %inb, align 4
   %mul = mul i8 %a, %b
   store i8 %mul, i8 addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}mul_v2i8:
 ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_mul_u32_u24_sdwa
 
 ; VI: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
 
 ; GFX9-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
 ; GFX9-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+
 ; GFX9: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 
+; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+
+
 define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) {
 entry:
   %a = load <2 x i8>, <2 x i8> addrspace(1)* %ina, align 4
   %b = load <2 x i8>, <2 x i8> addrspace(1)* %inb, align 4
   %mul = mul <2 x i8> %a, %b
   store <2 x i8> %mul, <2 x i8> addrspace(1)* %out, align 4
   ret void
 }

 ; VI-DAG: v_mul_u32_u24_sdwa
 ; VI-DAG: v_mul_u32_u24_sdwa
 ; VI-DAG: v_mul_u32_u24_sdwa
 
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+
 define amdgpu_kernel void @mul_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %ina, <4 x i8> addrspace(1)* %inb) {
 entry:
   %a = load <4 x i8>, <4 x i8> addrspace(1)* %ina, align 4
   %b = load <4 x i8>, <4 x i8> addrspace(1)* %inb, align 4
   %mul = mul <4 x i8> %a, %b
   store <4 x i8> %mul, <4 x i8> addrspace(1)* %out, align 4
   ret void
 }

 
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 ; GFX9-DAG: v_mul_lo_u16_sdwa
 
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+; GFX10-DAG: v_mul_lo_u16_e64
+
 define amdgpu_kernel void @mul_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(1)* %ina, <8 x i8> addrspace(1)* %inb) {
 entry:
   %a = load <8 x i8>, <8 x i8> addrspace(1)* %ina, align 4
   %b = load <8 x i8>, <8 x i8> addrspace(1)* %inb, align 4
   %mul = mul <8 x i8> %a, %b
   store <8 x i8> %mul, <8 x i8> addrspace(1)* %out, align 4
   ret void
 }

 ; NOSDWA: v_mac_f16_e32 v[[DST_MAC:[0-9]+]], v[[DST0]], v[[DST1]]
 ; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MAC]]
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
 ; NOSDWA-NOT: v_mac_f16_sdwa
 
 ; VI: v_mac_f16_sdwa v[[DST_MAC:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MAC]]
 
-; GFX9: v_pk_mul_f16 v[[DST_MUL:[0-9]+]], v{{[0-9]+}}, v[[SRC:[0-9]+]]
-; GFX9: v_pk_add_f16 v{{[0-9]+}}, v[[DST_MUL]], v[[SRC]]
+; GFX9_10: v_pk_mul_f16 v[[DST_MUL:[0-9]+]], v{{[0-9]+}}, v[[SRC:[0-9]+]]
+; GFX9_10: v_pk_add_f16 v{{[0-9]+}}, v[[DST_MUL]], v[[SRC]]
 
 define amdgpu_kernel void @mac_v2half(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %ina, <2 x half> addrspace(1)* %inb) {
 entry:
   %a = load <2 x half>, <2 x half> addrspace(1)* %ina, align 4
   %b = load <2 x half>, <2 x half> addrspace(1)* %inb, align 4
   %mul = fmul <2 x half> %a, %b
   %mac = fadd <2 x half> %mul, %b
   store <2 x half> %mac, <2 x half> addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}immediate_mul_v2i16:
 ; NOSDWA-NOT: v_mul_u32_u24_sdwa
 ; VI-DAG: v_mov_b32_e32 v[[M321:[0-9]+]], 0x141
 ; VI-DAG: v_mov_b32_e32 v[[M123:[0-9]+]], 0x7b
 ; VI-DAG: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M123]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-DAG: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M321]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 
 ; GFX9: s_mov_b32 s[[IMM:[0-9]+]], 0x141007b
 ; GFX9: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, s[[IMM]]
 
+; GFX10: v_pk_mul_lo_u16 v{{[0-9]+}}, 0x141007b, v{{[0-9]+}}
+
 define amdgpu_kernel void @immediate_mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %in, align 4
   %mul = mul <2 x i16> %a, <i16 123, i16 321>
   store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4
   ret void
 }
 
 ; Double use of same src - should not convert it
 ; GCN-LABEL: {{^}}mulmul_v2i16:
 ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_mul_u32_u24_sdwa
 
 ; VI: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 
-; GFX9: v_pk_mul_lo_u16 v[[DST1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; GFX9: v_pk_mul_lo_u16 v{{[0-9]+}}, v[[DST1]], v{{[0-9]+}}
+; GFX9_10: v_pk_mul_lo_u16 v[[DST1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v[[DST1]], v{{[0-9]+}}
 
 define amdgpu_kernel void @mulmul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
   %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
   %mul = mul <2 x i16> %a, %b
   %mul2 = mul <2 x i16> %mul, %b
   store <2 x i16> %mul2, <2 x i16> addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}add_bb_v2i16:
 ; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa
 
 ; VI: v_add_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 
-; GFX9: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9_10: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @add_bb_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
   %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
   br label %add_label
 add_label:
   %add = add <2 x i16> %a, %b

 ; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; NOSDWA-DAG: v_and_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; NOSDWA-NOT: v_and_b32_sdwa
 ; NOSDWA-NOT: v_or_b32_sdwa
 
 ; VI-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; SDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
+; GFX9_10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX89-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
+;
+; GFX10-DAG: v_lshrrev_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+;
 ; VI-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; SDWA-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; SDWA: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9_10-DAG: v_and_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX89-DAG: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
+;
+; GFX10-DAG: v_lshrrev_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+;
+; GFX89: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+;
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 
 define amdgpu_kernel void @pulled_out_test(<8 x i8> addrspace(1)* %sourceA, <8 x i8> addrspace(1)* %destValues) {
 entry:
   %idxprom = ashr exact i64 15, 32
   %arrayidx = getelementptr inbounds <8 x i8>, <8 x i8> addrspace(1)* %sourceA, i64 %idxprom
   %tmp = load <8 x i8>, <8 x i8> addrspace(1)* %arrayidx, align 8
 
   %tmp1 = extractelement <8 x i8> %tmp, i32 0

test/CodeGen/AMDGPU/sminmax.ll

define amdgpu_kernel void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
   store i32 %res2, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}v_abs_i32_repeat_user:
 ; SIVI: v_sub_{{i|u}}32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]]
 ; GFX9: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SRC:v[0-9]+]]
 ; GCN: v_max_i32_e32 [[MAX:v[0-9]+]], [[SRC]], [[NEG]]
-; GCN: v_mul_lo_i32 v{{[0-9]+}}, [[MAX]], [[MAX]]
+; GCN: v_mul_lo_u32 v{{[0-9]+}}, [[MAX]], [[MAX]]
 define amdgpu_kernel void @v_abs_i32_repeat_user(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
   %val = load i32, i32 addrspace(1)* %src, align 4
   %neg = sub i32 0, %val
   %cond = icmp sgt i32 %val, %neg
   %res = select i1 %cond, i32 %val, i32 %neg
   %mul = mul i32 %res, %res
   store i32 %mul, i32 addrspace(1)* %out, align 4
   ret void

test/CodeGen/AMDGPU/srem.ll

define amdgpu_kernel void @srem_i32_4(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %result = srem i32 %num, 4
   store i32 %result, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}srem_i32_7:
 ; SI: s_mov_b32 [[MAGIC:s[0-9]+]], 0x92492493
 ; SI: v_mul_hi_i32 {{v[0-9]+}}, {{v[0-9]+}}, [[MAGIC]]
-; SI: v_mul_lo_i32
+; SI: v_mul_lo_u32
 ; SI: v_sub_{{[iu]}}32
 ; SI: s_endpgm
 define amdgpu_kernel void @srem_i32_7(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %num = load i32, i32 addrspace(1) * %in
   %result = srem i32 %num, 7
   store i32 %result, i32 addrspace(1)* %out
   ret void
 }

test/CodeGen/AMDGPU/udivrem.ll

 ; EG-DAG: CNDE_INT
 ; EG-DAG: ADD_INT
 ; EG-DAG: SUB_INT
 ; EG-DAG: CNDE_INT
 ; EG-DAG: CNDE_INT
 
 ; SI: v_rcp_iflag_f32_e32 [[RCP:v[0-9]+]]
 ; SI-DAG: v_mul_hi_u32 [[RCP_HI:v[0-9]+]], [[RCP]]
-; SI-DAG: v_mul_lo_i32 [[RCP_LO:v[0-9]+]], [[RCP]]
+; SI-DAG: v_mul_lo_u32 [[RCP_LO:v[0-9]+]], [[RCP]]
 ; SI-DAG: v_sub_{{[iu]}}32_e32 [[NEG_RCP_LO:v[0-9]+]], vcc, 0, [[RCP_LO]]
 ; SI: v_cmp_eq_u32_e64 [[CC1:s\[[0-9:]+\]]], 0, [[RCP_HI]]
 ; SI: v_cndmask_b32_e64 [[CND1:v[0-9]+]], [[RCP_LO]], [[NEG_RCP_LO]], [[CC1]]
 ; SI: v_mul_hi_u32 [[E:v[0-9]+]], [[CND1]], [[RCP]]
 ; SI-DAG: v_add_{{[iu]}}32_e32 [[RCP_A_E:v[0-9]+]], vcc, [[E]], [[RCP]]
 ; SI-DAG: v_subrev_{{[iu]}}32_e32 [[RCP_S_E:v[0-9]+]], vcc, [[E]], [[RCP]]
 ; SI: v_cndmask_b32_e64 [[CND2:v[0-9]+]], [[RCP_S_E]], [[RCP_A_E]], [[CC1]]
 ; SI: v_mul_hi_u32 [[Quotient:v[0-9]+]], [[CND2]],
-; SI: v_mul_lo_i32 [[Num_S_Remainder:v[0-9]+]], [[CND2]]
+; SI: v_mul_lo_u32 [[Num_S_Remainder:v[0-9]+]], [[CND2]]
 ; SI-DAG: v_add_{{[iu]}}32_e32 [[Quotient_A_One:v[0-9]+]], vcc, 1, [[Quotient]]
 ; SI-DAG: v_sub_{{[iu]}}32_e32 [[Remainder:v[0-9]+]], vcc, {{[vs][0-9]+}}, [[Num_S_Remainder]]
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_subrev_{{[iu]}}32_e32 [[Quotient_S_One:v[0-9]+]],
 ; SI-DAG: v_subrev_{{[iu]}}32_e32 [[Remainder_S_Den:v[0-9]+]],
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32 [[Remainder_A_Den:v[0-9]+]],

 ; EG-DAG: CNDE_INT
 
 ; For SI, we used to have checks for the input and output registers
 ; of the instructions, but these are way too fragile.  The division for
 ; the two vector elements can be intermixed which makes it impossible to
 ; accurately check all the operands.
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32

 ; EG-DAG: CNDE_INT
 ; EG-DAG: ADD_INT
 ; EG-DAG: SUB_INT
 ; EG-DAG: CNDE_INT
 ; EG-DAG: CNDE_INT
 
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_lo_u32
 ; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_add_{{[iu]}}32_e32
 ; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-NOT: v_and_b32
 ; SI: s_endpgm
 define amdgpu_kernel void @test_udivrem_v4(<4 x i32> addrspace(1)* %out, <4 x i32> %x, <4 x i32> %y) {
   %result0 = udiv <4 x i32> %x, %y
   store <4 x i32> %result0, <4 x i32> addrspace(1)* %out
   %result1 = urem <4 x i32> %x, %y
   store <4 x i32> %result1, <4 x i32> addrspace(1)* %out
   ret void
 }

test/CodeGen/AMDGPU/urem.ll

define amdgpu_kernel void @test_urem_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   store i32 %result, i32 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}test_urem_i32_7:
 ; SI: s_mov_b32 [[MAGIC:s[0-9]+]], 0x24924925
 ; SI: v_mul_hi_u32 {{v[0-9]+}}, {{v[0-9]+}}, [[MAGIC]]
 ; SI: v_subrev_{{[iu]}}32
-; SI: v_mul_lo_i32
+; SI: v_mul_lo_u32
 ; SI: v_sub_{{[iu]}}32
 ; SI: buffer_store_dword
 ; SI: s_endpgm
 define amdgpu_kernel void @test_urem_i32_7(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %num = load i32, i32 addrspace(1) * %in
   %result = urem i32 %num, 7
   store i32 %result, i32 addrspace(1)* %out
   ret void

test/CodeGen/AMDGPU/wqm.ll

 }
 
 ; Check that WQM is re-enabled when required.
 ;
 ;CHECK-LABEL: {{^}}test4:
 ;CHECK-NEXT: ; %main_body
 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
 ;CHECK-NEXT: s_wqm_b64 exec, exec
-;CHECK: v_mul_lo_i32 [[MUL:v[0-9]+]], v0, v1
+;CHECK: v_mul_lo_u32 [[MUL:v[0-9]+]], v0, v1
 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
 ;CHECK: store
 ;CHECK: s_wqm_b64 exec, exec
 ;CHECK: image_sample
 ;CHECK: image_sample
 define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) {
 main_body:
   %c.1 = mul i32 %c, %d

test/CodeGen/AMDGPU/wwm-reserved.ll

; GFX9-O0: v_cmp_eq_u32_e64 s{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}, v[[FIRST]], v[[SECOND]]
   %tmp145 = bitcast i32 %tmp141 to float
   call void @llvm.amdgcn.raw.buffer.store.f32(float %tmp145, <4 x i32> %tmp14, i32 4, i32 0, i32 0)
   ret void
 }
 
 define i32 @called(i32 %a) noinline {
 ; GFX9: v_add_u32_e32 v1, v0, v0
   %add = add i32 %a, %a
-; GFX9: v_mul_lo_i32 v0, v1, v0
+; GFX9: v_mul_lo_u32 v0, v1, v0
   %mul = mul i32 %add, %a
 ; GFX9: v_sub_u32_e32 v0, v0, v1
   %sub = sub i32 %mul, %add
   ret i32 %sub
 }
 
 define amdgpu_kernel void @call(<4 x i32> inreg %tmp14, i32 inreg %arg) {
 ; GFX9-O0: v_mov_b32_e32 v0, s2