This is an archive of the discontinued LLVM Phabricator instance.

Differential D97062

[AMDGPU] Use divergent addresses for vector loads
ClosedPublic

Authored by foad on Feb 19 2021, 9:29 AM.

Details

Reviewers
arsenm
rampitec
Commits
rGfdaa2d02591b: [AMDGPU] Use divergent addresses for vector loads
Summary

Change some test cases to use divergent addresses for vector loads,
which should be the common case in real world code. Using uniform
addresses causes poor instruction selection for the surrounding
code which has to be fixed up post-register-allocation, and this causes
a lot of testsuite churn for a forthcoming patch to stop selecting
24-bit vector multiply instructions for uniform multiplies.

This shows up some problems in the idot tests where we fail to select
v_dot instructions because the patterns only match MUL_[UI]24 ISD nodes,
but the DAG contains i16 mul nodes instead.

Diff Detail

Unit TestsFailed

TimeTest
80 msx64 debian > Polly.Isl/CodeGen::scev-backedgetaken.ll
800 msx64 windows > MLIR.Examples/Toy/Ch6::jit.toy
750 msx64 windows > MLIR.Examples/Toy/Ch7::jit.toy

Event Timeline

foad created this revision.Feb 19 2021, 9:29 AM
Herald added subscribers: kerbowa, t-tye, tpr and 5 others. · View Herald TranscriptFeb 19 2021, 9:29 AM
foad requested review of this revision.Feb 19 2021, 9:29 AM
Herald added a project: Restricted Project. · View Herald TranscriptFeb 19 2021, 9:29 AM
Herald added subscribers: llvm-commits, wdng. · View Herald Transcript
foad added a child revision: D97063: [AMDGPU][SelectionDAG] Don't combine uniform multiplies to MUL_[UI]24.Feb 19 2021, 9:29 AM
arsenm accepted this revision.Feb 19 2021, 9:30 AM
This revision is now accepted and ready to land.Feb 19 2021, 9:30 AM
foad added a comment.Feb 19 2021, 9:32 AM

This shows up some problems in the idot tests where we fail to select
v_dot instructions because the patterns only match MUL_[UI]24 ISD nodes,
but the DAG contains i16 mul nodes instead.

Is selecting these from patterns actually an important use case? Or does everyone use the intrinsics instead?

Harbormaster completed remote builds in B89937: Diff 325014.Feb 19 2021, 10:02 AM
This revision was landed with ongoing or failed builds.Feb 23 2021, 5:33 AM
Closed by commit rGfdaa2d02591b: [AMDGPU] Use divergent addresses for vector loads (authored by foad). · Explain Why
This revision was automatically updated to reflect the committed changes.
foad added a commit: rGfdaa2d02591b: [AMDGPU] Use divergent addresses for vector loads.

Revision Contents

PathSize
llvm/
test/
CodeGen/
AMDGPU/
2859 lines
1341 lines
2651 lines
4448 lines
4688 lines
24 lines
171 lines

Diff 325014

llvm/test/CodeGen/AMDGPU/idot2.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX8 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-NODL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-NODL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX9-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-DL %s
; add(mul(S0.x, S1.y), ; add(mul(S0.x, S1.y),
; add (mul (S0.y, S1.y), S3)) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3) ; add (mul (S0.y, S1.y), S3)) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2: ; GFX7-LABEL: udot2:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s4
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: s_and_b32 s4, s4, s8
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2: ; GFX8-LABEL: udot2:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2: ; GFX9-NODL-LABEL: udot2:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2: ; GFX9-DL-LABEL: udot2:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0 ; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2: ; GFX10-DL-LABEL: udot2:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul nuw i32 %conv2, %conv %mul1 = mul nuw i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 12 Lines
; TODO: Support this pattern ; TODO: Support this pattern
; add(S3, ; add(S3,
; add (mul (S0.y, S1.y), mul (S0.y, S1.y))) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3) ; add (mul (S0.y, S1.y), mul (S0.y, S1.y))) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)
define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_MulMul: ; GFX7-LABEL: udot2_MulMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_and_b32 s4, s4, s8 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v0, s4 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_and_b32 s5, s5, s8 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mul_u32_u24_e32 v0, s5, v0 ; GFX7-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: v_mul_u32_u24_e32 v0, v0, v2
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, s4, v0 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, s5, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_MulMul: ; GFX8-LABEL: udot2_MulMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s6 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mul_u32_u24_e32 v0, s2, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s5, v0 ; GFX8-NEXT: v_mul_u32_u24_sdwa v1, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_MulMul: ; GFX9-NODL-LABEL: udot2_MulMul:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, s0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v1, s5, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
; GFX9-NODL-NEXT: v_add_u32_e32 v1, s9, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_MulMul: ; GFX9-DL-LABEL: udot2_MulMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, s0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v1, s5, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
; GFX9-DL-NEXT: v_add_u32_e32 v1, s9, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_MulMul: ; GFX10-DL-LABEL: udot2_MulMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_and_b32 s5, s0, s4 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_and_b32 s4, s1, s4 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 16 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: v_mul_u32_u24_e64 v0, s4, s5 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16 ; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, s2
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, s8, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul nuw i32 %conv2, %conv %mul1 = mul nuw i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 9 Lines
} }
define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: idot2: ; GFX7-LABEL: idot2:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i16 s7, s4
; GFX7-NEXT: s_ashr_i32 s4, s4, 16
; GFX7-NEXT: s_sext_i32_i16 s8, s5
; GFX7-NEXT: s_ashr_i32 s5, s5, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot2: ; GFX8-LABEL: idot2:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_sext_i32_i16 s5, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_ashr_i32 s2, s2, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_sext_i32_i16 s6, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_ashr_i32 s3, s3, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s4 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mov_b32_e32 v2, s5 ; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0 ; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot2: ; GFX9-NODL-LABEL: idot2:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot2: ; GFX9-DL-LABEL: idot2:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0 ; GFX9-DL-NEXT: v_dot2_i32_i16 v0, v3, v2, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot2_i32_i16 v1, s1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot2: ; GFX10-DL-LABEL: idot2:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_dot2_i32_i16 v0, s1, s0, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_dot2_i32_i16 v1, v2, v1, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32 %conv = sext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = sext i16 %s2.elt1 to i32 %conv2 = sext i16 %s2.elt1 to i32
%mul1 = mul nuw i32 %conv2, %conv %mul1 = mul nuw i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 10 Lines
} }
define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: idot2_MixedTypedMul: ; GFX7-LABEL: idot2_MixedTypedMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s4
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v1
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s7, s4, 16
; GFX7-NEXT: s_lshr_b32 s8, s5, 16
; GFX7-NEXT: s_sext_i32_i16 s4, s4
; GFX7-NEXT: v_mov_b32_e32 v0, s7
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v0, v1
; GFX7-NEXT: s_sext_i32_i16 s5, s5
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_i32_i24 v0, s5, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot2_MixedTypedMul: ; GFX8-LABEL: idot2_MixedTypedMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_sext_i32_i16 s5, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_lshr_b32 s2, s2, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_sext_i32_i16 s6, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s4 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mov_b32_e32 v2, s5 ; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
; GFX8-NEXT: v_mad_u32_u24 v0, s3, v1, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s2
; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot2_MixedTypedMul: ; GFX9-NODL-LABEL: idot2_MixedTypedMul:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot2_MixedTypedMul: ; GFX9-DL-LABEL: idot2_MixedTypedMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v2, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot2_MixedTypedMul: ; GFX10-DL-LABEL: idot2_MixedTypedMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_lshr_b32 s4, s0, 16 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_lshr_b32 s5, s1, 16 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_sext_i32_i16 s1, s1 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32 %conv = sext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = sext i16 %s2.elt1 to i32 %conv2 = sext i16 %s2.elt1 to i32
%mul1 = mul nuw i32 %conv2, %conv %mul1 = mul nuw i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 9 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_alt_AddOperands: ; GFX7-LABEL: udot2_alt_AddOperands:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s4
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: s_and_b32 s4, s4, s8
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_alt_AddOperands: ; GFX8-LABEL: udot2_alt_AddOperands:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_alt_AddOperands: ; GFX9-NODL-LABEL: udot2_alt_AddOperands:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_mov_b32 s0, 0xffff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_and_b32_e32 v3, s0, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v2
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v4, v3, v1
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_alt_AddOperands: ; GFX9-DL-LABEL: udot2_alt_AddOperands:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0 ; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_alt_AddOperands: ; GFX10-DL-LABEL: udot2_alt_AddOperands:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul nuw i32 %conv2, %conv %mul1 = mul nuw i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 10 Lines
} }
define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: idot2_MixedExt: ; GFX7-LABEL: idot2_MixedExt:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v0
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i16 s7, s4
; GFX7-NEXT: s_ashr_i32 s4, s4, 16
; GFX7-NEXT: s_and_b32 s8, s5, 0xffff
; GFX7-NEXT: s_ashr_i32 s5, s5, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot2_MixedExt: ; GFX8-LABEL: idot2_MixedExt:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_sext_i32_i16 s5, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_ashr_i32 s2, s2, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s6, s3, 0xffff ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_ashr_i32 s3, s3, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s4 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mov_b32_e32 v2, s5 ; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0 ; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v0
; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot2_MixedExt: ; GFX9-NODL-LABEL: idot2_MixedExt:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-NODL-NEXT: s_and_b32 s5, s1, 0xffff
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot2_MixedExt: ; GFX9-DL-LABEL: idot2_MixedExt:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-DL-NEXT: s_and_b32 s5, s1, 0xffff
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot2_MixedExt: ; GFX10-DL-LABEL: idot2_MixedExt:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_ashr_i32 s4, s0, 16 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_ashr_i32 s5, s1, 16 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_and_b32 s1, s1, 0xffff ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32 %conv = sext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul nuw i32 %conv2, %conv %mul1 = mul nuw i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 10 Lines
} }
define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: notudot2_SameVec: ; GFX7-LABEL: notudot2_SameVec:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v0, s4
; GFX7-NEXT: s_load_dword s7, s[0:1], 0x0 ; GFX7-NEXT: v_mad_u32_u24 v0, v1, v1, v0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s5, s6, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s7
; GFX7-NEXT: s_and_b32 s4, s4, 0xffff
; GFX7-NEXT: v_mad_u32_u24 v0, s5, s5, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s4, s4, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: notudot2_SameVec: ; GFX8-LABEL: notudot2_SameVec:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: flat_load_dword v2, v[2:3]
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshr_b32 s2, s2, 16 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v1, s2
; GFX8-NEXT: v_mov_b32_e32 v0, s3 ; GFX8-NEXT: v_mad_u32_u24 v2, v0, v0, v1
; GFX8-NEXT: s_and_b32 s4, s4, 0xffff
; GFX8-NEXT: v_mad_u32_u24 v0, s2, s2, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s4, s4, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: notudot2_SameVec: ; GFX9-NODL-LABEL: notudot2_SameVec:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, s0, v1
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NODL-NEXT: s_and_b32 s0, s0, 0xffff
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, s1, v1
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, s0, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: notudot2_SameVec: ; GFX9-DL-LABEL: notudot2_SameVec:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v2, s0, v1
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-DL-NEXT: s_and_b32 s0, s0, 0xffff
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, s1, v1
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, s0, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: notudot2_SameVec: ; GFX10-DL-LABEL: notudot2_SameVec:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s1, s8 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_and_b32 s0, s0, 0xffff ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s0, s0, v0 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec1, i64 0 %s2.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec2, i64 1 %s1.elt2 = extractelement <2 x i16> %vec2, i64 1
Show All 9 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_v4i16: ; GFX7-LABEL: udot2_v4i16:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
; GFX7-NEXT: s_mov_b64 s[6:7], s[10:11]
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v3, s4, v0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s6, s4, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, s4
; GFX7-NEXT: s_and_b32 s7, s5, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v0
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: s_lshr_b32 s4, s4, 16
; GFX7-NEXT: s_lshr_b32 s5, s5, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_v4i16: ; GFX8-LABEL: udot2_v4i16:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v3, s2, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, v1, v0, s3
; GFX8-NEXT: v_mad_u32_u24 v2, v3, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_v4i16: ; GFX9-NODL-LABEL: udot2_v4i16:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_v4i16: ; GFX9-DL-LABEL: udot2_v4i16:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0 ; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_v4i16: ; GFX10-DL-LABEL: udot2_v4i16:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2, <4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2 %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 0 %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <4 x i16> %vec2, i64 0 %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <4 x i16> %vec1, i64 1 %s1.elt2 = extractelement <4 x i16> %vec1, i64 1
Show All 9 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_v4i16_Hi: ; GFX7-LABEL: udot2_v4i16_Hi:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x1 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 offset:4
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 offset:4
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v3, s4, v0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s6, s4, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, s4
; GFX7-NEXT: s_and_b32 s7, s5, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v0
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: s_lshr_b32 s4, s4, 16
; GFX7-NEXT: s_lshr_b32 s5, s5, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_v4i16_Hi: ; GFX8-LABEL: udot2_v4i16_Hi:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x4 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x4 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s4, v0
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, s6, v0
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v2
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v4
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_and_b32_e32 v1, s2, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v3, s2, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, s3
; GFX8-NEXT: v_mad_u32_u24 v2, v3, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_v4i16_Hi: ; GFX9-NODL-LABEL: udot2_v4i16_Hi:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5] offset:4
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7] offset:4
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x4 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x4
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_v4i16_Hi: ; GFX9-DL-LABEL: udot2_v4i16_Hi:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x4 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5] offset:4
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7] offset:4
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x4 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0 ; GFX9-DL-NEXT: v_dot2_u32_u16 v0, v3, v2, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_v4i16_Hi: ; GFX10-DL-LABEL: udot2_v4i16_Hi:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x4 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5] offset:4
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x4 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7] offset:4
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_dot2_u32_u16 v0, s1, s0, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2, <4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2 %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 2 %s1.elt1 = extractelement <4 x i16> %vec1, i64 2
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <4 x i16> %vec2, i64 2 %s2.elt1 = extractelement <4 x i16> %vec2, i64 2
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <4 x i16> %vec1, i64 3 %s1.elt2 = extractelement <4 x i16> %vec1, i64 3
Show All 9 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1, define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
; GFX7-LABEL: notudot2_v4i16_Even: ; GFX7-LABEL: notudot2_v4i16_Even:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
; GFX7-NEXT: s_mov_b64 s[6:7], s[10:11]
; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s5, s5, s8 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s4
; GFX7-NEXT: s_and_b32 s4, s4, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: s_and_b32 s6, s6, s8
; GFX7-NEXT: s_and_b32 s7, s7, s8
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v0, s5
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: notudot2_v4i16_Even: ; GFX8-LABEL: notudot2_v4i16_Even:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s8, 0xffff ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: s_and_b32 s3, s3, s8 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s2, s2, s8 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: s_and_b32 s5, s5, s8 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s6 ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_and_b32 s4, s4, s8 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v3, s2, v3
; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
; GFX8-NEXT: v_and_b32_e32 v2, s2, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v1, v3, v1, s3
; GFX8-NEXT: v_mad_u32_u24 v2, v2, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: notudot2_v4i16_Even: ; GFX9-NODL-LABEL: notudot2_v4i16_Even:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s10, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
; GFX9-NODL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s11, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NODL-NEXT: s_and_b32 s1, s1, s10 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s0, s0, s10 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: s_and_b32 s5, s9, s10 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, s0, v0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v1, v2 ; GFX9-NODL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-NODL-NEXT: s_and_b32 s4, s8, s10
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: notudot2_v4i16_Even: ; GFX9-DL-LABEL: notudot2_v4i16_Even:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_mov_b32 s10, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
; GFX9-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX9-DL-NEXT: s_load_dword s11, s[2:3], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DL-NEXT: s_and_b32 s1, s1, s10 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s0, s0, s10 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: s_and_b32 s5, s9, s10 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: v_add3_u32 v0, v1, s0, v0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v1, v2 ; GFX9-DL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-DL-NEXT: s_and_b32 s4, s8, s10
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: notudot2_v4i16_Even: ; GFX10-DL-LABEL: notudot2_v4i16_Even:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s10, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
; GFX10-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s10 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_and_b32 s1, s1, s4 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_and_b32 s5, s9, s4 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_and_b32 s0, s0, s4 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s1, v0 ; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
; GFX10-DL-NEXT: s_and_b32 s1, s8, s4 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2, <4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2 %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 0 %s1.elt1 = extractelement <4 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <4 x i16> %vec2, i64 0 %s2.elt1 = extractelement <4 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <4 x i16> %vec1, i64 2 %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
Show All 9 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1, define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
; GFX7-LABEL: notudot2_v4i16_Middle: ; GFX7-LABEL: notudot2_v4i16_Middle:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
; GFX7-NEXT: s_mov_b64 s[6:7], s[10:11]
; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s5, s5, s8 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s4
; GFX7-NEXT: s_and_b32 s7, s7, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: s_lshr_b32 s4, s4, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s5
; GFX7-NEXT: s_lshr_b32 s6, s6, 16
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: notudot2_v4i16_Middle: ; GFX8-LABEL: notudot2_v4i16_Middle:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s8, 0xffff ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_and_b32_e32 v1, s2, v1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v3, s2, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0 ; GFX8-NEXT: v_mad_u32_u24 v1, v3, v1, s3
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0 ; GFX8-NEXT: v_mad_u32_u24 v2, v2, v0, v1
; GFX8-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s3, s3, s8
; GFX8-NEXT: s_lshr_b32 s2, s2, 16
; GFX8-NEXT: s_and_b32 s5, s5, s8
; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0
; GFX8-NEXT: s_lshr_b32 s4, s4, 16
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: notudot2_v4i16_Middle: ; GFX9-NODL-LABEL: notudot2_v4i16_Middle:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s10, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
; GFX9-NODL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s11, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NODL-NEXT: s_and_b32 s1, s1, s10 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_and_b32 s5, s9, s10 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, s0, v0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v1, v2 ; GFX9-NODL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-NODL-NEXT: s_lshr_b32 s4, s8, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: notudot2_v4i16_Middle: ; GFX9-DL-LABEL: notudot2_v4i16_Middle:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_mov_b32 s10, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
; GFX9-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX9-DL-NEXT: s_load_dword s11, s[2:3], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DL-NEXT: s_and_b32 s1, s1, s10 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: s_and_b32 s5, s9, s10 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: v_add3_u32 v0, v1, s0, v0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v1, v2 ; GFX9-DL-NEXT: global_store_dword v4, v0, s[2:3]
; GFX9-DL-NEXT: s_lshr_b32 s4, s8, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: notudot2_v4i16_Middle: ; GFX10-DL-LABEL: notudot2_v4i16_Middle:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s10, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
; GFX10-DL-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s10 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX10-DL-NEXT: s_and_b32 s1, s1, s4 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_and_b32 s4, s9, s4 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 16 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s1, v0 ; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
; GFX10-DL-NEXT: s_lshr_b32 s1, s8, 16 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i16> addrspace(1)* %src2, <4 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %src2 %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i16>, <4 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i16>, <4 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <4 x i16> %vec1, i64 1 %s1.elt1 = extractelement <4 x i16> %vec1, i64 1
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <4 x i16> %vec2, i64 1 %s2.elt1 = extractelement <4 x i16> %vec2, i64 1
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <4 x i16> %vec1, i64 2 %s1.elt2 = extractelement <4 x i16> %vec1, i64 2
Show All 9 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: notudot2_DiffIndex: ; GFX7-LABEL: notudot2_DiffIndex:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v1, s4
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v2, v0
; GFX7-NEXT: s_and_b32 s4, s4, s8
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: notudot2_DiffIndex: ; GFX8-LABEL: notudot2_DiffIndex:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mad_u32_u24 v0, s2, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_lshr_b32 s7, s4, 16 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: notudot2_DiffIndex: ; GFX9-NODL-LABEL: notudot2_DiffIndex:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_lshr_b32 s5, s1, 16
; GFX9-NODL-NEXT: s_and_b32 s1, s1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: notudot2_DiffIndex: ; GFX9-DL-LABEL: notudot2_DiffIndex:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_0
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 16
; GFX9-DL-NEXT: s_and_b32 s1, s1, s8
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: notudot2_DiffIndex: ; GFX10-DL-LABEL: notudot2_DiffIndex:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_0
; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_1
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: v_add3_u32 v0, v1, s2, v0
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
; GFX10-DL-NEXT: s_lshr_b32 s5, s0, 16
; GFX10-DL-NEXT: s_and_b32 s6, s1, s4
; GFX10-DL-NEXT: s_and_b32 s0, s0, s4
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s6, s5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 1 %s2.elt1 = extractelement <2 x i16> %vec2, i64 1
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 9 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_MultipleUses_add1: ; GFX7-LABEL: udot2_MultipleUses_add1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s5
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: s_and_b32 s4, s4, s8
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_u32_u24 v1, s5, v1, v0
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_MultipleUses_add1: ; GFX8-LABEL: udot2_MultipleUses_add1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_u32_u24 v1, s2, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1 ; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s3
; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_MultipleUses_add1: ; GFX9-NODL-LABEL: udot2_MultipleUses_add1:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s5, v2, v1
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_MultipleUses_add1: ; GFX9-DL-LABEL: udot2_MultipleUses_add1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v2, v1
; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_MultipleUses_add1: ; GFX10-DL-LABEL: udot2_MultipleUses_add1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s2
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v0
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_mov_b32 s6, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
; GFX10-DL-NEXT: s_lshr_b32 s4, s0, 16
; GFX10-DL-NEXT: s_lshr_b32 s5, s1, 16
; GFX10-DL-NEXT: s_and_b32 s0, s0, s6
; GFX10-DL-NEXT: s_and_b32 s1, s1, s6
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v0
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v0
; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 12 Lines
} }
define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: idot2_MultipleUses_add1: ; GFX7-LABEL: idot2_MultipleUses_add1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v1, v3, v1, v0
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i16 s7, s4
; GFX7-NEXT: s_ashr_i32 s4, s4, 16
; GFX7-NEXT: s_sext_i32_i16 s8, s5
; GFX7-NEXT: s_ashr_i32 s5, s5, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: v_mad_i32_i24 v1, s8, v1, v0
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot2_MultipleUses_add1: ; GFX8-LABEL: idot2_MultipleUses_add1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_mad_i32_i24 v1, v2, v1, v0
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i16 s5, s2
; GFX8-NEXT: s_ashr_i32 s2, s2, 16
; GFX8-NEXT: s_sext_i32_i16 s6, s3
; GFX8-NEXT: s_ashr_i32 s3, s3, 16
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mov_b32_e32 v2, s5
; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
; GFX8-NEXT: v_mad_i32_i24 v1, s6, v2, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot2_MultipleUses_add1: ; GFX9-NODL-LABEL: idot2_MultipleUses_add1:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s5, v2, v1
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot2_MultipleUses_add1: ; GFX9-DL-LABEL: idot2_MultipleUses_add1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v1
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s5, v2, v1
; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot2_MultipleUses_add1: ; GFX10-DL-LABEL: idot2_MultipleUses_add1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_ashrrev_i32_e32 v0, 16, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_ashrrev_i32_e32 v3, 16, v2
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s2
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v0
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8
; GFX10-DL-NEXT: s_ashr_i32 s4, s0, 16
; GFX10-DL-NEXT: s_ashr_i32 s5, s1, 16
; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0
; GFX10-DL-NEXT: s_sext_i32_i16 s1, s1
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s1, s0, v0
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v0
; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32 %conv = sext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = sext i16 %s2.elt1 to i32 %conv2 = sext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 11 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_MultipleUses_mul1: ; GFX7-LABEL: udot2_MultipleUses_mul1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: v_mad_u32_u24 v4, v0, v2, s4
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, v4
; GFX7-NEXT: s_and_b32 s4, s4, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v1, s5, v0, v1
; GFX7-NEXT: v_mad_u32_u24 v1, s7, v2, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_MultipleUses_mul1: ; GFX8-LABEL: udot2_MultipleUses_mul1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mad_u32_u24 v0, s2, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_mov_b32_e32 v2, s3 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v2, v0 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v4, v2, v1, s3
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v4
; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_MultipleUses_mul1: ; GFX9-NODL-LABEL: udot2_MultipleUses_mul1:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_mov_b32 s0, 0xffff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_and_b32_e32 v3, s0, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v2, v4, v3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v4, v3, s1
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s5, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s1, v3, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_MultipleUses_mul1: ; GFX9-DL-LABEL: udot2_MultipleUses_mul1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_mov_b32 s0, 0xffff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v2
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, v4, v3
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, v4, v3, s1
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s0
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v3, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_MultipleUses_mul1: ; GFX10-DL-LABEL: udot2_MultipleUses_mul1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_mov_b32 s3, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: v_and_b32_e32 v0, s3, v1
; GFX10-DL-NEXT: s_and_b32 s5, s0, s4 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_and_b32 s4, s1, s4 ; GFX10-DL-NEXT: v_and_b32_e32 v3, s3, v2
; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 16 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 16 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s2
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0 ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 13 Lines
} }
define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: idot2_MultipleUses_mul1: ; GFX7-LABEL: idot2_MultipleUses_mul1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v4, v3, v1, s4
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v4
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i16 s7, s4
; GFX7-NEXT: s_sext_i32_i16 s8, s5
; GFX7-NEXT: s_ashr_i32 s4, s4, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s7
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: s_ashr_i32 s5, s5, 16
; GFX7-NEXT: v_mad_i32_i24 v1, s8, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s4
; GFX7-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot2_MultipleUses_mul1: ; GFX8-LABEL: idot2_MultipleUses_mul1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i16 s5, s2
; GFX8-NEXT: s_sext_i32_i16 s6, s3
; GFX8-NEXT: s_ashr_i32 s2, s2, 16
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_ashr_i32 s3, s3, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v2, s2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mad_i32_i24 v0, s3, v2, v0 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v1, v0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v4, v2, v1, s2
; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, v4
; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot2_MultipleUses_mul1: ; GFX9-NODL-LABEL: idot2_MultipleUses_mul1:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 16
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 16
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v2, v4, v3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v4, v3, s0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v3, v1
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot2_MultipleUses_mul1: ; GFX9-DL-LABEL: idot2_MultipleUses_mul1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 16
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 16
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, v4, v3
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_mad_i32_i24 v3, v4, v3, s0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s0
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v3, v1
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot2_MultipleUses_mul1: ; GFX10-DL-LABEL: idot2_MultipleUses_mul1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: s_sext_i32_i16 s4, s0 ; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 16
; GFX10-DL-NEXT: s_sext_i32_i16 s5, s1 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_ashr_i32 s0, s0, 16 ; GFX10-DL-NEXT: v_bfe_i32 v3, v2, 0, 16
; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 16 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s2
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32 %conv = sext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = sext i16 %s2.elt1 to i32 %conv2 = sext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 12 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_MultipleUses_mul2: ; GFX7-LABEL: udot2_MultipleUses_mul2:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s8, 0xffff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: v_mad_u32_u24 v4, v3, v1, s4
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, v4
; GFX7-NEXT: s_and_b32 s4, s4, s8 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_mad_u32_u24 v1, s7, v0, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s4
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_MultipleUses_mul2: ; GFX8-LABEL: udot2_MultipleUses_mul2:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s2, 0xffff ; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s6, s3, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_lshr_b32 s3, s3, 16 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_lshr_b32 s4, s4, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v4, v0, v3, s3
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v4
; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_MultipleUses_mul2: ; GFX9-NODL-LABEL: udot2_MultipleUses_mul2:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v4, v2, v1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-NODL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s1, v1, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_MultipleUses_mul2: ; GFX9-DL-LABEL: udot2_MultipleUses_mul2:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_mov_b32 s8, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v2, v1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_mad_u32_u24 v1, v2, v1, s0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-DL-NEXT: s_load_dword s9, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s4, s0, s8
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-DL-NEXT: s_and_b32 s5, s1, s8
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v1, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_MultipleUses_mul2: ; GFX10-DL-LABEL: udot2_MultipleUses_mul2:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_mov_b32 s6, 0xffff ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v1
; GFX10-DL-NEXT: s_lshr_b32 s4, s0, 16 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_lshr_b32 s5, s1, 16 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX10-DL-NEXT: s_and_b32 s0, s0, s6 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: s_and_b32 s1, s1, s6 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s2
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s1, s0, v0 ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_add3_u32 v0, v2, v0, v1
; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = zext i16 %s1.elt1 to i32 %conv = zext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = zext i16 %s2.elt1 to i32 %conv2 = zext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 13 Lines
} }
define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: idot2_MultipleUses_mul2: ; GFX7-LABEL: idot2_MultipleUses_mul2:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v4, v0, v2, s4
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v4
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i16 s7, s4
; GFX7-NEXT: s_ashr_i32 s4, s4, 16
; GFX7-NEXT: s_sext_i32_i16 s8, s5
; GFX7-NEXT: s_ashr_i32 s5, s5, 16
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_i32_i24 v1, s5, v0, v1
; GFX7-NEXT: v_mad_i32_i24 v0, s5, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot2_MultipleUses_mul2: ; GFX8-LABEL: idot2_MultipleUses_mul2:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16
; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mad_i32_i24 v4, v0, v3, s2
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, v4
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i16 s5, s2
; GFX8-NEXT: s_ashr_i32 s2, s2, 16
; GFX8-NEXT: s_sext_i32_i16 s6, s3
; GFX8-NEXT: s_ashr_i32 s3, s3, 16
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v2, s5
; GFX8-NEXT: v_mad_i32_i24 v0, s3, v1, v0
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot2_MultipleUses_mul2: ; GFX9-NODL-LABEL: idot2_MultipleUses_mul2:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v4, v2, v1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-NODL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot2_MultipleUses_mul2: ; GFX9-DL-LABEL: idot2_MultipleUses_mul2:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 16, v1
; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 16, v2
; GFX9-DL-NEXT: v_mul_i32_i24_e32 v4, v2, v1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: v_add3_u32 v1, v4, v1, v3
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i16 s4, s0
; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-DL-NEXT: s_sext_i32_i16 s5, s1
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 16
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot2_MultipleUses_mul2: ; GFX10-DL-LABEL: idot2_MultipleUses_mul2:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: s_ashr_i32 s4, s0, 16 ; GFX10-DL-NEXT: v_ashrrev_i32_e32 v0, 16, v1
; GFX10-DL-NEXT: s_ashr_i32 s5, s1, 16 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_sext_i32_i16 s0, s0 ; GFX10-DL-NEXT: v_ashrrev_i32_e32 v3, 16, v2
; GFX10-DL-NEXT: s_sext_i32_i16 s1, s1 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s4, v0 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s1, s0, v0 ; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s2
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX10-DL-NEXT: v_add3_u32 v0, v2, v0, v1
; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i16> %vec1, i64 0 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0
%conv = sext i16 %s1.elt1 to i32 %conv = sext i16 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i16> %vec2, i64 0 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0
%conv2 = sext i16 %s2.elt1 to i32 %conv2 = sext i16 %s2.elt1 to i32
%mul1 = mul i32 %conv2, %conv %mul1 = mul i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i16> %vec1, i64 1 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1
Show All 13 Lines
} }
define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1, define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
; GFX7-LABEL: udot2_acc16: ; GFX7-LABEL: udot2_acc16:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s8, 0xffff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_lshr_b32 s6, s4, 16 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_lshr_b32 s7, s5, 16 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v1, s7 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_and_b32 s5, s5, s8 ; GFX7-NEXT: buffer_load_ushort v4, off, s[0:3], 0
; GFX7-NEXT: s_and_b32 s4, s4, s8 ; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_acc16: ; GFX8-LABEL: udot2_acc16:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_mov_b32 s0, 0xffff ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_and_b32 s3, s2, s0 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_lshr_b32 s2, s2, 16 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_and_b32 s0, s1, s0 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_lshr_b32 s1, s1, 16 ; GFX8-NEXT: flat_load_ushort v6, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2 ; GFX8-NEXT: v_mad_u16 v1, v1, v5, v6
; GFX8-NEXT: v_mov_b32_e32 v3, s3 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_acc16: ; GFX9-NODL-LABEL: udot2_acc16:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_mov_b32 s0, 0xffff ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3] ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_ushort v5, v1, s[2:3]
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_lshr_b32 s5, s8, 16 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_acc16: ; GFX9-DL-LABEL: udot2_acc16:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: global_load_ushort v5, v1, s[2:3]
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot2_u32_u16 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_acc16: ; GFX10-DL-LABEL: udot2_acc16:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX10-DL-NEXT: v_dot2_u32_u16 v1, s6, s7, v1 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v0, v7, v5, v4
; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
; GFX10-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i16> addrspace(1)* %src2, <2 x i16> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%v1 = load <2 x i16>, <2 x i16> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%v2 = load <2 x i16>, <2 x i16> addrspace(1)* %src2 %gep1 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src1, i32 %idx
%v1 = load <2 x i16>, <2 x i16> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %src2, i32 %idx
%v2 = load <2 x i16>, <2 x i16> addrspace(1)* %gep2
%v1e1 = extractelement <2 x i16> %v1, i64 0 %v1e1 = extractelement <2 x i16> %v1, i64 0
%v2e1 = extractelement <2 x i16> %v2, i64 0 %v2e1 = extractelement <2 x i16> %v2, i64 0
%mul1 = mul i16 %v1e1, %v2e1 %mul1 = mul i16 %v1e1, %v2e1
%v1e2 = extractelement <2 x i16> %v1, i64 1 %v1e2 = extractelement <2 x i16> %v1, i64 1
%v2e2 = extractelement <2 x i16> %v2, i64 1 %v2e2 = extractelement <2 x i16> %v2, i64 1
%mul2 = mul i16 %v1e2, %v2e2 %mul2 = mul i16 %v1e2, %v2e2
%s2 = load i16, i16 addrspace(1)* %dst, align 2 %s2 = load i16, i16 addrspace(1)* %dst, align 2
%add1 = add i16 %mul2, %s2 %add1 = add i16 %mul2, %s2
%add2 = add i16 %add1, %mul1 %add2 = add i16 %add1, %mul1
store i16 %add2, i16 addrspace(1)* %dst, align 2 store i16 %add2, i16 addrspace(1)* %dst, align 2
ret void ret void
} }
define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1, define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
; GFX7-LABEL: notsdot2_sext8: ; GFX7-LABEL: notsdot2_sext8:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s10, s2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s0, s4
; GFX7-NEXT: s_mov_b32 s1, s5
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s7
; GFX7-NEXT: s_mov_b32 s6, s2
; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
; GFX7-NEXT: buffer_load_ushort v1, off, s[4:7], 0
; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
; GFX7-NEXT: s_mov_b32 s11, s3 ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_ushort v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_ushort v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1) ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v2, v0, 0, 8 ; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
; GFX7-NEXT: v_bfe_i32 v2, v2, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v3, v1, 0, 8 ; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 8
; GFX7-NEXT: v_bfe_i32 v0, v0, 8, 8 ; GFX7-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX7-NEXT: v_bfe_i32 v1, v1, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_i32_i24 v0, v1, v0, s0 ; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s4
; GFX7-NEXT: v_mad_i32_i24 v0, v3, v2, v0 ; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: notsdot2_sext8: ; GFX8-LABEL: notsdot2_sext8:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v2, s6 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s7 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ushort v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ushort v0, v[0:1] ; GFX8-NEXT: flat_load_ushort v0, v[0:1]
; GFX8-NEXT: flat_load_ushort v1, v[2:3]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1) ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 8
; GFX8-NEXT: v_lshrrev_b16_e32 v3, 8, v3
; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8 ; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8
; GFX8-NEXT: v_lshrrev_b16_e32 v0, 8, v0 ; GFX8-NEXT: v_lshrrev_b16_e32 v0, 8, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v0, v1, v0, s2 ; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s2
; GFX8-NEXT: v_mad_i32_i24 v2, v3, v2, v0 ; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: notsdot2_sext8: ; GFX9-NODL-LABEL: notsdot2_sext8:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[6:7] ; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1) ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 8 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v2, 8, v2 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v2, 8, v2
; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s0 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v4, v3, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: notsdot2_sext8: ; GFX9-DL-LABEL: notsdot2_sext8:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX9-DL-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX9-DL-NEXT: global_load_ushort v2, v0, s[6:7] ; GFX9-DL-NEXT: global_load_ushort v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1) ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX9-DL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 8 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: v_lshrrev_b16_e32 v1, 8, v1
; GFX9-DL-NEXT: v_lshrrev_b16_e32 v2, 8, v2 ; GFX9-DL-NEXT: v_lshrrev_b16_e32 v2, 8, v2
; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s0 ; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3
; GFX9-DL-NEXT: v_mad_i32_i24 v1, v4, v3, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: notsdot2_sext8: ; GFX10-DL-LABEL: notsdot2_sext8:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX10-DL-NEXT: global_load_ushort v2, v0, s[6:7] ; GFX10-DL-NEXT: global_load_ushort v2, v0, s[6:7]
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1) ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, v1 ; GFX10-DL-NEXT: v_lshrrev_b16_e64 v0, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, v2 ; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, v2
; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 8 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: v_bfe_i32 v3, v3, 0, 8 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: v_bfe_i32 v4, v4, 0, 8
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_i32_i24 v3, v4, v3, s2 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, s2, v1
; GFX10-DL-NEXT: v_mad_i32_i24 v1, v2, v1, v3 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<2 x i8> addrspace(1)* %src2, <2 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <2 x i8>, <2 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <2 x i8>, <2 x i8> addrspace(1)* %src2 %gep1 = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <2 x i8>, <2 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <2 x i8>, <2 x i8> addrspace(1)* %gep2
%s1.elt1 = extractelement <2 x i8> %vec1, i64 0 %s1.elt1 = extractelement <2 x i8> %vec1, i64 0
%conv = sext i8 %s1.elt1 to i32 %conv = sext i8 %s1.elt1 to i32
%s2.elt1 = extractelement <2 x i8> %vec2, i64 0 %s2.elt1 = extractelement <2 x i8> %vec2, i64 0
%conv2 = sext i8 %s2.elt1 to i32 %conv2 = sext i8 %s2.elt1 to i32
%mul1 = mul nuw i32 %conv2, %conv %mul1 = mul nuw i32 %conv2, %conv
%s1.elt2 = extractelement <2 x i8> %vec1, i64 1 %s1.elt2 = extractelement <2 x i8> %vec1, i64 1
%conv3 = sext i8 %s1.elt2 to i32 %conv3 = sext i8 %s1.elt2 to i32
%s2.elt2 = extractelement <2 x i8> %vec2, i64 1 %s2.elt2 = extractelement <2 x i8> %vec2, i64 1
%conv4 = sext i8 %s2.elt2 to i32 %conv4 = sext i8 %s2.elt2 to i32
%mul2 = mul nuw i32 %conv4, %conv3 %mul2 = mul nuw i32 %conv4, %conv3
%s3 = load i32, i32 addrspace(1)* %dst, align 4 %s3 = load i32, i32 addrspace(1)* %dst, align 4
%add = add i32 %mul2, %s3 %add = add i32 %mul2, %s3
%add6 = add i32 %add, %mul1 %add6 = add i32 %add, %mul1
store i32 %add6, i32 addrspace(1)* %dst, align 4 store i32 %add6, i32 addrspace(1)* %dst, align 4
ret void ret void
} }
declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/idot4s.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: idot4_acc32: ; GFX7-LABEL: idot4_acc32:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8
; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i8 s6, s4 ; GFX7-NEXT: v_mad_i32_i24 v1, v1, v5, s4
; GFX7-NEXT: s_sext_i32_i8 s7, s5 ; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
; GFX7-NEXT: s_bfe_i32 s9, s5, 0x80008 ; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v0, s7 ; GFX7-NEXT: v_mad_i32_i24 v1, v3, v6, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80010 ; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1 ; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v1
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008 ; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s9
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s11
; GFX7-NEXT: s_ashr_i32 s5, s5, 24
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
; GFX7-NEXT: s_ashr_i32 s4, s4, 24
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot4_acc32: ; GFX8-LABEL: idot4_acc32:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_sext_i32_i8 s4, s2
; GFX8-NEXT: s_sext_i32_i8 s5, s3
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s10
; GFX8-NEXT: s_bfe_i32 s9, s3, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x80010 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v1, s9 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_ashr_i32 s3, s3, 24 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_ashr_i32 s2, s2, 24 ; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 8
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 8
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0 ; GFX8-NEXT: v_bfe_i32 v6, v3, 16, 8
; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8
; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s2
; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v1
; GFX8-NEXT: v_ashrrev_i32_e32 v0, 24, v0
; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot4_acc32: ; GFX9-NODL-LABEL: idot4_acc32:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0 ; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
; GFX9-NODL-NEXT: s_bfe_i32 s9, s1, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v1, v2
; GFX9-NODL-NEXT: s_bfe_i32 s6, s0, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NODL-NEXT: s_bfe_i32 s8, s0, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s6, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 24
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s8, v2, v1
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot4_acc32: ; GFX9-DL-LABEL: idot4_acc32:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v2, v3, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot4_i32_i8 v1, s0, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot4_acc32: ; GFX10-DL-LABEL: idot4_acc32:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_dot4_i32_i8 v0, s0, s1, v0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: v_dot4_i32_i8 v1, v1, v2, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i32 %cv1e0 = sext i8 %v1e0 to i32
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = sext i8 %v2e0 to i32 %cv2e0 = sext i8 %v2e0 to i32
%mul1 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 26 Lines
; TODO: Currently, vector elements{0 and 3} get zero_extended from i16 to i32 which should ; TODO: Currently, vector elements{0 and 3} get zero_extended from i16 to i32 which should
; be sign_extended directly to i32; prevents the pattern recognizer to recognize this pattern. ; be sign_extended directly to i32; prevents the pattern recognizer to recognize this pattern.
define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: idot4_acc16: ; GFX7-LABEL: idot4_acc16:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s8, 0xffff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_sext_i32_i8 s6, s4 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_sext_i32_i8 s7, s5 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80008 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_and_b32 s7, s7, s8 ; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
; GFX7-NEXT: s_bfe_i32 s12, s5, 0x80010 ; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x80008 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_and_b32 s10, s10, s8 ; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
; GFX7-NEXT: s_and_b32 s6, s6, s8 ; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s7 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x80010 ; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8
; GFX7-NEXT: s_ashr_i32 s5, s5, 24 ; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 8
; GFX7-NEXT: s_and_b32 s12, s12, s8 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
; GFX7-NEXT: s_and_b32 s9, s9, s8 ; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
; GFX7-NEXT: v_mov_b32_e32 v2, s10 ; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
; GFX7-NEXT: s_ashr_i32 s4, s4, 24 ; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX7-NEXT: s_and_b32 s11, s11, s8 ; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
; GFX7-NEXT: s_and_b32 s5, s5, s8 ; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: v_mov_b32_e32 v3, s12
; GFX7-NEXT: s_and_b32 s4, s4, s8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot4_acc16: ; GFX8-LABEL: idot4_acc16:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_sext_i32_i8 s2, s0
; GFX8-NEXT: s_sext_i32_i8 s3, s1
; GFX8-NEXT: s_bfe_i32 s5, s1, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: s_bfe_i32 s7, s1, 0x80010
; GFX8-NEXT: s_bfe_i32 s4, s0, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v4, s5
; GFX8-NEXT: s_bfe_i32 s6, s0, 0x80010
; GFX8-NEXT: s_ashr_i32 s1, s1, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s7
; GFX8-NEXT: s_ashr_i32 s0, s0, 24
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v4, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v5, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_ushort v1, v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v4
; GFX8-NEXT: v_bfe_i32 v7, v4, 0, 8
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX8-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v10, 8, v0
; GFX8-NEXT: v_bfe_i32 v8, v0, 0, 8
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX8-NEXT: v_bfe_i32 v10, v10, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u16 v1, v7, v8, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 8
; GFX8-NEXT: v_mad_u16 v1, v9, v10, v1
; GFX8-NEXT: v_bfe_i32 v4, v4, 0, 8
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX8-NEXT: v_mad_u16 v1, v5, v6, v1
; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot4_acc16: ; GFX9-NODL-LABEL: idot4_acc16:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80008 ; GFX9-NODL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_bfe_i32 s9, s1, 0x80010 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
; GFX9-NODL-NEXT: s_bfe_i32 s6, s0, 0x80008 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s7 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 8, v3
; GFX9-NODL-NEXT: s_bfe_i32 s8, s0, 0x80010 ; GFX9-NODL-NEXT: v_bfe_i32 v6, v2, 0, 8
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 24 ; GFX9-NODL-NEXT: v_bfe_i32 v7, v3, 0, 8
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s9 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 24 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-NODL-NEXT: v_bfe_i32 v8, v8, 0, 8
; GFX9-NODL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v4, v6, v7, v4
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s6, v3, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s8, v4, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NODL-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s0, v2, v1 ; GFX9-NODL-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NODL-NEXT: v_mad_legacy_u16 v4, v8, v9, v4
; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-NODL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot4_acc16: ; GFX9-DL-LABEL: idot4_acc16:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 8, v3
; GFX9-DL-NEXT: v_bfe_i32 v6, v2, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v7, v3, 0, 8
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-DL-NEXT: v_bfe_i32 v8, v8, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_i32_i8 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v4, v6, v7, v4
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX9-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX9-DL-NEXT: v_mad_legacy_u16 v4, v8, v9, v4
; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot4_acc16: ; GFX10-DL-LABEL: idot4_acc16:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_dot4_i32_i8 v1, s6, s7, v1 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX10-DL-NEXT: v_bfe_i32 v4, v1, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v10, v2, 0, 8
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; GFX10-DL-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v6, v6, 0, 8
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: v_bfe_i32 v4, v8, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v10, v9, 0, 8
; GFX10-DL-NEXT: v_mad_u16 v3, v5, v6, v3
; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
; GFX10-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i16 %cv1e0 = sext i8 %v1e0 to i16
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = sext i8 %v2e0 to i16 %cv2e0 = sext i8 %v2e0 to i16
%mul1 = mul nsw i16 %cv1e0, %cv2e0 %mul1 = mul nsw i16 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 24 Lines
} }
define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: idot4_acc8: ; GFX7-LABEL: idot4_acc8:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_and_b32 s7, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_and_b32 s6, s5, s8 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_mov_b32_e32 v2, s8 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot4_acc8: ; GFX8-LABEL: idot4_acc8:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_and_b32 s3, s2, s0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_u32 s4, s2, 0x80008 ; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
; GFX8-NEXT: s_and_b32 s0, s1, s0 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v3, s3 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_bfe_u32 s6, s2, 0x80010 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
; GFX8-NEXT: v_mov_b32_e32 v4, s4 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX8-NEXT: s_bfe_u32 s7, s1, 0x80010 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v4
; GFX8-NEXT: s_lshr_b32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: s_lshr_b32 s1, s1, 24
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v5
; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v7, v8, v0
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: v_mad_u16 v0, v1, v6, v0
; GFX8-NEXT: v_mad_u16 v0, v9, v10, v0
; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot4_acc8: ; GFX9-NODL-LABEL: idot4_acc8:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_bfe_u32 s6, s1, 0x80008 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-NODL-NEXT: s_bfe_u32 s5, s8, 0x80008 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-NODL-NEXT: s_bfe_u32 s7, s8, 0x80010
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v3, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot4_acc8: ; GFX9-DL-LABEL: idot4_acc8:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot4_acc8: ; GFX10-DL-LABEL: idot4_acc8:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s6, s7, v1 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 8, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v4, v2, v3, v4
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%mul1 = mul i8 %v1e0, %v2e0 %mul1 = mul i8 %v1e0, %v2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
%v2e1 = extractelement <4 x i8> %vec2, i64 1 %v2e1 = extractelement <4 x i8> %vec2, i64 1
%mul2 = mul i8 %v1e1, %v2e1 %mul2 = mul i8 %v1e1, %v2e1
Show All 16 Lines
} }
define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: idot4_multiuse_mul1: ; GFX7-LABEL: idot4_multiuse_mul1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8
; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i8 s6, s4 ; GFX7-NEXT: v_mad_i32_i24 v8, v1, v5, s4
; GFX7-NEXT: s_sext_i32_i8 s7, s5 ; GFX7-NEXT: v_mad_i32_i24 v3, v3, v6, v8
; GFX7-NEXT: s_bfe_i32 s9, s5, 0x80008 ; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v0, s7 ; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: v_mad_i32_i24 v1, v1, v5, v3
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008 ; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
; GFX7-NEXT: v_mad_i32_i24 v1, s6, v0, v1 ; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
; GFX7-NEXT: v_mov_b32_e32 v2, s9 ; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v1
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80010 ; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
; GFX7-NEXT: v_mad_i32_i24 v1, s8, v2, v1
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s11
; GFX7-NEXT: s_ashr_i32 s5, s5, 24
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
; GFX7-NEXT: s_ashr_i32 s4, s4, 24
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot4_multiuse_mul1: ; GFX8-LABEL: idot4_multiuse_mul1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 8
; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 8
; GFX8-NEXT: v_bfe_i32 v6, v3, 16, 8
; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8
; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i8 s4, s2 ; GFX8-NEXT: v_mad_i32_i24 v8, v1, v2, s2
; GFX8-NEXT: s_sext_i32_i8 s5, s3 ; GFX8-NEXT: v_mad_i32_i24 v4, v4, v5, v8
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80008 ; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, v4
; GFX8-NEXT: v_mov_b32_e32 v1, s10 ; GFX8-NEXT: v_ashrrev_i32_e32 v0, 24, v0
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x80008 ; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
; GFX8-NEXT: v_mad_i32_i24 v1, s4, v0, v1 ; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s7
; GFX8-NEXT: s_bfe_i32 s9, s3, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v1, s6, v2, v1
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_ashr_i32 s3, s3, 24
; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX8-NEXT: s_ashr_i32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot4_multiuse_mul1: ; GFX9-NODL-LABEL: idot4_multiuse_mul1:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 8
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v2, v3, v4
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0 ; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v3, v4, s0
; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1 ; GFX9-NODL-NEXT: v_add3_u32 v2, v5, v3, v2
; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80008 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
; GFX9-NODL-NEXT: s_bfe_i32 s6, s0, 0x80008
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s4, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NODL-NEXT: s_bfe_i32 s9, s1, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s6, v3, v2
; GFX9-NODL-NEXT: s_bfe_i32 s8, s0, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: s_ashr_i32 s1, s1, 24
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s8, v2, v1
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot4_multiuse_mul1: ; GFX9-DL-LABEL: idot4_multiuse_mul1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s10, s[2:3], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 8
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v4
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i8 s4, s0 ; GFX9-DL-NEXT: v_mad_i32_i24 v3, v3, v4, s0
; GFX9-DL-NEXT: s_sext_i32_i8 s5, s1 ; GFX9-DL-NEXT: v_add3_u32 v2, v5, v3, v2
; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x80008 ; GFX9-DL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s10
; GFX9-DL-NEXT: s_bfe_i32 s6, s0, 0x80008
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s4, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x80010
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s6, v3, v2
; GFX9-DL-NEXT: s_bfe_i32 s8, s0, 0x80010
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 24
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s8, v2, v1
; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 24
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s0, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot4_multiuse_mul1: ; GFX10-DL-LABEL: idot4_multiuse_mul1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 8
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_bfe_i32 v3, v2, 0, 8
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v7, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX10-DL-NEXT: v_mul_i32_i24_e32 v5, v0, v3
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: v_mad_i32_i24 v0, v0, v3, s2
; GFX10-DL-NEXT: s_sext_i32_i8 s4, s0 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX10-DL-NEXT: s_sext_i32_i8 s5, s1 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT: s_bfe_i32 s6, s0, 0x80008 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_bfe_i32 s7, s1, 0x80008 ; GFX10-DL-NEXT: v_add3_u32 v0, v7, v0, v5
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s6, s7, v0 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
; GFX10-DL-NEXT: s_bfe_i32 s4, s0, 0x80010
; GFX10-DL-NEXT: s_bfe_i32 s5, s1, 0x80010
; GFX10-DL-NEXT: s_ashr_i32 s0, s0, 24
; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 24
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s0, s1, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i32 %cv1e0 = sext i8 %v1e0 to i32
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = sext i8 %v2e0 to i32 %cv2e0 = sext i8 %v2e0 to i32
%mul1 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 27 Lines
; TODO: Support this pattern. ; TODO: Support this pattern.
define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: idot4_acc32_vecMul: ; GFX7-LABEL: idot4_acc32_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 24, v2
; GFX7-NEXT: v_bfe_i32 v3, v2, 16, 8
; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 8
; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_ashrrev_i32_e32 v5, 24, v0
; GFX7-NEXT: v_bfe_i32 v6, v0, 16, 8
; GFX7-NEXT: v_bfe_i32 v7, v0, 8, 8
; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_ashr_i32 s6, s4, 24 ; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, s4
; GFX7-NEXT: s_ashr_i32 s9, s5, 24 ; GFX7-NEXT: v_mad_i32_i24 v0, v4, v7, v0
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80010 ; GFX7-NEXT: v_mad_i32_i24 v0, v3, v6, v0
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80008 ; GFX7-NEXT: v_mad_i32_i24 v0, v1, v5, v0
; GFX7-NEXT: s_sext_i32_i8 s5, s5
; GFX7-NEXT: s_bfe_i32 s7, s4, 0x80010
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
; GFX7-NEXT: s_sext_i32_i8 s4, s4
; GFX7-NEXT: v_mov_b32_e32 v0, s5
; GFX7-NEXT: v_mov_b32_e32 v1, s12
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s11
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s10
; GFX7-NEXT: v_mad_i32_i24 v0, s7, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s9
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot4_acc32_vecMul: ; GFX8-LABEL: idot4_acc32_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s8, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_lshrrev_b16_e64 v0, 8, s2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_lshrrev_b16_e64 v1, 8, s3 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_ashr_i32 s6, s3, 24 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80010 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_sext_i32_i8 s3, s3 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_ashr_i32 s4, s2, 24 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_bfe_i32 s5, s2, 0x80010 ; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3
; GFX8-NEXT: s_sext_i32_i8 s2, s2 ; GFX8-NEXT: v_ashrrev_i32_e32 v4, 24, v3
; GFX8-NEXT: v_mov_b32_e32 v2, s3 ; GFX8-NEXT: v_bfe_i32 v5, v3, 16, 8
; GFX8-NEXT: v_mov_b32_e32 v3, s8 ; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0
; GFX8-NEXT: v_ashrrev_i32_e32 v6, 24, v0
; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v2, v3 ; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX8-NEXT: v_mad_i32_i24 v0, v0, v1, v2 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: v_mad_i32_i24 v0, v3, v0, s2
; GFX8-NEXT: v_mad_i32_i24 v0, s5, v1, v0 ; GFX8-NEXT: v_mad_i32_i24 v0, v1, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: v_mad_i32_i24 v0, v5, v7, v0
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v1, v0 ; GFX8-NEXT: v_mad_i32_i24 v2, v4, v6, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot4_acc32_vecMul: ; GFX9-NODL-LABEL: idot4_acc32_vecMul:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s0 ; GFX9-NODL-NEXT: v_add3_u32 v2, v5, s0, v2
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v2, 8, s1 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-NODL-NEXT: s_ashr_i32 s6, s1, 24
; GFX9-NODL-NEXT: s_bfe_i32 s7, s1, 0x80010
; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s1
; GFX9-NODL-NEXT: s_ashr_i32 s4, s0, 24
; GFX9-NODL-NEXT: s_bfe_i32 s5, s0, 0x80010
; GFX9-NODL-NEXT: s_sext_i32_i8 s0, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s8
; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-NODL-NEXT: v_mad_i32_i24 v3, s0, v3, v4
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v1, v2, v3
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot4_acc32_vecMul: ; GFX9-DL-LABEL: idot4_acc32_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s0 ; GFX9-DL-NEXT: v_add3_u32 v2, v5, s0, v2
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s1 ; GFX9-DL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-DL-NEXT: s_ashr_i32 s6, s1, 24
; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x80010
; GFX9-DL-NEXT: s_sext_i32_i8 s1, s1
; GFX9-DL-NEXT: s_ashr_i32 s4, s0, 24
; GFX9-DL-NEXT: s_bfe_i32 s5, s0, 0x80010
; GFX9-DL-NEXT: s_sext_i32_i8 s0, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s8
; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-DL-NEXT: v_mad_i32_i24 v3, s0, v3, v4
; GFX9-DL-NEXT: v_mad_i32_i24 v1, v1, v2, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s6
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot4_acc32_vecMul: ; GFX10-DL-LABEL: idot4_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v0, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, v2
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v7, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v0, 8, s0 ; GFX10-DL-NEXT: v_add3_u32 v0, v7, s2, v0
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s1 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
; GFX10-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_sext_i32_i8 s4, s0
; GFX10-DL-NEXT: s_sext_i32_i8 s5, s1
; GFX10-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s4, s5, v2
; GFX10-DL-NEXT: s_bfe_i32 s4, s0, 0x80010
; GFX10-DL-NEXT: s_bfe_i32 s5, s1, 0x80010
; GFX10-DL-NEXT: s_ashr_i32 s0, s0, 24
; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 24
; GFX10-DL-NEXT: v_mad_i32_i24 v0, v0, v1, v2
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s0, s1, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = sext <4 x i8> %vec1 to <4 x i32> %cvec1 = sext <4 x i8> %vec1 to <4 x i32>
%cvec2 = sext <4 x i8> %vec2 to <4 x i32> %cvec2 = sext <4 x i8> %vec2 to <4 x i32>
%mul = mul <4 x i32> %cvec1, %cvec2 %mul = mul <4 x i32> %cvec1, %cvec2
%mul0 = extractelement <4 x i32> %mul, i64 0 %mul0 = extractelement <4 x i32> %mul, i64 0
%mul1 = extractelement <4 x i32> %mul, i64 1 %mul1 = extractelement <4 x i32> %mul, i64 1
%mul2 = extractelement <4 x i32> %mul, i64 2 %mul2 = extractelement <4 x i32> %mul, i64 2
Show All 10 Lines
} }
define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: idot4_acc16_vecMul: ; GFX7-LABEL: idot4_acc16_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_ashr_i32 s6, s4, 24 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80010 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80008 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_ashr_i32 s9, s5, 24 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_sext_i32_i8 s5, s5 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_i32 s7, s4, 0x80010 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_sext_i32_i8 s4, s4 ; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: v_mov_b32_e32 v2, s11 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: v_bfe_i32 v1, v2, 8, 8
; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 8
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX7-NEXT: v_bfe_i32 v6, v0, 0, 8
; GFX7-NEXT: v_or_b32_e32 v1, v3, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: v_and_b32_e32 v5, s4, v6
; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8
; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
; GFX7-NEXT: v_and_b32_e32 v6, s4, v7
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v8
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v2, v0 ; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2
; GFX7-NEXT: v_mad_i32_i24 v0, s7, v3, v0 ; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s9 ; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v7, v1
; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v6, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot4_acc16_vecMul: ; GFX8-LABEL: idot4_acc16_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_lshrrev_b16_e64 v3, 8, s0 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_lshrrev_b16_e64 v4, 8, s1 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_i32 s5, s1, 0x80010 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_ashr_i32 s4, s1, 24 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_sext_i32_i8 s1, s1 ; GFX8-NEXT: flat_load_ushort v1, v[2:3]
; GFX8-NEXT: s_ashr_i32 s2, s0, 24 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_bfe_i32 s3, s0, 0x80010 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX8-NEXT: s_sext_i32_i8 s0, s0 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 8, v4
; GFX8-NEXT: v_mov_b32_e32 v5, s1
; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX8-NEXT: v_bfe_i32 v4, v4, 0, 8 ; GFX8-NEXT: v_bfe_i32 v4, v4, 0, 8
; GFX8-NEXT: v_mov_b32_e32 v6, s5 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 8, v5
; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX8-NEXT: v_ashrrev_i16_e32 v8, 8, v0
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v2, s0, v5, v2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
; GFX8-NEXT: v_mad_i32_i24 v2, v3, v4, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 8, v6
; GFX8-NEXT: v_mad_i32_i24 v2, s3, v6, v2 ; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 8
; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u16 v0, v7, v8, v0
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v5, v6, v0
; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: v_mad_u16 v0, v9, v10, v0
; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: idot4_acc16_vecMul: ; GFX9-NODL-LABEL: idot4_acc16_vecMul:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v5, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 16 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_lshr_b32 s5, s1, 16 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v4, 8, s5 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_bfe_i32 s5, s5, 0x80000 ; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v3, 8, s4 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: v_and_b32_e32 v6, s5, v5 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX9-NODL-NEXT: s_bfe_i32 s4, s4, 0x80000 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshl_or_b32 v4, v4, 16, v6 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX9-NODL-NEXT: v_and_b32_e32 v6, s4, v5 ; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v7, 8, v1
; GFX9-NODL-NEXT: v_lshl_or_b32 v3, v3, 16, v6 ; GFX9-NODL-NEXT: v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v2, 8, s1 ; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v8, 8, v2
; GFX9-NODL-NEXT: s_bfe_i32 s1, s1, 0x80000 ; GFX9-NODL-NEXT: v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_ashrrev_i16_e64 v1, 8, s0 ; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v8, 16, v2
; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v3, v3, v4 ; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v7, 16, v1
; GFX9-NODL-NEXT: v_and_b32_e32 v4, s1, v5 ; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v10, 8, v6
; GFX9-NODL-NEXT: s_bfe_i32 s0, s0, 0x80000 ; GFX9-NODL-NEXT: v_and_b32_sdwa v6, v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v2, 16, v4
; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v5
; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v1, 16, v4
; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v2 ; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[2:3] ; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v9, 8, v5
; GFX9-NODL-NEXT: v_and_b32_sdwa v4, v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_add_u32_e32 v2, v1, v2 ; GFX9-NODL-NEXT: v_add_u16_e32 v3, v1, v3
; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NODL-NEXT: v_lshl_or_b32 v5, v10, 16, v6
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_lshl_or_b32 v4, v9, 16, v4
; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot4_acc16_vecMul: ; GFX9-DL-LABEL: idot4_acc16_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v5, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 16 ; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 16 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_ashrrev_i16_e64 v4, 8, s5 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX9-DL-NEXT: s_bfe_i32 s5, s5, 0x80000 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_ashrrev_i16_e64 v3, 8, s4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX9-DL-NEXT: v_and_b32_e32 v6, s5, v5 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 8, v1
; GFX9-DL-NEXT: s_bfe_i32 s4, s4, 0x80000 ; GFX9-DL-NEXT: v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v6 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 8, v2
; GFX9-DL-NEXT: v_and_b32_e32 v6, s4, v5 ; GFX9-DL-NEXT: v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v6 ; GFX9-DL-NEXT: v_lshl_or_b32 v2, v8, 16, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e64 v2, 8, s1 ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v7, 16, v1
; GFX9-DL-NEXT: s_bfe_i32 s1, s1, 0x80000 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 8, v6
; GFX9-DL-NEXT: v_ashrrev_i16_e64 v1, 8, s0 ; GFX9-DL-NEXT: v_and_b32_sdwa v6, v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4
; GFX9-DL-NEXT: v_and_b32_e32 v4, s1, v5
; GFX9-DL-NEXT: s_bfe_i32 s0, s0, 0x80000
; GFX9-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v4
; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v5
; GFX9-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v4
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
; GFX9-DL-NEXT: global_load_ushort v2, v0, s[2:3] ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 8, v5
; GFX9-DL-NEXT: v_and_b32_sdwa v4, v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u32_e32 v2, v1, v2 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v1, v3
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_lshl_or_b32 v5, v10, 16, v6
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v9, 16, v4
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
; GFX9-DL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-DL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: idot4_acc16_vecMul: ; GFX10-DL-LABEL: idot4_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0xffff ; GFX10-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_bfe_i32 s1, s6, 0x80000 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_bfe_i32 s2, s7, 0x80000 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: v_and_b32_e32 v6, s1, v2 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v3, 8, s6 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v4, 8, s7 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
; GFX10-DL-NEXT: v_and_b32_e32 v5, s2, v2 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 16 ; GFX10-DL-NEXT: v_ashrrev_i16_e64 v5, 8, v1
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 16 ; GFX10-DL-NEXT: v_and_b32_sdwa v8, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v6 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v7, 8, s0 ; GFX10-DL-NEXT: v_ashrrev_i16_e64 v6, 8, v2
; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5 ; GFX10-DL-NEXT: v_and_b32_sdwa v7, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT: s_bfe_i32 s2, s1, 0x80000 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX10-DL-NEXT: s_bfe_i32 s0, s0, 0x80000 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX10-DL-NEXT: v_and_b32_e32 v6, s2, v2 ; GFX10-DL-NEXT: v_lshl_or_b32 v5, v5, 16, v8
; GFX10-DL-NEXT: v_and_b32_e32 v2, s0, v2 ; GFX10-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4 ; GFX10-DL-NEXT: v_ashrrev_i16_e64 v7, 8, v1
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v5, 8, s1 ; GFX10-DL-NEXT: v_and_b32_sdwa v1, v4, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v7, 16, v2 ; GFX10-DL-NEXT: v_ashrrev_i16_e64 v8, 8, v2
; GFX10-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v6 ; GFX10-DL-NEXT: v_and_b32_sdwa v2, v4, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v4 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v6
; GFX10-DL-NEXT: v_lshl_or_b32 v1, v7, 16, v1
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v8, 16, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v1 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v4, v3
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v2, v3, v5
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v2, v1
; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v3
; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = sext <4 x i8> %vec1 to <4 x i16> %cvec1 = sext <4 x i8> %vec1 to <4 x i16>
%cvec2 = sext <4 x i8> %vec2 to <4 x i16> %cvec2 = sext <4 x i8> %vec2 to <4 x i16>
%mul = mul <4 x i16> %cvec1, %cvec2 %mul = mul <4 x i16> %cvec1, %cvec2
%mul0 = extractelement <4 x i16> %mul, i64 0 %mul0 = extractelement <4 x i16> %mul, i64 0
%mul1 = extractelement <4 x i16> %mul, i64 1 %mul1 = extractelement <4 x i16> %mul, i64 1
%mul2 = extractelement <4 x i16> %mul, i64 2 %mul2 = extractelement <4 x i16> %mul, i64 2
%mul3 = extractelement <4 x i16> %mul, i64 3 %mul3 = extractelement <4 x i16> %mul, i64 3
%acc = load i16, i16 addrspace(1)* %dst, align 4 %acc = load i16, i16 addrspace(1)* %dst, align 4
%add1 = add i16 %mul0, %acc %add1 = add i16 %mul0, %acc
%add2 = add i16 %add1, %mul1 %add2 = add i16 %add1, %mul1
%add3 = add i16 %add2, %mul2 %add3 = add i16 %add2, %mul2
%add4 = add i16 %add3, %mul3 %add4 = add i16 %add3, %mul3
store i16 %add4, i16 addrspace(1)* %dst, align 4 store i16 %add4, i16 addrspace(1)* %dst, align 4
ret void ret void
} }
declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/idot4u.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_acc32: ; GFX7-LABEL: udot4_acc32:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s6, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_and_b32 s7, s5, s8 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s7 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v0, v1 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mov_b32_e32 v1, s9 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s11 ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, s4
; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_acc32: ; GFX8-LABEL: udot4_acc32:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff ; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s5, s3, s2
; GFX8-NEXT: s_and_b32 s2, s4, s2
; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s10
; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1
; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v1, s9 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_lshr_b32 s4, s4, 24 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_lshr_b32 s3, s3, 24 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_mov_b32_e32 v1, s4 ; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0 ; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s3
; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_acc32: ; GFX9-NODL-LABEL: udot4_acc32:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8 ; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10
; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v1, v2
; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_acc32: ; GFX9-DL-LABEL: udot4_acc32:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v1, v2 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, s0
; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_acc32: ; GFX10-DL-LABEL: udot4_acc32:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_dot4_u32_u8 v0, s0, s1, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i32 %cv1e0 = zext i8 %v1e0 to i32
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = zext i8 %v2e0 to i32 %cv2e0 = zext i8 %v2e0 to i32
%mul1 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 25 Lines
} }
define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_acc16: ; GFX7-LABEL: udot4_acc16:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_and_b32 s7, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_and_b32 s6, s5, s8 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_mov_b32_e32 v2, s8 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_acc16: ; GFX8-LABEL: udot4_acc16:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_ushort v1, v[2:3]
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: s_movk_i32 s0, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v5, s0
; GFX8-NEXT: s_and_b32 s3, s1, s0 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_and_b32 s0, s2, s0 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v4
; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008 ; GFX8-NEXT: v_and_b32_e32 v6, s0, v4
; GFX8-NEXT: v_mov_b32_e32 v3, s0 ; GFX8-NEXT: v_and_b32_e32 v8, s0, v8
; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010 ; GFX8-NEXT: v_and_b32_sdwa v10, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v4
; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v0
; GFX8-NEXT: s_lshr_b32 s2, s2, 24 ; GFX8-NEXT: v_and_b32_e32 v7, s0, v0
; GFX8-NEXT: v_mov_b32_e32 v5, s7 ; GFX8-NEXT: v_and_b32_e32 v9, s0, v9
; GFX8-NEXT: s_lshr_b32 s1, s1, 24
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v3, v2 ; GFX8-NEXT: v_mad_u16 v1, v6, v7, v1
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v4, v2 ; GFX8-NEXT: v_and_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2 ; GFX8-NEXT: v_mad_u16 v1, v8, v9, v1
; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2 ; GFX8-NEXT: v_mad_u16 v1, v10, v5, v1
; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_acc16: ; GFX9-NODL-LABEL: udot4_acc16:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3] ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_and_b32 s4, s1, s0 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-NODL-NEXT: s_and_b32 s0, s8, s0 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: s_bfe_u32 s6, s8, 0x80008 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NODL-NEXT: v_and_b32_e32 v0, s0, v2
; GFX9-NODL-NEXT: s_bfe_u32 s9, s8, 0x80010 ; GFX9-NODL-NEXT: v_and_b32_e32 v5, s0, v3
; GFX9-NODL-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX9-NODL-NEXT: v_and_b32_e32 v6, s0, v6
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s6 ; GFX9-NODL-NEXT: v_and_b32_e32 v7, s0, v7
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010
; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s9
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v3, v1 ; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s7, v4, v1 ; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v6, v7, v0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_acc16: ; GFX9-DL-LABEL: udot4_acc16:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-DL-NEXT: v_and_b32_e32 v0, s0, v2
; GFX9-DL-NEXT: v_and_b32_e32 v5, s0, v3
; GFX9-DL-NEXT: v_and_b32_e32 v6, s0, v6
; GFX9-DL-NEXT: v_and_b32_e32 v7, s0, v7
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v4
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_and_b32_sdwa v9, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v6, v7, v0
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_acc16: ; GFX10-DL-LABEL: udot4_acc16:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_movk_i32 s0, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s6, s7, v1 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX10-DL-NEXT: v_and_b32_e32 v4, s0, v1
; GFX10-DL-NEXT: v_and_b32_e32 v10, s0, v2
; GFX10-DL-NEXT: v_and_b32_e32 v5, s0, v5
; GFX10-DL-NEXT: v_and_b32_e32 v6, s0, v6
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
; GFX10-DL-NEXT: v_and_b32_sdwa v4, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-DL-NEXT: v_and_b32_sdwa v10, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: v_mad_u16 v3, v5, v6, v3
; GFX10-DL-NEXT: v_mad_u16 v3, v4, v10, v3
; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
; GFX10-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i16 %cv1e0 = zext i8 %v1e0 to i16
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = zext i8 %v2e0 to i16 %cv2e0 = zext i8 %v2e0 to i16
%mul1 = mul nuw nsw i16 %cv1e0, %cv2e0 %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 25 Lines
} }
define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_acc8: ; GFX7-LABEL: udot4_acc8:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s7, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_and_b32 s6, s5, s8 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v2, s8 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 ; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v8
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_acc8: ; GFX8-LABEL: udot4_acc8:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_and_b32 s3, s2, s0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_u32 s4, s2, 0x80008 ; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
; GFX8-NEXT: s_and_b32 s0, s1, s0 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v3, s3 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_bfe_u32 s6, s2, 0x80010 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
; GFX8-NEXT: v_mov_b32_e32 v4, s4 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX8-NEXT: s_bfe_u32 s7, s1, 0x80010 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v4
; GFX8-NEXT: s_lshr_b32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: s_lshr_b32 s1, s1, 24
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v5
; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v7, v8, v0
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: v_mad_u16 v0, v1, v6, v0
; GFX8-NEXT: v_mad_u16 v0, v9, v10, v0
; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_acc8: ; GFX9-NODL-LABEL: udot4_acc8:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_bfe_u32 s6, s1, 0x80008 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-NODL-NEXT: s_bfe_u32 s5, s8, 0x80008 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-NODL-NEXT: s_bfe_u32 s7, s8, 0x80010
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010
; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v3, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_acc8: ; GFX9-DL-LABEL: udot4_acc8:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v7, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v9, v0
; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_acc8: ; GFX10-DL-LABEL: udot4_acc8:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s6, s7, v1 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 8, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v4, v2, v3, v4
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%mul1 = mul nuw nsw i8 %v1e0, %v2e0 %mul1 = mul nuw nsw i8 %v1e0, %v2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
%v2e1 = extractelement <4 x i8> %vec2, i64 1 %v2e1 = extractelement <4 x i8> %vec2, i64 1
%mul2 = mul nuw nsw i8 %v1e1, %v2e1 %mul2 = mul nuw nsw i8 %v1e1, %v2e1
Show All 18 Lines
; TODO: Generate udot4? ; TODO: Generate udot4?
define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot2_8: ; GFX7-LABEL: udot2_8:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_and_b32 s7, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_and_b32 s6, s5, s8 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s5, s5, 0x80008 ; GFX7-NEXT: buffer_load_ubyte v4, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s4, s4, 0x80008 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v3, s4, v0
; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot2_8: ; GFX8-LABEL: udot2_8:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_and_b32 s3, s2, s0 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_and_b32 s0, s1, s0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s3 ; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
; GFX8-NEXT: s_bfe_u32 s2, s2, 0x80008 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_bfe_u32 s1, s1, 0x80008 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v4
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v5
; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v1, v6, v0
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot2_8: ; GFX9-NODL-LABEL: udot2_8:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s8, s0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_and_b32 s0, s1, s0 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-NODL-NEXT: s_bfe_u32 s5, s8, 0x80008 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_bfe_u32 s1, s1, 0x80008 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 8, v2
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot2_8: ; GFX9-DL-LABEL: udot2_8:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_movk_i32 s0, 0xff ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s4, s8, s0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_and_b32 s0, s1, s0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-DL-NEXT: s_bfe_u32 s5, s8, 0x80008 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x80008 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 8, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s5 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot2_8: ; GFX10-DL-LABEL: udot2_8:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-NEXT: s_movk_i32 s0, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_and_b32 s1, s7, s0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_and_b32 s0, s6, s0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 8, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v2, v2, v3, v4
; GFX10-DL-NEXT: s_bfe_u32 s0, s7, 0x80008 ; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v2
; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x80008 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%mul1 = mul nuw nsw i8 %v1e0, %v2e0 %mul1 = mul nuw nsw i8 %v1e0, %v2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
%v2e1 = extractelement <4 x i8> %vec2, i64 1 %v2e1 = extractelement <4 x i8> %vec2, i64 1
%mul2 = mul nuw nsw i8 %v1e1, %v2e1 %mul2 = mul nuw nsw i8 %v1e1, %v2e1
%acc = load i8, i8 addrspace(1)* %dst, align 2 %acc = load i8, i8 addrspace(1)* %dst, align 2
%mad1 = add i8 %mul1, %acc %mad1 = add i8 %mul1, %acc
%mad2 = add i8 %mad1, %mul2 %mad2 = add i8 %mad1, %mul2
store i8 %mad2, i8 addrspace(1)* %dst, align 2 store i8 %mad2, i8 addrspace(1)* %dst, align 2
ret void ret void
} }
define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_CommutationInsideMAD: ; GFX7-LABEL: udot4_CommutationInsideMAD:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s6, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_and_b32 s7, s5, s8 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v2, s8 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 ; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v1, v8
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s4 ; GFX7-NEXT: v_mad_u32_u24 v1, v6, v3, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_CommutationInsideMAD: ; GFX8-LABEL: udot4_CommutationInsideMAD:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_movk_i32 s0, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s3, s1, s0
; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
; GFX8-NEXT: s_and_b32 s0, s2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010
; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v4, s4
; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010
; GFX8-NEXT: s_lshr_b32 s1, s1, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: s_lshr_b32 s2, s2, 24
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ubyte v5, v[2:3]
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v0
; GFX8-NEXT: v_mad_u16 v0, v0, v4, v5
; GFX8-NEXT: v_mad_u16 v0, v8, v7, v0
; GFX8-NEXT: v_mad_u16 v0, v6, v1, v0
; GFX8-NEXT: v_mad_u16 v0, v10, v9, v0
; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_CommutationInsideMAD: ; GFX9-NODL-LABEL: udot4_CommutationInsideMAD:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s1, s0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_and_b32 s0, s8, s0 ; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-NODL-NEXT: s_bfe_u32 s6, s8, 0x80008 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-NODL-NEXT: s_bfe_u32 s9, s8, 0x80010 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v3, v2, v4
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v3, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v7, v6, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v5, v0, v2
; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v9, v8, v0
; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_CommutationInsideMAD: ; GFX9-DL-LABEL: udot4_CommutationInsideMAD:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_u32_u8 v1, s1, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v3, v2, v4
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v6, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 24, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v5, v0, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v9, v8, v0
; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_CommutationInsideMAD: ; GFX10-DL-LABEL: udot4_CommutationInsideMAD:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: v_dot4_u32_u8 v1, s7, s6, v1 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 8, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v4, v3, v2, v4
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX10-DL-NEXT: v_mad_u16 v0, v5, v11, v4
; GFX10-DL-NEXT: v_mad_u16 v0, v7, v6, v0
; GFX10-DL-NEXT: v_mad_u16 v0, v3, v2, v0
; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%mul1 = mul nuw nsw i8 %v2e0, %v1e0 %mul1 = mul nuw nsw i8 %v2e0, %v1e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
%v2e1 = extractelement <4 x i8> %vec2, i64 1 %v2e1 = extractelement <4 x i8> %vec2, i64 1
%mul2 = mul nuw nsw i8 %v2e1, %v1e1 %mul2 = mul nuw nsw i8 %v2e1, %v1e1
Show All 18 Lines
; TODO: Support commutation accross the adds. ; TODO: Support commutation accross the adds.
define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_CommutationAccrossMADs: ; GFX7-LABEL: udot4_CommutationAccrossMADs:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_and_b32 s6, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_and_b32 s7, s5, s8 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 ; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
; GFX7-NEXT: v_mov_b32_e32 v1, s8 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_mov_b32_e32 v2, s6 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v3, v6, v3, v8
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v2, v0 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s4 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v1, v3
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_CommutationAccrossMADs: ; GFX8-LABEL: udot4_CommutationAccrossMADs:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
; GFX8-NEXT: s_and_b32 s3, s1, s0
; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v3, s4
; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010
; GFX8-NEXT: s_and_b32 s0, s2, s0
; GFX8-NEXT: v_mov_b32_e32 v4, s3
; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010
; GFX8-NEXT: s_lshr_b32 s1, s1, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: s_lshr_b32 s2, s2, 24
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v3, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v4, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_load_ubyte v10, v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v8, 24, v4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u16 v6, v7, v6, v10
; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v0
; GFX8-NEXT: v_mad_u16 v0, v0, v4, v6
; GFX8-NEXT: v_mad_u16 v0, v5, v1, v0
; GFX8-NEXT: v_mad_u16 v0, v9, v8, v0
; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_CommutationAccrossMADs: ; GFX9-NODL-LABEL: udot4_CommutationAccrossMADs:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_and_b32 s4, s1, s0 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_bfe_u32 s6, s8, 0x80008 ; GFX9-NODL-NEXT: global_load_ubyte v9, v1, s[2:3]
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-NODL-NEXT: s_and_b32 s0, s8, s0 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
; GFX9-NODL-NEXT: s_bfe_u32 s9, s8, 0x80010
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s7
; GFX9-NODL-NEXT: s_lshr_b32 s8, s8, 24
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v5, v6, v5, v9
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v3, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 24, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v3, v2, v5
; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v3
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v4, v0, v2
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v8, v7, v0
; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_CommutationAccrossMADs: ; GFX9-DL-LABEL: udot4_CommutationAccrossMADs:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_movk_i32 s0, 0xff ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_and_b32 s4, s1, s0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_bfe_u32 s6, s8, 0x80008 ; GFX9-DL-NEXT: global_load_ubyte v9, v1, s[2:3]
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s5 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80010 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-DL-NEXT: s_and_b32 s0, s8, s0 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
; GFX9-DL-NEXT: s_bfe_u32 s9, s8, 0x80010
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s7
; GFX9-DL-NEXT: s_lshr_b32 s8, s8, 24
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v5, v6, v5, v9
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v3, v1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 24, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v3, v2, v5
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v4, v0, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v8, v7, v0
; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_CommutationAccrossMADs: ; GFX10-DL-LABEL: udot4_CommutationAccrossMADs:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-NEXT: s_movk_i32 s2, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80008 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80008 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v5, v7, v4
; GFX10-DL-NEXT: s_and_b32 s0, s6, s2 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX10-DL-NEXT: s_and_b32 s1, s7, s2 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v3, v2, v0
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80010 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80010 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v5, v7, v0
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 24 ; GFX10-DL-NEXT: v_mad_u16 v0, v3, v2, v0
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 24 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s1, s0, v1
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%mul1 = mul nuw nsw i8 %v2e0, %v1e0 %mul1 = mul nuw nsw i8 %v2e0, %v1e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
%v2e1 = extractelement <4 x i8> %vec2, i64 1 %v2e1 = extractelement <4 x i8> %vec2, i64 1
%mul2 = mul nuw nsw i8 %v2e1, %v1e1 %mul2 = mul nuw nsw i8 %v2e1, %v1e1
Show All 16 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_multiuse_mul1: ; GFX7-LABEL: udot4_multiuse_mul1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s6, s4, s8 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_and_b32 s7, s5, s8 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s7 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mad_u32_u24 v1, s6, v0, v1 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: v_mov_b32_e32 v2, s9 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v0, v1 ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s11 ; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v8, v1, v5, s4
; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, v8
; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_multiuse_mul1: ; GFX8-LABEL: udot4_multiuse_mul1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff ; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s5, s3, s2 ; GFX8-NEXT: v_mad_u32_u24 v8, v1, v2, s3
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, v8
; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX8-NEXT: v_mov_b32_e32 v0, s2 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v4
; GFX8-NEXT: v_mov_b32_e32 v1, s10 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008 ; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
; GFX8-NEXT: v_mad_u32_u24 v1, s5, v0, v1 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s7
; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010
; GFX8-NEXT: v_mad_u32_u24 v1, s6, v2, v1
; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_lshr_b32 s4, s4, 24
; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0
; GFX8-NEXT: s_lshr_b32 s3, s3, 24
; GFX8-NEXT: v_mov_b32_e32 v1, s4
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_multiuse_mul1: ; GFX9-NODL-LABEL: udot4_multiuse_mul1:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008 ; GFX9-NODL-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008 ; GFX9-NODL-NEXT: v_and_b32_e32 v3, s0, v1
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s4, v1, v2 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s7 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, s0, v2
; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s6, v3, v2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v1, v2 ; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v2, v3, v4
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v3, v4, s1
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX9-NODL-NEXT: v_add3_u32 v2, v5, v3, v2
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_multiuse_mul1: ; GFX9-DL-LABEL: udot4_multiuse_mul1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_movk_i32 s8, 0xff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s10, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_and_b32 s4, s0, s8 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_and_b32 s5, s1, s8 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80008 ; GFX9-DL-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s10 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80008 ; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v1
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s4, v1, v2 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7 ; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v2
; GFX9-DL-NEXT: s_bfe_u32 s9, s1, 0x80010 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v3, v2 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x80010 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v1, v2 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v4
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, v3, v4, s1
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX9-DL-NEXT: v_add3_u32 v2, v5, v3, v2
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX9-DL-NEXT: v_add3_u32 v1, v2, v6, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_multiuse_mul1: ; GFX10-DL-LABEL: udot4_multiuse_mul1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_movk_i32 s3, 0xff
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX10-DL-NEXT: s_movk_i32 s4, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_and_b32 s5, s0, s4 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_and_b32 s4, s1, s4 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x80008 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x80008 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_and_b32_e32 v0, s3, v1
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s6, s7, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s4, v0 ; GFX10-DL-NEXT: v_and_b32_e32 v3, s3, v2
; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80010 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80010 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v0, v3
; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v3, s2
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s0, s1, v0 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: v_add3_u32 v0, v7, v0, v5
; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i32 %cv1e0 = zext i8 %v1e0 to i32
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = zext i8 %v2e0 to i32 %cv2e0 = zext i8 %v2e0 to i32
%mul1 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 25 Linesentry:
ret void ret void
} }
define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_multiuse_add1: ; GFX7-LABEL: udot4_multiuse_add1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s6, s4, s8 ; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, s5
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 ; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: s_and_b32 s7, s5, s8 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v0, s9 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v3
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v0, v1 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s7 ; GFX7-NEXT: v_add_i32_e32 v6, vcc, s5, v3
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: v_add_i32_e32 v1, vcc, s12, v0 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v6
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v2, v0
; GFX7-NEXT: v_mov_b32_e32 v2, s11
; GFX7-NEXT: s_lshr_b32 s5, s5, 24
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v2, v0
; GFX7-NEXT: s_lshr_b32 s4, s4, 24
; GFX7-NEXT: v_mov_b32_e32 v2, s5
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v2, v0
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_multiuse_add1: ; GFX8-LABEL: udot4_multiuse_add1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff ; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
; GFX8-NEXT: v_and_b32_e32 v1, s2, v3
; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s5, s3, s2 ; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, s3
; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX8-NEXT: v_and_b32_e32 v2, s2, v0
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v4
; GFX8-NEXT: v_mov_b32_e32 v0, s7 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s10 ; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
; GFX8-NEXT: v_mad_u32_u24 v0, s6, v0, v1 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, s3, v4
; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010 ; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s2 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v5
; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010
; GFX8-NEXT: v_add_u32_e32 v1, vcc, s10, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v2, s9
; GFX8-NEXT: s_lshr_b32 s4, s4, 24
; GFX8-NEXT: v_mad_u32_u24 v0, s8, v2, v0
; GFX8-NEXT: s_lshr_b32 s3, s3, 24
; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: v_mad_u32_u24 v0, s3, v2, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_multiuse_add1: ; GFX9-NODL-LABEL: udot4_multiuse_add1:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_load_dword s10, s[2:3], 0x0 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008 ; GFX9-NODL-NEXT: v_bfe_u32 v4, v1, 8, 8
; GFX9-NODL-NEXT: s_and_b32 s5, s1, s8 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s7 ; GFX9-NODL-NEXT: v_bfe_u32 v5, v2, 8, 8
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v1, v2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: s_and_b32 s4, s0, s8 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v4, v5, s0
; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010 ; GFX9-NODL-NEXT: v_add_u32_e32 v4, s0, v2
; GFX9-NODL-NEXT: v_add_u32_e32 v2, s10, v1 ; GFX9-NODL-NEXT: v_add3_u32 v2, v2, v3, v6
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v3, v1 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v1, v4
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s9
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v3, v1
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v3, v1
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_multiuse_add1: ; GFX9-DL-LABEL: udot4_multiuse_add1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_movk_i32 s8, 0xff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s10, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80008 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80008 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_and_b32 s5, s1, s8 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s7 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s10 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v1, v2 ; GFX9-DL-NEXT: v_bfe_u32 v4, v1, 8, 8
; GFX9-DL-NEXT: s_and_b32 s4, s0, s8 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s9, s1, 0x80010 ; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 8, 8
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x80010 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT: v_add_u32_e32 v2, s10, v1 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v3, v1 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s9 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v4, v5, s0
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX9-DL-NEXT: v_add_u32_e32 v4, s0, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v3, v1 ; GFX9-DL-NEXT: v_add3_u32 v2, v2, v3, v6
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX9-DL-NEXT: v_add3_u32 v1, v2, v1, v4
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v3, v1
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_multiuse_add1: ; GFX10-DL-LABEL: udot4_multiuse_add1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_movk_i32 s6, 0xff ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 8, 8
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: v_mad_u32_u24 v6, v0, v3, s2
; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80008 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, s2, v6
; GFX10-DL-NEXT: s_and_b32 s4, s0, s6 ; GFX10-DL-NEXT: v_add3_u32 v0, v6, v4, v3
; GFX10-DL-NEXT: s_and_b32 s5, s1, s6 ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s4, s5, v0 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80010 ; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80010
; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s4, s5, v1
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, s8, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v0
; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = zext i8 %v1e0 to i32 %cv1e0 = zext i8 %v1e0 to i32
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = zext i8 %v2e0 to i32 %cv2e0 = zext i8 %v2e0 to i32
%mul1 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 26 Lines
} }
define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: notdot4_mixedtypes: ; GFX7-LABEL: notdot4_mixedtypes:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s8, 0xffff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i8 s6, s4 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_sext_i32_i8 s7, s5 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_and_b32 s7, s7, s8 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80008 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v1, s9 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_and_b32 s6, s6, s8 ; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0
; GFX7-NEXT: v_mov_b32_e32 v3, s7 ; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x80010 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_lshr_b32 s5, s5, 24 ; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 8
; GFX7-NEXT: v_mov_b32_e32 v2, s11 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v6, v0, 0, 8
; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
; GFX7-NEXT: v_and_b32_e32 v3, s4, v3
; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v3, v0 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0 ; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: notdot4_mixedtypes: ; GFX8-LABEL: notdot4_mixedtypes:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: v_mov_b32_e32 v5, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_u32 s4, s0, 0x80008 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80008 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_sext_i32_i8 s3, s1
; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: s_bfe_u32 s7, s1, 0x80010
; GFX8-NEXT: s_sext_i32_i8 s2, s0
; GFX8-NEXT: v_mov_b32_e32 v4, s3
; GFX8-NEXT: s_bfe_u32 s6, s0, 0x80010
; GFX8-NEXT: s_lshr_b32 s1, s1, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s7
; GFX8-NEXT: s_lshr_b32 s0, s0, 24
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v3, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v4, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: flat_load_ushort v10, v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4
; GFX8-NEXT: v_and_b32_e32 v7, s2, v7
; GFX8-NEXT: v_bfe_i32 v1, v4, 0, 8
; GFX8-NEXT: v_and_b32_sdwa v9, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v0
; GFX8-NEXT: v_and_b32_e32 v8, s2, v8
; GFX8-NEXT: v_bfe_i32 v6, v0, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u16 v7, v7, v8, v10
; GFX8-NEXT: v_and_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u16 v1, v1, v6, v7
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX8-NEXT: v_mad_u16 v1, v9, v5, v1
; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: notdot4_mixedtypes: ; GFX9-NODL-LABEL: notdot4_mixedtypes:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[2:3] ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80008 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80008 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s1 ; GFX9-NODL-NEXT: global_load_ushort v9, v1, s[2:3]
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: s_bfe_u32 s9, s1, 0x80010 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
; GFX9-NODL-NEXT: s_bfe_u32 s8, s0, 0x80010 ; GFX9-NODL-NEXT: v_and_b32_e32 v5, s0, v5
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX9-NODL-NEXT: v_and_b32_e32 v6, s0, v6
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s9 ; GFX9-NODL-NEXT: v_bfe_i32 v0, v2, 0, 8
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX9-NODL-NEXT: v_bfe_i32 v4, v3, 0, 8
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v5, v5, v6, v9
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v3, v1 ; GFX9-NODL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s8, v4, v1 ; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v7, v8, v0
; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-NODL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: notdot4_mixedtypes: ; GFX9-DL-LABEL: notdot4_mixedtypes:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3] ; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80008 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80008 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_sext_i32_i8 s5, s1 ; GFX9-DL-NEXT: global_load_ushort v9, v1, s[2:3]
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_bfe_u32 s9, s1, 0x80010 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-DL-NEXT: s_sext_i32_i8 s4, s0 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v3
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x80010 ; GFX9-DL-NEXT: v_and_b32_e32 v5, s0, v5
; GFX9-DL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX9-DL-NEXT: v_and_b32_e32 v6, s0, v6
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s9 ; GFX9-DL-NEXT: v_bfe_i32 v0, v2, 0, 8
; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX9-DL-NEXT: v_bfe_i32 v4, v3, 0, 8
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v5, v5, v6, v9
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v3, v1 ; GFX9-DL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v4, v1 ; GFX9-DL-NEXT: v_and_b32_sdwa v8, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v4, v5
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v7, v8, v0
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v0
; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: notdot4_mixedtypes: ; GFX10-DL-LABEL: notdot4_mixedtypes:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NEXT: s_movk_i32 s0, 0xff
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80008 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80008 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v9, v2, 0, 8
; GFX10-DL-NEXT: v_and_b32_e32 v4, s0, v4
; GFX10-DL-NEXT: v_and_b32_e32 v5, s0, v5
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
; GFX10-DL-NEXT: s_sext_i32_i8 s0, s6 ; GFX10-DL-NEXT: v_and_b32_sdwa v4, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-DL-NEXT: s_sext_i32_i8 s1, s7 ; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x80010 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x80010 ; GFX10-DL-NEXT: v_mad_u16 v3, v6, v9, v3
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 24 ; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 24 ; GFX10-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%v1e0 = extractelement <4 x i8> %vec1, i64 0 %v1e0 = extractelement <4 x i8> %vec1, i64 0
%cv1e0 = sext i8 %v1e0 to i16 %cv1e0 = sext i8 %v1e0 to i16
%v2e0 = extractelement <4 x i8> %vec2, i64 0 %v2e0 = extractelement <4 x i8> %vec2, i64 0
%cv2e0 = sext i8 %v2e0 to i16 %cv2e0 = sext i8 %v2e0 to i16
%mul1 = mul nuw nsw i16 %cv1e0, %cv2e0 %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
%v1e1 = extractelement <4 x i8> %vec1, i64 1 %v1e1 = extractelement <4 x i8> %vec1, i64 1
Show All 25 Lines
} }
; TODO: cleanup s_lshr_b32 and support this pattern. ; TODO: cleanup s_lshr_b32 and support this pattern.
define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_acc32_vecMul: ; GFX7-LABEL: udot4_acc32_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_movk_i32 s11, 0xff
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 24 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_lshr_b32 s8, s5, 24 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_and_b32 s5, s5, s11 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_and_b32 s4, s4, s11 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_load_dword s11, s[0:1], 0x0 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mov_b32_e32 v0, s5 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s11 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1 ; GFX7-NEXT: v_and_b32_e32 v2, s4, v2
; GFX7-NEXT: v_mov_b32_e32 v1, s9 ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s8 ; GFX7-NEXT: v_and_b32_e32 v0, s4, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s4
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v4, v7, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v1, v5, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_acc32_vecMul: ; GFX8-LABEL: udot4_acc32_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff ; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_lshr_b32 s5, s3, 24 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_lshr_b32 s6, s4, 24 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_u32 s7, s3, 0x80010 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_lshrrev_b16_e64 v0, 8, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_and_b32 s3, s3, s2 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_and_b32 s2, s4, s2 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX8-NEXT: s_bfe_u32 s8, s4, 0x80010 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b16_e64 v1, 8, s4 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v3
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 8
; GFX8-NEXT: v_mov_b32_e32 v2, s2 ; GFX8-NEXT: v_lshrrev_b16_e32 v5, 8, v3
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_and_b32_e32 v3, s2, v3
; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v2, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v0
; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, v2 ; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 8
; GFX8-NEXT: v_mov_b32_e32 v1, s8 ; GFX8-NEXT: v_lshrrev_b16_e32 v7, 8, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, s3
; GFX8-NEXT: v_mad_u32_u24 v0, v5, v7, v0
; GFX8-NEXT: v_mad_u32_u24 v0, v4, v6, v0
; GFX8-NEXT: v_mad_u32_u24 v2, v1, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_acc32_vecMul: ; GFX9-NODL-LABEL: udot4_acc32_vecMul:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: s_movk_i32 s8, 0xff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 24 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_lshr_b32 s5, s1, 24 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_bfe_u32 s6, s0, 0x80010 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_and_b32 s0, s0, s8 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: s_bfe_u32 s7, s1, 0x80010 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v2, 8, s1 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: s_and_b32 s1, s1, s8 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s8 ; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
; GFX9-NODL-NEXT: v_mad_u32_u24 v3, s0, v3, v4 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v1, v2, v3
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s5
; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_acc32_vecMul: ; GFX9-DL-LABEL: udot4_acc32_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_movk_i32 s8, 0xff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 24 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 24 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x80010 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_and_b32 s0, s0, s8 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s7, s1, 0x80010 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s1 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DL-NEXT: s_and_b32 s1, s1, s8 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s8 ; GFX9-DL-NEXT: v_add3_u32 v2, v3, s0, v4
; GFX9-DL-NEXT: v_mad_u32_u24 v3, s0, v3, v4 ; GFX9-DL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-DL-NEXT: v_mad_u32_u24 v1, v1, v2, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v2, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s5
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_acc32_vecMul: ; GFX10-DL-LABEL: udot4_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_mov_b32 s3, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_movk_i32 s5, 0xff ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_mov_b32 s4, 0xffff ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_and_b32_sdwa v0, s3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_and_b32_sdwa v3, s3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT: v_mul_u32_u24_e32 v0, v0, v3
; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_add3_u32 v0, v7, s2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v3, v1
; GFX10-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: s_and_b32 s6, s0, s5
; GFX10-DL-NEXT: s_and_b32 s5, s1, s5
; GFX10-DL-NEXT: v_and_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX10-DL-NEXT: v_and_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX10-DL-NEXT: v_mad_u32_u24 v2, s6, s5, v2
; GFX10-DL-NEXT: s_bfe_u32 s4, s0, 0x80010
; GFX10-DL-NEXT: s_bfe_u32 s5, s1, 0x80010
; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24
; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v1, v2
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s0, s1, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = zext <4 x i8> %vec1 to <4 x i32> %cvec1 = zext <4 x i8> %vec1 to <4 x i32>
%cvec2 = zext <4 x i8> %vec2 to <4 x i32> %cvec2 = zext <4 x i8> %vec2 to <4 x i32>
%mul = mul <4 x i32> %cvec1, %cvec2 %mul = mul <4 x i32> %cvec1, %cvec2
%mul0 = extractelement <4 x i32> %mul, i64 0 %mul0 = extractelement <4 x i32> %mul, i64 0
%mul1 = extractelement <4 x i32> %mul, i64 1 %mul1 = extractelement <4 x i32> %mul, i64 1
%mul2 = extractelement <4 x i32> %mul, i64 2 %mul2 = extractelement <4 x i32> %mul, i64 2
Show All 11 Lines
; TODO: This pattern should be recognized. ; TODO: This pattern should be recognized.
define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_acc16_vecMul: ; GFX7-LABEL: udot4_acc16_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_lshr_b32 s6, s4, 24 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_lshr_b32 s9, s5, 24 ; GFX7-NEXT: buffer_load_ushort v8, off, s[0:3], 0
; GFX7-NEXT: s_and_b32 s5, s5, s8 ; GFX7-NEXT: s_mov_b32 s4, 0xff00
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX7-NEXT: s_movk_i32 s5, 0xff
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_and_b32 s4, s4, s8 ; GFX7-NEXT: v_and_b32_e32 v1, s4, v2
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_and_b32_e32 v3, s5, v2
; GFX7-NEXT: v_mov_b32_e32 v2, s10 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; GFX7-NEXT: v_mov_b32_e32 v3, s12 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v5, s4, v0
; GFX7-NEXT: v_or_b32_e32 v1, v3, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v5
; GFX7-NEXT: v_and_b32_e32 v6, s5, v0
; GFX7-NEXT: v_or_b32_e32 v3, v6, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v3
; GFX7-NEXT: v_and_b32_e32 v1, s5, v1
; GFX7-NEXT: v_and_b32_e32 v3, s5, v3
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v8
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v2, v0 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
; GFX7-NEXT: v_mov_b32_e32 v1, s9 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v6, v1
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_acc16_vecMul: ; GFX8-LABEL: udot4_acc16_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: v_mov_b32_e32 v5, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_and_b32 s6, s1, s0 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_and_b32 s0, s2, s0 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, s0 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010 ; GFX8-NEXT: flat_load_ushort v10, v[2:3]
; GFX8-NEXT: v_lshrrev_b16_e64 v3, 8, s2 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b16_e64 v4, 8, s1 ; GFX8-NEXT: v_and_b32_sdwa v9, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_lshr_b32 s4, s2, 24 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v4
; GFX8-NEXT: s_bfe_u32 s5, s1, 0x80010 ; GFX8-NEXT: v_lshrrev_b16_e32 v6, 8, v4
; GFX8-NEXT: v_mov_b32_e32 v6, s7 ; GFX8-NEXT: v_and_b32_e32 v4, s2, v4
; GFX8-NEXT: s_lshr_b32 s3, s1, 24 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 24, v0
; GFX8-NEXT: v_lshrrev_b16_e32 v8, 8, v0
; GFX8-NEXT: v_and_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_and_b32_e32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v10
; GFX8-NEXT: v_mad_u32_u24 v2, v4, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v6, v8, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v6, v2 ; GFX8-NEXT: v_mad_u16 v0, v9, v5, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u16 v0, v1, v7, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v3, v2 ; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_acc16_vecMul: ; GFX9-NODL-LABEL: udot4_acc16_vecMul:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, 0xffff ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_lshr_b32 s5, s0, 16 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: s_lshr_b32 s7, s1, 16 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 24 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: v_and_b32_sdwa v5, v3, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX9-NODL-NEXT: s_lshr_b32 s6, s1, 24 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: v_and_b32_sdwa v4, v3, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v5, 8, v1
; GFX9-NODL-NEXT: v_lshl_or_b32 v4, s6, 16, v4 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshl_or_b32 v5, s4, 16, v5 ; GFX9-NODL-NEXT: v_and_b32_sdwa v10, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v4, v5, v4 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 24, v1
; GFX9-NODL-NEXT: v_and_b32_sdwa v5, v3, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v2, 8, s1 ; GFX9-NODL-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s0 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-NODL-NEXT: v_and_b32_sdwa v3, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v2, 16, v5 ; GFX9-NODL-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v1, 16, v3 ; GFX9-NODL-NEXT: v_lshl_or_b32 v2, v7, 16, v2
; GFX9-NODL-NEXT: v_lshl_or_b32 v1, v5, 16, v1
; GFX9-NODL-NEXT: v_and_b32_e32 v10, v4, v10
; GFX9-NODL-NEXT: v_and_b32_e32 v4, v4, v9
; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v2 ; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[2:3]
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_add_u32_e32 v2, v1, v2 ; GFX9-NODL-NEXT: v_add_u16_e32 v3, v1, v3
; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NODL-NEXT: v_lshl_or_b32 v5, v8, 16, v10
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v4 ; GFX9-NODL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_acc16_vecMul: ; GFX9-DL-LABEL: udot4_acc16_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0xffff ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_lshr_b32 s5, s0, 16 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 16 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 24 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: v_and_b32_sdwa v5, v3, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX9-DL-NEXT: s_lshr_b32 s6, s1, 24 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_and_b32_sdwa v4, v3, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_lshrrev_b16_e32 v5, 8, v1
; GFX9-DL-NEXT: v_lshl_or_b32 v4, s6, 16, v4 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshl_or_b32 v5, s4, 16, v5 ; GFX9-DL-NEXT: v_and_b32_sdwa v10, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 24, v1
; GFX9-DL-NEXT: v_and_b32_sdwa v5, v3, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_and_b32_sdwa v9, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s1 ; GFX9-DL-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s0 ; GFX9-DL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-DL-NEXT: v_and_b32_sdwa v3, v3, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 24, v2
; GFX9-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v5 ; GFX9-DL-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v3 ; GFX9-DL-NEXT: v_lshl_or_b32 v2, v7, 16, v2
; GFX9-DL-NEXT: v_lshl_or_b32 v1, v5, 16, v1
; GFX9-DL-NEXT: v_and_b32_e32 v10, v4, v10
; GFX9-DL-NEXT: v_and_b32_e32 v4, v4, v9
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
; GFX9-DL-NEXT: global_load_ushort v2, v0, s[2:3]
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u32_e32 v2, v1, v2 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v1, v3
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_lshl_or_b32 v5, v8, 16, v10
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v4 ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
; GFX9-DL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-DL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_acc16_vecMul: ; GFX10-DL-LABEL: udot4_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0xffff ; GFX10-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_movk_i32 s2, 0xff
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s6 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_and_b32_sdwa v6, v2, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, s7 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_lshr_b32 s0, s7, 16 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
; GFX10-DL-NEXT: s_lshr_b32 s1, s6, 16 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v6 ; GFX10-DL-NEXT: v_lshrrev_b16_e64 v5, 8, v1
; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v5 ; GFX10-DL-NEXT: v_and_b32_sdwa v8, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_and_b32_sdwa v2, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX10-DL-NEXT: v_lshrrev_b16_e64 v6, 8, v2
; GFX10-DL-NEXT: s_lshr_b32 s1, s6, 24 ; GFX10-DL-NEXT: v_and_b32_sdwa v7, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT: s_lshr_b32 s0, s7, 24 ; GFX10-DL-NEXT: v_and_b32_sdwa v9, v1, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4 ; GFX10-DL-NEXT: v_and_b32_sdwa v10, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX10-DL-NEXT: v_lshl_or_b32 v4, s0, 16, v5 ; GFX10-DL-NEXT: v_lshl_or_b32 v5, v5, 16, v8
; GFX10-DL-NEXT: v_lshl_or_b32 v2, s1, 16, v2 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v4 ; GFX10-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: v_and_b32_e32 v7, v4, v10
; GFX10-DL-NEXT: v_and_b32_e32 v4, v4, v9
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v5, v5, v6
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v7
; GFX10-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v4
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v1 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v5, v3
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v2, v3, v4
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v2, v1
; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v3
; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%cvec1 = zext <4 x i8> %vec1 to <4 x i16> %cvec1 = zext <4 x i8> %vec1 to <4 x i16>
%cvec2 = zext <4 x i8> %vec2 to <4 x i16> %cvec2 = zext <4 x i8> %vec2 to <4 x i16>
%mul = mul <4 x i16> %cvec1, %cvec2 %mul = mul <4 x i16> %cvec1, %cvec2
%mul0 = extractelement <4 x i16> %mul, i64 0 %mul0 = extractelement <4 x i16> %mul, i64 0
%mul1 = extractelement <4 x i16> %mul, i64 1 %mul1 = extractelement <4 x i16> %mul, i64 1
%mul2 = extractelement <4 x i16> %mul, i64 2 %mul2 = extractelement <4 x i16> %mul, i64 2
Show All 11 Lines
; TODO: Support this pattern. ; TODO: Support this pattern.
define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1, define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-LABEL: udot4_acc8_vecMul: ; GFX7-LABEL: udot4_acc8_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_bfe_u32 s6, s4, 0x80008 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_lshr_b32 s11, s5, 16 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_lshr_b32 s12, s5, 24 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_lshr_b32 s7, s4, 16 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v2, s11 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_lshr_b32 s9, s4, 24 ; GFX7-NEXT: buffer_load_ubyte v8, off, s[0:3], 0
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_mul_i32 s4, s4, s5 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_mul_u32_u24_e32 v1, s9, v1 ; GFX7-NEXT: v_and_b32_e32 v3, s4, v2
; GFX7-NEXT: v_mul_u32_u24_e32 v2, s7, v2 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
; GFX7-NEXT: v_mul_u32_u24_e32 v3, s6, v3 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_and_b32 s5, s4, s8 ; GFX7-NEXT: v_and_b32_e32 v6, s4, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1 ; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
; GFX7-NEXT: v_and_b32_e32 v2, s8, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
; GFX7-NEXT: v_or_b32_e32 v2, s5, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 8, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, s4, v0 ; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, v8
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v3 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v0
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v3, v4, v7, v3
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v3
; GFX7-NEXT: v_mad_u32_u24 v0, v1, v5, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot4_acc8_vecMul: ; GFX8-LABEL: udot4_acc8_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_mov_b32_e32 v3, s0 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mul_lo_u16_sdwa v7, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX8-NEXT: s_lshr_b32 s2, s0, 24 ; GFX8-NEXT: v_mul_lo_u16_e32 v9, v5, v6
; GFX8-NEXT: s_lshr_b32 s4, s1, 24 ; GFX8-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: s_lshr_b32 s3, s0, 16 ; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX8-NEXT: v_mov_b32_e32 v4, s0 ; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v7
; GFX8-NEXT: v_mov_b32_e32 v5, s1 ; GFX8-NEXT: v_or_b32_e32 v8, v8, v9
; GFX8-NEXT: s_mul_i32 s0, s0, s1 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v8
; GFX8-NEXT: s_lshr_b32 s5, s1, 16
; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v4, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: v_and_b32_e32 v3, s0, v3
; GFX8-NEXT: v_mov_b32_e32 v6, s4
; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
; GFX8-NEXT: v_mul_u32_u24_e32 v5, s3, v5
; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX8-NEXT: v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: v_or_b32_e32 v4, v3, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5 ; GFX8-NEXT: v_add_u16_e32 v2, v2, v8
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v7
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2
; GFX8-NEXT: v_add_u16_e32 v2, v2, v7
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-NODL-LABEL: udot4_acc8_vecMul: ; GFX9-NODL-LABEL: udot4_acc8_vecMul:
; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: global_load_ubyte v4, v0, s[2:3]
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 16 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_lshr_b32 s6, s1, 16 ; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[2:3]
; GFX9-NODL-NEXT: s_lshr_b32 s7, s1, 24 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v1, s0, v1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v2, s0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s6 ; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_lshr_b32 s5, s0, 24 ; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v8, v4, v5
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7 ; GFX9-NODL-NEXT: v_or_b32_sdwa v6, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v2, s5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v7, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v3, s4, v3 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v8, 16, v6
; GFX9-NODL-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX9-NODL-NEXT: v_or_b32_e32 v7, v7, v8
; GFX9-NODL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v7
; GFX9-NODL-NEXT: v_or_b32_e32 v2, v1, v2
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 8, v2
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v4 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v7
; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX9-NODL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1
; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NODL-NEXT: s_endpgm ; GFX9-NODL-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot4_acc8_vecMul: ; GFX9-DL-LABEL: udot4_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: global_load_ubyte v4, v0, s[2:3] ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 16 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_lshr_b32 s6, s1, 16 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 24 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v1, s0, v1 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v8, v4, v5
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, s0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX9-DL-NEXT: v_or_b32_sdwa v6, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s6 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v8, 16, v6
; GFX9-DL-NEXT: s_lshr_b32 s5, s0, 24 ; GFX9-DL-NEXT: v_or_b32_e32 v7, v7, v8
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v7
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, s5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v3, s4, v3
; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-DL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_e32 v2, v1, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v4 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v7
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1
; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot4_acc8_vecMul: ; GFX10-DL-LABEL: udot4_acc8_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s6 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s7 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 24 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 24 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s0, s1 ; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[0:1]
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v2, v2, v3 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v3, s6, s7 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v1
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 16 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 16 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v2
; GFX10-DL-NEXT: v_lshlrev_b16_e64 v2, 8, v2 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v1
; GFX10-DL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX10-DL-NEXT: v_lshlrev_b16_e64 v3, 8, v4 ; GFX10-DL-NEXT: v_lshrrev_b16_e64 v8, 8, v2
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s0, s1 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, v4, v5
; GFX10-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2 ; GFX10-DL-NEXT: v_lshrrev_b16_e64 v5, 8, v1
; GFX10-DL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, v6, v7
; GFX10-DL-NEXT: v_or_b32_e32 v3, v2, v3
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v3
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1 ; GFX10-DL-NEXT: v_mad_u16 v3, v1, v2, v3
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v4 ; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v4
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, v5, v8
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX10-DL-NEXT: v_or_b32_sdwa v4, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: v_lshlrev_b16_e64 v5, 8, v5
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v8, 16, v4
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v4
; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v5
; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v3, v5
; GFX10-DL-NEXT: v_mad_u16 v1, v6, v7, v1
; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v2
; GFX10-DL-NEXT: global_store_byte v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2, <4 x i8> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2 %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src1, i32 %idx
%vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1
%gep2 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %src2, i32 %idx
%vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %gep2
%mul = mul <4 x i8> %vec1, %vec2 %mul = mul <4 x i8> %vec1, %vec2
%mul0 = extractelement <4 x i8> %mul, i64 0 %mul0 = extractelement <4 x i8> %mul, i64 0
%mul1 = extractelement <4 x i8> %mul, i64 1 %mul1 = extractelement <4 x i8> %mul, i64 1
%mul2 = extractelement <4 x i8> %mul, i64 2 %mul2 = extractelement <4 x i8> %mul, i64 2
%mul3 = extractelement <4 x i8> %mul, i64 3 %mul3 = extractelement <4 x i8> %mul, i64 3
%acc = load i8, i8 addrspace(1)* %dst, align 4 %acc = load i8, i8 addrspace(1)* %dst, align 4
%add1 = add i8 %mul0, %acc %add1 = add i8 %mul0, %acc
%add2 = add i8 %add1, %mul1 %add2 = add i8 %add1, %mul1
%add3 = add i8 %add2, %mul2 %add3 = add i8 %add2, %mul2
%add4 = add i8 %add3, %mul3 %add4 = add i8 %add3, %mul3
store i8 %add4, i8 addrspace(1)* %dst, align 4 store i8 %add4, i8 addrspace(1)* %dst, align 4
ret void ret void
} }
declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/idot8s.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-XNACK %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-XNACK %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-XNACK %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-XNACK %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-NOXNACK %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-NOXNACK %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1031 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-NOXNACK %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1031 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL-NOXNACK %s
define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc32: ; GFX7-LABEL: idot8_acc32:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000
; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000
; GFX7-NEXT: s_bfe_i32 s9, s5, 0x40004
; GFX7-NEXT: v_mov_b32_e32 v0, s7
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x40004
; GFX7-NEXT: v_mov_b32_e32 v1, s9
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x40008
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40008
; GFX7-NEXT: v_mov_b32_e32 v1, s11
; GFX7-NEXT: s_bfe_i32 s13, s5, 0x4000c
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
; GFX7-NEXT: s_bfe_i32 s12, s4, 0x4000c
; GFX7-NEXT: v_mov_b32_e32 v1, s13
; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40010
; GFX7-NEXT: v_mad_i32_i24 v0, s12, v1, v0
; GFX7-NEXT: s_bfe_i32 s14, s4, 0x40010
; GFX7-NEXT: v_mov_b32_e32 v1, s15
; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40014
; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40018
; GFX7-NEXT: v_mad_i32_i24 v0, s14, v1, v0
; GFX7-NEXT: s_bfe_i32 s16, s4, 0x40014
; GFX7-NEXT: v_mov_b32_e32 v1, s17
; GFX7-NEXT: s_bfe_i32 s18, s4, 0x40018
; GFX7-NEXT: v_mad_i32_i24 v0, s16, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s19
; GFX7-NEXT: s_ashr_i32 s5, s5, 28
; GFX7-NEXT: v_mad_i32_i24 v0, s18, v1, v0
; GFX7-NEXT: s_ashr_i32 s4, s4, 28
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 4
; GFX7-NEXT: v_bfe_i32 v3, v2, 4, 4
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v9, v0, 0, 4
; GFX7-NEXT: v_bfe_i32 v10, v0, 4, 4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_i32_i24 v1, v1, v9, s4
; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 4
; GFX7-NEXT: v_bfe_i32 v11, v0, 8, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v3, v10, v1
; GFX7-NEXT: v_bfe_i32 v5, v2, 12, 4
; GFX7-NEXT: v_bfe_i32 v12, v0, 12, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v4, v11, v1
; GFX7-NEXT: v_bfe_i32 v6, v2, 16, 4
; GFX7-NEXT: v_bfe_i32 v13, v0, 16, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v5, v12, v1
; GFX7-NEXT: v_bfe_i32 v7, v2, 20, 4
; GFX7-NEXT: v_bfe_i32 v14, v0, 20, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v6, v13, v1
; GFX7-NEXT: v_bfe_i32 v8, v2, 24, 4
; GFX7-NEXT: v_bfe_i32 v15, v0, 24, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v7, v14, v1
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 28, v2
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
; GFX7-NEXT: v_mad_i32_i24 v1, v8, v15, v1
; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot8_acc32: ; GFX8-LABEL: idot8_acc32:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s23, 0xe80000 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_add_u32 s20, s20, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_bfe_i32 s4, s2, 0x40000
; GFX8-NEXT: s_bfe_i32 s5, s3, 0x40000
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s18
; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_i32 s9, s3, 0x40008 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40008 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s9 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_bfe_i32 s11, s3, 0x4000c ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_bfe_i32 s10, s2, 0x4000c ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: v_mov_b32_e32 v1, s11 ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_bfe_i32 s13, s3, 0x40010 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_mad_i32_i24 v0, s10, v1, v0 ; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 4
; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010 ; GFX8-NEXT: v_bfe_i32 v4, v3, 4, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s13 ; GFX8-NEXT: v_bfe_i32 v6, v3, 8, 4
; GFX8-NEXT: s_bfe_i32 s15, s3, 0x40014 ; GFX8-NEXT: v_bfe_i32 v8, v3, 12, 4
; GFX8-NEXT: s_bfe_i32 s17, s3, 0x40018 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v0, s12, v1, v0 ; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 4
; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014 ; GFX8-NEXT: v_bfe_i32 v5, v0, 4, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s15 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018 ; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s2
; GFX8-NEXT: v_mad_i32_i24 v0, s14, v1, v0 ; GFX8-NEXT: v_bfe_i32 v7, v0, 8, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s17 ; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v1
; GFX8-NEXT: s_ashr_i32 s3, s3, 28 ; GFX8-NEXT: v_bfe_i32 v9, v0, 12, 4
; GFX8-NEXT: v_mad_i32_i24 v0, s16, v1, v0 ; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
; GFX8-NEXT: s_ashr_i32 s2, s2, 28 ; GFX8-NEXT: v_bfe_i32 v10, v3, 16, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s3 ; GFX8-NEXT: v_bfe_i32 v11, v0, 16, 4
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0 ; GFX8-NEXT: v_mad_i32_i24 v1, v8, v9, v1
; GFX8-NEXT: v_bfe_i32 v12, v3, 20, 4
; GFX8-NEXT: v_bfe_i32 v13, v0, 20, 4
; GFX8-NEXT: v_mad_i32_i24 v1, v10, v11, v1
; GFX8-NEXT: v_bfe_i32 v14, v3, 24, 4
; GFX8-NEXT: v_bfe_i32 v15, v0, 24, 4
; GFX8-NEXT: v_mad_i32_i24 v1, v12, v13, v1
; GFX8-NEXT: v_ashrrev_i32_e32 v3, 28, v3
; GFX8-NEXT: v_ashrrev_i32_e32 v0, 28, v0
; GFX8-NEXT: v_mad_i32_i24 v1, v14, v15, v1
; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: idot8_acc32: ; GFX9-LABEL: idot8_acc32:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0 ; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_bfe_i32 v3, v1, 0, 4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_bfe_i32 v4, v2, 0, 4
; GFX9-NEXT: v_bfe_i32 v5, v1, 4, 4
; GFX9-NEXT: v_bfe_i32 v6, v2, 4, 4
; GFX9-NEXT: v_mul_i32_i24_e32 v3, v3, v4
; GFX9-NEXT: v_bfe_i32 v7, v1, 8, 4
; GFX9-NEXT: v_bfe_i32 v8, v2, 8, 4
; GFX9-NEXT: v_mul_i32_i24_e32 v4, v5, v6
; GFX9-NEXT: v_bfe_i32 v9, v1, 12, 4
; GFX9-NEXT: v_bfe_i32 v10, v2, 12, 4
; GFX9-NEXT: v_bfe_i32 v11, v1, 16, 4
; GFX9-NEXT: v_bfe_i32 v12, v2, 16, 4
; GFX9-NEXT: v_bfe_i32 v13, v1, 20, 4
; GFX9-NEXT: v_bfe_i32 v15, v1, 24, 4
; GFX9-NEXT: v_bfe_i32 v14, v2, 20, 4
; GFX9-NEXT: v_bfe_i32 v16, v2, 24, 4
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 28, v1
; GFX9-NEXT: v_ashrrev_i32_e32 v2, 28, v2
; GFX9-NEXT: v_mul_i32_i24_e32 v5, v7, v8
; GFX9-NEXT: v_mul_i32_i24_e32 v6, v9, v10
; GFX9-NEXT: v_mul_i32_i24_e32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s4, s0, 0x40000 ; GFX9-NEXT: v_add3_u32 v2, v3, s0, v4
; GFX9-NEXT: s_bfe_i32 s5, s1, 0x40000 ; GFX9-NEXT: v_mul_i32_i24_e32 v7, v11, v12
; GFX9-NEXT: s_bfe_i32 s7, s1, 0x40004 ; GFX9-NEXT: v_mul_i32_i24_e32 v8, v13, v14
; GFX9-NEXT: v_mov_b32_e32 v1, s5 ; GFX9-NEXT: v_add3_u32 v2, v2, v5, v6
; GFX9-NEXT: v_mov_b32_e32 v2, s18 ; GFX9-NEXT: v_mul_i32_i24_e32 v9, v15, v16
; GFX9-NEXT: v_mad_i32_i24 v1, s4, v1, v2 ; GFX9-NEXT: v_add3_u32 v2, v2, v7, v8
; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40004 ; GFX9-NEXT: v_add3_u32 v1, v2, v9, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40008
; GFX9-NEXT: v_mad_i32_i24 v1, s6, v2, v1
; GFX9-NEXT: s_bfe_i32 s8, s0, 0x40008
; GFX9-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NEXT: s_bfe_i32 s11, s1, 0x4000c
; GFX9-NEXT: v_mad_i32_i24 v1, s8, v2, v1
; GFX9-NEXT: s_bfe_i32 s10, s0, 0x4000c
; GFX9-NEXT: v_mov_b32_e32 v2, s11
; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010
; GFX9-NEXT: v_mad_i32_i24 v1, s10, v2, v1
; GFX9-NEXT: s_bfe_i32 s12, s0, 0x40010
; GFX9-NEXT: v_mov_b32_e32 v2, s13
; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014
; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018
; GFX9-NEXT: v_mad_i32_i24 v1, s12, v2, v1
; GFX9-NEXT: s_bfe_i32 s14, s0, 0x40014
; GFX9-NEXT: v_mov_b32_e32 v2, s15
; GFX9-NEXT: s_bfe_i32 s16, s0, 0x40018
; GFX9-NEXT: v_mad_i32_i24 v1, s14, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s17
; GFX9-NEXT: s_ashr_i32 s1, s1, 28
; GFX9-NEXT: v_mad_i32_i24 v1, s16, v2, v1
; GFX9-NEXT: s_ashr_i32 s0, s0, 28
; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: v_mad_i32_i24 v1, s0, v2, v1
; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot8_acc32: ; GFX9-DL-LABEL: idot8_acc32:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s14, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s12, s12, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: v_dot8_i32_i4 v0, v2, v3, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot8_i32_i4 v1, s0, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-XNACK-LABEL: idot8_acc32: ; GFX10-DL-XNACK-LABEL: idot8_acc32:
; GFX10-DL-XNACK: ; %bb.0: ; %entry ; GFX10-DL-XNACK: ; %bb.0: ; %entry
; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-XNACK-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0 ; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-XNACK-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v1, v1, v2, s2
; GFX10-DL-XNACK-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm ; GFX10-DL-XNACK-NEXT: s_endpgm
; ;
; GFX10-DL-NOXNACK-LABEL: idot8_acc32: ; GFX10-DL-NOXNACK-LABEL: idot8_acc32:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry ; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: s_load_dword s6, s[4:5], 0x0 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, s6 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0 ; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, v1, v0, s2
; GFX10-DL-NOXNACK-NEXT: global_store_dword v1, v0, s[4:5] ; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm ; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc32: ; GFX10-DL-LABEL: idot8_acc32:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
Show All 9 Lines
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s6 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s6
; GFX10-DL-NEXT: v_dot8_i32_i4 v0, s0, s1, v0 ; GFX10-DL-NEXT: v_dot8_i32_i4 v0, s0, s1, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[4:5] ; GFX10-DL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i32 %cv1e0 = sext i4 %v1e0 to i32
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = sext i4 %v2e0 to i32 %cv2e0 = sext i4 %v2e0 to i32
%mul0 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Linesentry:
ret void ret void
} }
; TODO: Once the unnecessary zero extentions of the elements are removed; ; TODO: Once the unnecessary zero extentions of the elements are removed;
; pattern recognizer will kick in. ; pattern recognizer will kick in.
define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc16: ; GFX7-LABEL: idot8_acc16:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_mov_b32 s8, 0xffff ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x40004 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_and_b32 s7, s7, s8 ; GFX7-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40004 ; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_bfe_i32 s12, s5, 0x40008 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_and_b32 s10, s10, s8 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_and_b32 s6, s6, s8 ; GFX7-NEXT: v_bfe_i32 v1, v3, 0, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s7 ; GFX7-NEXT: v_bfe_i32 v4, v3, 4, 4
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008 ; GFX7-NEXT: v_bfe_i32 v5, v3, 8, 4
; GFX7-NEXT: s_bfe_i32 s14, s5, 0x4000c ; GFX7-NEXT: v_bfe_i32 v6, v3, 12, 4
; GFX7-NEXT: s_and_b32 s12, s12, s8 ; GFX7-NEXT: v_bfe_i32 v7, v3, 16, 4
; GFX7-NEXT: s_and_b32 s9, s9, s8 ; GFX7-NEXT: v_bfe_i32 v8, v3, 20, 4
; GFX7-NEXT: v_mov_b32_e32 v2, s10 ; GFX7-NEXT: v_bfe_i32 v9, v3, 24, 4
; GFX7-NEXT: s_bfe_i32 s13, s4, 0x4000c ; GFX7-NEXT: v_ashrrev_i32_e32 v3, 28, v3
; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40010
; GFX7-NEXT: s_and_b32 s14, s14, s8
; GFX7-NEXT: s_and_b32 s11, s11, s8
; GFX7-NEXT: v_mov_b32_e32 v3, s12
; GFX7-NEXT: s_bfe_i32 s15, s4, 0x40010
; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40014
; GFX7-NEXT: s_and_b32 s16, s16, s8
; GFX7-NEXT: s_and_b32 s13, s13, s8
; GFX7-NEXT: v_mov_b32_e32 v4, s14
; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40018
; GFX7-NEXT: s_bfe_i32 s17, s4, 0x40014
; GFX7-NEXT: s_and_b32 s18, s18, s8
; GFX7-NEXT: s_and_b32 s15, s15, s8
; GFX7-NEXT: v_mov_b32_e32 v5, s16
; GFX7-NEXT: s_bfe_i32 s19, s4, 0x40018
; GFX7-NEXT: s_ashr_i32 s5, s5, 28
; GFX7-NEXT: s_and_b32 s20, s20, s8
; GFX7-NEXT: s_and_b32 s17, s17, s8
; GFX7-NEXT: v_mov_b32_e32 v6, s18
; GFX7-NEXT: s_ashr_i32 s4, s4, 28
; GFX7-NEXT: s_and_b32 s19, s19, s8
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: v_mov_b32_e32 v7, s20
; GFX7-NEXT: s_and_b32 s4, s4, s8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: v_bfe_i32 v10, v0, 0, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_bfe_i32 v11, v0, 4, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_i32 v12, v0, 8, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s13, v4, v0 ; GFX7-NEXT: v_bfe_i32 v13, v0, 12, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s15, v5, v0 ; GFX7-NEXT: v_bfe_i32 v14, v0, 16, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s17, v6, v0 ; GFX7-NEXT: v_bfe_i32 v15, v0, 20, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s19, v7, v0 ; GFX7-NEXT: v_bfe_i32 v16, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_and_b32_e32 v9, v2, v9
; GFX7-NEXT: v_and_b32_e32 v3, v2, v3
; GFX7-NEXT: v_and_b32_e32 v15, v2, v15
; GFX7-NEXT: v_and_b32_e32 v16, v2, v16
; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
; GFX7-NEXT: buffer_load_ushort v2, off, s[0:3], 0
; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
; GFX7-NEXT: v_and_b32_e32 v10, s4, v10
; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
; GFX7-NEXT: v_and_b32_e32 v11, s4, v11
; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
; GFX7-NEXT: v_and_b32_e32 v12, s4, v12
; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: v_and_b32_e32 v13, s4, v13
; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
; GFX7-NEXT: v_and_b32_e32 v14, s4, v14
; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v1, v1, v10, v2
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v11, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v5, v12, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v6, v13, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v7, v14, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v8, v15, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v9, v16, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot8_acc16: ; GFX8-LABEL: idot8_acc16:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: v_mov_b32_e32 v5, 12
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s19, 0xe80000
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_addc_u32 s17, s17, 0 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_bfe_i32 s4, s0, 0x40000 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_bfe_i32 s5, s1, 0x40000 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 4, v3
; GFX8-NEXT: s_bfe_i32 s7, s1, 0x40004 ; GFX8-NEXT: v_lshlrev_b16_e32 v16, 12, v3
; GFX8-NEXT: s_bfe_i32 s9, s1, 0x40008 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 28, v3
; GFX8-NEXT: v_mov_b32_e32 v6, s5 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 20, v3
; GFX8-NEXT: s_lshr_b32 s2, s0, 12 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_lshr_b32 s3, s1, 12 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 4, v2
; GFX8-NEXT: s_bfe_i32 s6, s0, 0x40004 ; GFX8-NEXT: v_lshlrev_b16_e32 v17, 12, v2
; GFX8-NEXT: s_bfe_i32 s8, s0, 0x40008 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
; GFX8-NEXT: v_mov_b32_e32 v7, s7 ; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s2 ; GFX8-NEXT: v_lshrrev_b32_e32 v12, 20, v2
; GFX8-NEXT: v_lshlrev_b16_e64 v5, 12, s3 ; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
; GFX8-NEXT: v_mul_i32_i24_e32 v3, s8, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v2
; GFX8-NEXT: s_bfe_i32 s11, s1, 0x40010 ; GFX8-NEXT: v_lshlrev_b16_e32 v10, 12, v10
; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX8-NEXT: v_lshlrev_b16_e32 v15, 12, v15
; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5 ; GFX8-NEXT: v_lshlrev_b16_sdwa v18, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_bfe_i32 s13, s1, 0x40014 ; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: s_bfe_i32 s10, s0, 0x40010 ; GFX8-NEXT: v_lshlrev_b16_sdwa v19, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: s_bfe_i32 s15, s1, 0x40018 ; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v16
; GFX8-NEXT: s_bfe_i32 s12, s0, 0x40014 ; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v17
; GFX8-NEXT: v_mov_b32_e32 v9, s13 ; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX8-NEXT: s_bfe_i32 s14, s0, 0x40018 ; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX8-NEXT: s_ashr_i32 s1, s1, 28 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX8-NEXT: v_mov_b32_e32 v10, s15 ; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v15
; GFX8-NEXT: s_ashr_i32 s0, s0, 28
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v6, v2 ; GFX8-NEXT: v_mad_u16 v4, v5, v16, v4
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v7, v2 ; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX8-NEXT: v_mad_u32_u24 v2, v4, v5, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX8-NEXT: v_mad_i32_i24 v2, s10, v8, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX8-NEXT: v_mad_i32_i24 v2, s12, v9, v2 ; GFX8-NEXT: v_mad_u16 v4, v10, v15, v4
; GFX8-NEXT: v_mad_i32_i24 v2, s14, v10, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u16 v4, v9, v14, v4
; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v19
; GFX8-NEXT: v_mad_u16 v4, v8, v13, v4
; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX8-NEXT: v_mad_u16 v4, v17, v18, v4
; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX8-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX8-NEXT: v_mad_u16 v4, v7, v12, v4
; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
; GFX8-NEXT: v_mad_u16 v2, v6, v11, v2
; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: idot8_acc16: ; GFX9-LABEL: idot8_acc16:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v4, 12
; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40000 ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: s_bfe_i32 s7, s1, 0x40000 ; GFX9-NEXT: v_lshrrev_b32_e32 v9, 4, v1
; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40004 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_i32 s11, s1, 0x40008 ; GFX9-NEXT: v_lshrrev_b32_e32 v14, 4, v2
; GFX9-NEXT: v_mov_b32_e32 v5, s7 ; GFX9-NEXT: v_lshlrev_b16_e32 v15, 12, v1
; GFX9-NEXT: s_lshr_b32 s4, s0, 12 ; GFX9-NEXT: v_lshlrev_b16_e32 v16, 12, v2
; GFX9-NEXT: s_lshr_b32 s5, s1, 12 ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v1
; GFX9-NEXT: s_bfe_i32 s8, s0, 0x40004 ; GFX9-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX9-NEXT: s_bfe_i32 s10, s0, 0x40008 ; GFX9-NEXT: v_lshrrev_b32_e32 v7, 12, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
; GFX9-NEXT: v_mov_b32_e32 v6, s9 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s4 ; GFX9-NEXT: v_lshrrev_b32_e32 v11, 20, v2
; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s5 ; GFX9-NEXT: v_lshrrev_b32_e32 v12, 12, v2
; GFX9-NEXT: v_mul_i32_i24_e32 v2, s10, v2 ; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v2
; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010 ; GFX9-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3 ; GFX9-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX9-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014 ; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: s_bfe_i32 s12, s0, 0x40010 ; GFX9-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018 ; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v15
; GFX9-NEXT: s_bfe_i32 s14, s0, 0x40014 ; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-NEXT: v_mov_b32_e32 v8, s15 ; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX9-NEXT: s_bfe_i32 s16, s0, 0x40018 ; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-NEXT: s_ashr_i32 s1, s1, 28 ; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX9-NEXT: v_mov_b32_e32 v9, s17 ; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-NEXT: s_ashr_i32 s0, s0, 28
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_i32_i24 v1, s6, v5, v1 ; GFX9-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
; GFX9-NEXT: v_mad_i32_i24 v1, s8, v6, v1 ; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-NEXT: v_mad_u32_u24 v1, v3, v4, v1 ; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-NEXT: v_mad_i32_i24 v1, s12, v7, v1 ; GFX9-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX9-NEXT: v_mad_i32_i24 v1, s14, v8, v1 ; GFX9-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
; GFX9-NEXT: v_mad_i32_i24 v1, s16, v9, v1 ; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX9-NEXT: v_mad_i32_i24 v1, s0, v2, v1 ; GFX9-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v17
; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX9-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v5
; GFX9-NEXT: v_lshlrev_b16_e32 v10, 12, v10
; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot8_acc16: ; GFX9-DL-LABEL: idot8_acc16:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, 12
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_bfe_i32 s6, s0, 0x40000 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x40000 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x40004 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_bfe_i32 s11, s1, 0x40008 ; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s7 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 12 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 4, v1
; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 12 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_bfe_i32 s8, s0, 0x40004 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 4, v2
; GFX9-DL-NEXT: s_bfe_i32 s10, s0, 0x40008 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v15, 12, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v16, 12, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s9 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v1
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s5 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 12, v1
; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, s10, v2 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
; GFX9-DL-NEXT: s_bfe_i32 s13, s1, 0x40010 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 20, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 12, v2
; GFX9-DL-NEXT: s_bfe_i32 s15, s1, 0x40014 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v13, 8, v2
; GFX9-DL-NEXT: s_bfe_i32 s12, s0, 0x40010 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX9-DL-NEXT: s_bfe_i32 s17, s1, 0x40018 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: s_bfe_i32 s14, s0, 0x40014 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s15 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: s_bfe_i32 s16, s0, 0x40018 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 28 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v15
; GFX9-DL-NEXT: v_mov_b32_e32 v9, s17 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 28 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s6, v5, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s8, v6, v1 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_mad_u32_u24 v1, v3, v4, v1 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s12, v7, v1 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s14, v8, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s16, v9, v1 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v17
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v5
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v10, 12, v10
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-XNACK-LABEL: idot8_acc16: ; GFX10-DL-XNACK-LABEL: idot8_acc16:
; GFX10-DL-XNACK: ; %bb.0: ; %entry ; GFX10-DL-XNACK: ; %bb.0: ; %entry
; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s0, s6, 12 ; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s7, 12 ; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s2, s6, 0x40000 ; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s3, s7, 0x40000 ; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s0 ; GFX10-DL-XNACK-NEXT: global_load_ushort v3, v0, s[0:1]
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s1 ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s8, s6, 0x40004 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s9, s6, 0x40008 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s10, s7, 0x40008 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s7, 0x40004 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40010 ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, v2
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v18, v17, v3
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s8, s0, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
; GFX10-DL-XNACK-NEXT: s_mov_b32 s0, 0xffff ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, s0, v3 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v10, v16, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40010 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40014 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v9
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40014 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v3, v2, v1
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40018 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v7
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40018 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
; GFX10-DL-XNACK-NEXT: s_ashr_i32 s0, s6, 28 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v8, v15, v1
; GFX10-DL-XNACK-NEXT: s_ashr_i32 s1, s7, 28 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v5
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v6, v7, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v5
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v4, v7, v1
; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm ; GFX10-DL-XNACK-NEXT: s_endpgm
; ;
; GFX10-DL-NOXNACK-LABEL: idot8_acc16: ; GFX10-DL-NOXNACK-LABEL: idot8_acc16:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry ; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s15, 0x31c16000 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s12, s12, s3 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NOXNACK-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s2, s0, 12 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s1, 12 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s6, s0, 0x40000 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s7, s1, 0x40000 ; GFX10-DL-NOXNACK-NEXT: global_load_ushort v3, v2, s[0:1]
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s2 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s3 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s8, s0, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s9, s0, 0x40008 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s10, s1, 0x40008 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s1, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40010 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s6, s7, v1 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v18, v17, v3
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s8, s2, v1 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s2, 0xffff ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, s2, v3 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v10, v16, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40010 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v8
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40014 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40014 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v3, v0, v1
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40018 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v7
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40018 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s0, s0, 28 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s1, s1, 28 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v15, v9, v0
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
; GFX10-DL-NOXNACK-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v4
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v7, v5, v0
; GFX10-DL-NOXNACK-NEXT: global_store_short v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm ; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc16: ; GFX10-DL-LABEL: idot8_acc16:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-NEXT: s_mov_b32 s14, -1
Show All 39 Lines
; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 28 ; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 28
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NEXT: v_mad_i32_i24 v1, s2, s3, v1
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i16 %cv1e0 = sext i4 %v1e0 to i16
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = sext i4 %v2e0 to i16 %cv2e0 = sext i4 %v2e0 to i16
%mul0 = mul nuw nsw i16 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Linesentry:
store i16 %add8, i16 addrspace(1)* %dst, align 4 store i16 %add8, i16 addrspace(1)* %dst, align 4
ret void ret void
} }
; TODO: Support this pattern. ; TODO: Support this pattern.
define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc8: ; GFX7-LABEL: idot8_acc8:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x40004 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_and_b32 s7, s7, s8 ; GFX7-NEXT: v_mov_b32_e32 v2, 0xff
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40004 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: s_bfe_i32 s12, s5, 0x40008 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_and_b32 s10, s10, s8 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_and_b32 s6, s6, s8 ; GFX7-NEXT: v_bfe_i32 v1, v3, 0, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s7 ; GFX7-NEXT: v_bfe_i32 v4, v3, 4, 4
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008 ; GFX7-NEXT: v_bfe_i32 v5, v3, 8, 4
; GFX7-NEXT: s_bfe_i32 s14, s5, 0x4000c ; GFX7-NEXT: v_bfe_i32 v6, v3, 12, 4
; GFX7-NEXT: s_and_b32 s12, s12, s8 ; GFX7-NEXT: v_bfe_i32 v7, v3, 16, 4
; GFX7-NEXT: s_and_b32 s9, s9, s8 ; GFX7-NEXT: v_bfe_i32 v8, v3, 20, 4
; GFX7-NEXT: v_mov_b32_e32 v2, s10 ; GFX7-NEXT: v_bfe_i32 v9, v3, 24, 4
; GFX7-NEXT: s_bfe_i32 s13, s4, 0x4000c ; GFX7-NEXT: v_ashrrev_i32_e32 v3, 28, v3
; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40010
; GFX7-NEXT: s_and_b32 s14, s14, s8
; GFX7-NEXT: s_and_b32 s11, s11, s8
; GFX7-NEXT: v_mov_b32_e32 v3, s12
; GFX7-NEXT: s_bfe_i32 s15, s4, 0x40010
; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40014
; GFX7-NEXT: s_and_b32 s16, s16, s8
; GFX7-NEXT: s_and_b32 s13, s13, s8
; GFX7-NEXT: v_mov_b32_e32 v4, s14
; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40018
; GFX7-NEXT: s_bfe_i32 s17, s4, 0x40014
; GFX7-NEXT: s_and_b32 s18, s18, s8
; GFX7-NEXT: s_and_b32 s15, s15, s8
; GFX7-NEXT: v_mov_b32_e32 v5, s16
; GFX7-NEXT: s_bfe_i32 s19, s4, 0x40018
; GFX7-NEXT: s_ashr_i32 s5, s5, 28
; GFX7-NEXT: s_and_b32 s20, s20, s8
; GFX7-NEXT: s_and_b32 s17, s17, s8
; GFX7-NEXT: v_mov_b32_e32 v6, s18
; GFX7-NEXT: s_ashr_i32 s4, s4, 28
; GFX7-NEXT: s_and_b32 s19, s19, s8
; GFX7-NEXT: s_and_b32 s5, s5, s8
; GFX7-NEXT: v_mov_b32_e32 v7, s20
; GFX7-NEXT: s_and_b32 s4, s4, s8
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: v_bfe_i32 v10, v0, 0, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_bfe_i32 v11, v0, 4, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_bfe_i32 v12, v0, 8, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s13, v4, v0 ; GFX7-NEXT: v_bfe_i32 v13, v0, 12, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s15, v5, v0 ; GFX7-NEXT: v_bfe_i32 v14, v0, 16, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s17, v6, v0 ; GFX7-NEXT: v_bfe_i32 v15, v0, 20, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s19, v7, v0 ; GFX7-NEXT: v_bfe_i32 v16, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_and_b32_e32 v9, v2, v9
; GFX7-NEXT: v_and_b32_e32 v3, v2, v3
; GFX7-NEXT: v_and_b32_e32 v15, v2, v15
; GFX7-NEXT: v_and_b32_e32 v16, v2, v16
; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
; GFX7-NEXT: buffer_load_ubyte v2, off, s[0:3], 0
; GFX7-NEXT: v_and_b32_e32 v1, s4, v1
; GFX7-NEXT: v_and_b32_e32 v10, s4, v10
; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
; GFX7-NEXT: v_and_b32_e32 v11, s4, v11
; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
; GFX7-NEXT: v_and_b32_e32 v12, s4, v12
; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: v_and_b32_e32 v13, s4, v13
; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
; GFX7-NEXT: v_and_b32_e32 v14, s4, v14
; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v1, v1, v10, v2
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v11, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v5, v12, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v6, v13, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v7, v14, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v8, v15, v1
; GFX7-NEXT: v_mad_u32_u24 v1, v9, v16, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot8_acc8: ; GFX8-LABEL: idot8_acc8:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: v_mov_b32_e32 v5, 12
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s23, 0xe80000
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_add_u32 s20, s20, s3 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_lshr_b32 s3, s1, 12 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 4, v3
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40000 ; GFX8-NEXT: v_lshlrev_b16_e32 v16, 12, v3
; GFX8-NEXT: s_lshr_b32 s4, s2, 12 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 28, v3
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40004 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 20, v3
; GFX8-NEXT: s_bfe_i32 s10, s2, 0x40008 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_bfe_i32 s5, s1, 0x40000 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 4, v2
; GFX8-NEXT: v_mov_b32_e32 v6, s6 ; GFX8-NEXT: v_lshlrev_b16_e32 v17, 12, v2
; GFX8-NEXT: v_lshlrev_b16_e64 v4, 12, s3 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
; GFX8-NEXT: v_lshlrev_b16_e64 v5, 12, s4 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
; GFX8-NEXT: s_bfe_i32 s7, s1, 0x40004 ; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX8-NEXT: s_bfe_i32 s9, s1, 0x40008 ; GFX8-NEXT: v_lshrrev_b32_e32 v12, 20, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s10 ; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
; GFX8-NEXT: v_mov_b32_e32 v7, s8 ; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v2
; GFX8-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX8-NEXT: v_lshlrev_b16_e32 v10, 12, v10
; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5 ; GFX8-NEXT: v_lshlrev_b16_e32 v15, 12, v15
; GFX8-NEXT: v_mul_i32_i24_e32 v3, s9, v3 ; GFX8-NEXT: v_lshlrev_b16_sdwa v18, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010 ; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: v_and_b32_e32 v4, s0, v4 ; GFX8-NEXT: v_lshlrev_b16_sdwa v19, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_and_b32_e32 v5, s0, v5 ; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014 ; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v16
; GFX8-NEXT: s_bfe_i32 s11, s1, 0x40010 ; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v17
; GFX8-NEXT: v_mov_b32_e32 v8, s12 ; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018 ; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX8-NEXT: s_bfe_i32 s13, s1, 0x40014 ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX8-NEXT: v_mov_b32_e32 v9, s14 ; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v15
; GFX8-NEXT: s_bfe_i32 s15, s1, 0x40018
; GFX8-NEXT: s_ashr_i32 s2, s2, 28
; GFX8-NEXT: v_mov_b32_e32 v10, s16
; GFX8-NEXT: s_ashr_i32 s1, s1, 28
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v2, s5, v6, v2 ; GFX8-NEXT: v_mad_u16 v4, v5, v16, v4
; GFX8-NEXT: v_mad_i32_i24 v2, s7, v7, v2 ; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX8-NEXT: v_mad_u32_u24 v2, v4, v5, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX8-NEXT: v_mad_i32_i24 v2, s11, v8, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX8-NEXT: v_mad_i32_i24 v2, s13, v9, v2 ; GFX8-NEXT: v_mad_u16 v4, v10, v15, v4
; GFX8-NEXT: v_mad_i32_i24 v2, s15, v10, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX8-NEXT: v_mad_i32_i24 v2, s1, v3, v2 ; GFX8-NEXT: v_mad_u16 v4, v9, v14, v4
; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v19
; GFX8-NEXT: v_mad_u16 v4, v8, v13, v4
; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX8-NEXT: v_mad_u16 v4, v17, v18, v4
; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX8-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX8-NEXT: v_mad_u16 v4, v7, v12, v4
; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
; GFX8-NEXT: v_mad_u16 v2, v6, v11, v2
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: idot8_acc8: ; GFX9-LABEL: idot8_acc8:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v4, 12
; GFX9-NEXT: s_movk_i32 s0, 0xff ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s8, s[6:7], 0x0 ; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ubyte v3, v0, s[2:3]
; GFX9-NEXT: s_lshr_b32 s4, s1, 12 ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: s_bfe_i32 s7, s8, 0x40000 ; GFX9-NEXT: v_lshrrev_b32_e32 v9, 4, v1
; GFX9-NEXT: s_lshr_b32 s5, s8, 12 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_i32 s10, s8, 0x40004 ; GFX9-NEXT: v_lshrrev_b32_e32 v14, 4, v2
; GFX9-NEXT: s_bfe_i32 s12, s8, 0x40008 ; GFX9-NEXT: v_lshlrev_b16_e32 v15, 12, v1
; GFX9-NEXT: s_bfe_i32 s6, s1, 0x40000 ; GFX9-NEXT: v_lshlrev_b16_e32 v16, 12, v2
; GFX9-NEXT: v_mov_b32_e32 v5, s7 ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v1
; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s4 ; GFX9-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s5 ; GFX9-NEXT: v_lshrrev_b32_e32 v7, 12, v1
; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40004 ; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
; GFX9-NEXT: s_bfe_i32 s11, s1, 0x40008 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s12 ; GFX9-NEXT: v_lshrrev_b32_e32 v11, 20, v2
; GFX9-NEXT: v_mov_b32_e32 v6, s10 ; GFX9-NEXT: v_lshrrev_b32_e32 v12, 12, v2
; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3 ; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v2
; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX9-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX9-NEXT: v_mul_i32_i24_e32 v2, s11, v2 ; GFX9-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX9-NEXT: s_bfe_i32 s14, s8, 0x40010 ; GFX9-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_e32 v3, s0, v3 ; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: v_and_b32_e32 v4, s0, v4 ; GFX9-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_bfe_i32 s16, s8, 0x40014 ; GFX9-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010 ; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v15
; GFX9-NEXT: v_mov_b32_e32 v7, s14 ; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-NEXT: s_bfe_i32 s18, s8, 0x40018 ; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014 ; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-NEXT: v_mov_b32_e32 v8, s16 ; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018 ; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-NEXT: s_ashr_i32 s8, s8, 28
; GFX9-NEXT: v_mov_b32_e32 v9, s18
; GFX9-NEXT: s_ashr_i32 s1, s1, 28
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_i32_i24 v1, s6, v5, v1 ; GFX9-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
; GFX9-NEXT: v_mad_i32_i24 v1, s9, v6, v1 ; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-NEXT: v_mad_u32_u24 v1, v3, v4, v1 ; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-NEXT: v_mad_i32_i24 v1, s13, v7, v1 ; GFX9-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX9-NEXT: v_mad_i32_i24 v1, s15, v8, v1 ; GFX9-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
; GFX9-NEXT: v_mad_i32_i24 v1, s17, v9, v1 ; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX9-NEXT: v_mad_i32_i24 v1, s1, v2, v1 ; GFX9-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v17
; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX9-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v5
; GFX9-NEXT: v_lshlrev_b16_e32 v10, 12, v10
; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot8_acc8: ; GFX9-DL-LABEL: idot8_acc8:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v4, 12
; GFX9-DL-NEXT: s_movk_i32 s0, 0xff ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_lshr_b32 s4, s1, 12 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_bfe_i32 s7, s8, 0x40000 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_lshr_b32 s5, s8, 12 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_bfe_i32 s10, s8, 0x40004 ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
; GFX9-DL-NEXT: s_bfe_i32 s12, s8, 0x40008 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_bfe_i32 s6, s1, 0x40000 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 4, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s7 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 4, v2
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s5 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v15, 12, v1
; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x40004 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v16, 12, v2
; GFX9-DL-NEXT: s_bfe_i32 s11, s1, 0x40008 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s12 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s10 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 12, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v2
; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, s11, v2 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 20, v2
; GFX9-DL-NEXT: s_bfe_i32 s14, s8, 0x40010 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 12, v2
; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v3 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v13, 8, v2
; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v4 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX9-DL-NEXT: s_bfe_i32 s16, s8, 0x40014 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX9-DL-NEXT: s_bfe_i32 s13, s1, 0x40010 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v17, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s14 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: s_bfe_i32 s18, s8, 0x40018 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v18, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: s_bfe_i32 s15, s1, 0x40014 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s16 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v15
; GFX9-DL-NEXT: s_bfe_i32 s17, s1, 0x40018 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-DL-NEXT: s_ashr_i32 s8, s8, 28 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_mov_b32_e32 v9, s18 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 28 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s6, v5, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v4, v15, v3
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s9, v6, v1 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_mad_u32_u24 v1, v3, v4, v1 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s13, v7, v1 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s15, v8, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v9, v14, v3
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s17, v9, v1 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s1, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v8, v13, v3
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v17
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v12, v3
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v16, v17, v3
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v5
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v10, 12, v10
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v6, v11, v3
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v10, v1
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-XNACK-LABEL: idot8_acc8: ; GFX10-DL-XNACK-LABEL: idot8_acc8:
; GFX10-DL-XNACK: ; %bb.0: ; %entry ; GFX10-DL-XNACK: ; %bb.0: ; %entry
; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: global_load_ubyte v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s0, s6, 12 ; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s7, 12 ; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s2, s6, 0x40000 ; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s3, s7, 0x40000 ; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s0 ; GFX10-DL-XNACK-NEXT: global_load_ubyte v3, v0, s[0:1]
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s1 ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s8, s6, 0x40004 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s9, s6, 0x40008 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s10, s7, 0x40008 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s7, 0x40004 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40010 ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, v2
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v18, v17, v3
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s8, s0, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
; GFX10-DL-XNACK-NEXT: s_movk_i32 s0, 0xff ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, s0, v3 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v10, v16, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40010 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40014 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v9
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40014 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v3, v2, v1
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s0, s6, 0x40018 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v7
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s1, s7, 0x40018 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
; GFX10-DL-XNACK-NEXT: s_ashr_i32 s0, s6, 28 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v8, v15, v1
; GFX10-DL-XNACK-NEXT: s_ashr_i32 s1, s7, 28 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
; GFX10-DL-XNACK-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v5
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v6, v7, v1
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v5
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v2, v3, v1
; GFX10-DL-XNACK-NEXT: v_mad_u16 v1, v4, v7, v1
; GFX10-DL-XNACK-NEXT: global_store_byte v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm ; GFX10-DL-XNACK-NEXT: s_endpgm
; ;
; GFX10-DL-NOXNACK-LABEL: idot8_acc8: ; GFX10-DL-NOXNACK-LABEL: idot8_acc8:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry ; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s15, 0x31c16000 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s12, s12, s3 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s2, s0, 12 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s1, 12 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s6, s0, 0x40000 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s7, s1, 0x40000 ; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v3, v2, s[0:1]
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s2 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s3 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v4, 28, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s8, s0, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s9, s0, 0x40008 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s10, s1, 0x40008 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s1, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e64 v4, s9, s10 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40010 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 4, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v17, 12, v17
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s6, s7, v1 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v18, v17, v3
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s8, s2, v1 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v9
; GFX10-DL-NOXNACK-NEXT: s_movk_i32 s2, 0xff ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v15
; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, s2, v3 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v1, v10, v16, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40010 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_mad_u32_u24 v1, v2, v3, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v8
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40014 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40014 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v3, v0, v1
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40018 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v7
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40018 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v13
; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s0, s0, 28 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v10
; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s1, s1, 28 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v15, v9, v0
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v12
; GFX10-DL-NOXNACK-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v11
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v8
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v4, 12, v4
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v4
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v1, v3, v0
; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v7, v5, v0
; GFX10-DL-NOXNACK-NEXT: global_store_byte v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm ; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc8: ; GFX10-DL-LABEL: idot8_acc8:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-NEXT: s_mov_b32 s14, -1
Show All 39 Lines
; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 28 ; GFX10-DL-NEXT: s_ashr_i32 s1, s1, 28
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NEXT: v_mad_i32_i24 v1, s2, s3, v1
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i8 %cv1e0 = sext i4 %v1e0 to i8
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = sext i4 %v2e0 to i8 %cv2e0 = sext i4 %v2e0 to i8
%mul0 = mul nuw nsw i8 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Linesentry:
ret void ret void
} }
; Make sure the pattern is not recognized if there are multiple uses of the ; Make sure the pattern is not recognized if there are multiple uses of the
; intermediate multiplications. ; intermediate multiplications.
define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_multiuses_mul1: ; GFX7-LABEL: idot8_multiuses_mul1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_bfe_i32 s7, s5, 0x40000
; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000
; GFX7-NEXT: v_mov_b32_e32 v0, s7
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mad_i32_i24 v1, s6, v0, v1
; GFX7-NEXT: s_bfe_i32 s9, s5, 0x40004
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x40004
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x40008
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s9
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v2, v0
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40008
; GFX7-NEXT: v_mov_b32_e32 v2, s11
; GFX7-NEXT: s_bfe_i32 s13, s5, 0x4000c
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v2, v0
; GFX7-NEXT: s_bfe_i32 s12, s4, 0x4000c
; GFX7-NEXT: v_mov_b32_e32 v2, s13
; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40010
; GFX7-NEXT: v_mad_i32_i24 v0, s12, v2, v0
; GFX7-NEXT: s_bfe_i32 s14, s4, 0x40010
; GFX7-NEXT: v_mov_b32_e32 v2, s15
; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40014
; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40018
; GFX7-NEXT: v_mad_i32_i24 v0, s14, v2, v0
; GFX7-NEXT: s_bfe_i32 s16, s4, 0x40014
; GFX7-NEXT: v_mov_b32_e32 v2, s17
; GFX7-NEXT: s_bfe_i32 s18, s4, 0x40018
; GFX7-NEXT: v_mad_i32_i24 v0, s16, v2, v0
; GFX7-NEXT: v_mov_b32_e32 v2, s19
; GFX7-NEXT: s_ashr_i32 s5, s5, 28
; GFX7-NEXT: v_mad_i32_i24 v0, s18, v2, v0
; GFX7-NEXT: s_ashr_i32 s4, s4, 28
; GFX7-NEXT: v_mov_b32_e32 v2, s5
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v2, v0
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 4
; GFX7-NEXT: v_bfe_i32 v3, v2, 4, 4
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_bfe_i32 v9, v0, 0, 4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_i32_i24 v16, v1, v9, s4
; GFX7-NEXT: v_bfe_i32 v10, v0, 4, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v1, v9, v16
; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 4
; GFX7-NEXT: v_bfe_i32 v11, v0, 8, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v3, v10, v1
; GFX7-NEXT: v_bfe_i32 v5, v2, 12, 4
; GFX7-NEXT: v_bfe_i32 v12, v0, 12, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v4, v11, v1
; GFX7-NEXT: v_bfe_i32 v6, v2, 16, 4
; GFX7-NEXT: v_bfe_i32 v13, v0, 16, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v5, v12, v1
; GFX7-NEXT: v_bfe_i32 v7, v2, 20, 4
; GFX7-NEXT: v_bfe_i32 v14, v0, 20, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v6, v13, v1
; GFX7-NEXT: v_bfe_i32 v8, v2, 24, 4
; GFX7-NEXT: v_bfe_i32 v15, v0, 24, 4
; GFX7-NEXT: v_mad_i32_i24 v1, v7, v14, v1
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 28, v2
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 28, v0
; GFX7-NEXT: v_mad_i32_i24 v1, v8, v15, v1
; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v16, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot8_multiuses_mul1: ; GFX8-LABEL: idot8_multiuses_mul1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s23, 0xe80000 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_add_u32 s20, s20, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 4
; GFX8-NEXT: v_bfe_i32 v4, v3, 4, 4
; GFX8-NEXT: v_bfe_i32 v6, v3, 8, 4
; GFX8-NEXT: v_bfe_i32 v8, v3, 12, 4
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 4
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_bfe_i32 s4, s2, 0x40000 ; GFX8-NEXT: v_mad_i32_i24 v16, v1, v2, s2
; GFX8-NEXT: s_bfe_i32 s5, s3, 0x40000 ; GFX8-NEXT: v_bfe_i32 v5, v0, 4, 4
; GFX8-NEXT: v_mov_b32_e32 v0, s5 ; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, v16
; GFX8-NEXT: v_mov_b32_e32 v1, s18 ; GFX8-NEXT: v_bfe_i32 v7, v0, 8, 4
; GFX8-NEXT: v_mad_i32_i24 v1, s4, v0, v1 ; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v1
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x40004 ; GFX8-NEXT: v_bfe_i32 v9, v0, 12, 4
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40004 ; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1
; GFX8-NEXT: s_bfe_i32 s9, s3, 0x40008 ; GFX8-NEXT: v_bfe_i32 v10, v3, 16, 4
; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1 ; GFX8-NEXT: v_bfe_i32 v11, v0, 16, 4
; GFX8-NEXT: v_mov_b32_e32 v2, s7 ; GFX8-NEXT: v_mad_i32_i24 v1, v8, v9, v1
; GFX8-NEXT: v_mad_i32_i24 v0, s6, v2, v0 ; GFX8-NEXT: v_bfe_i32 v12, v3, 20, 4
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x40008 ; GFX8-NEXT: v_bfe_i32 v13, v0, 20, 4
; GFX8-NEXT: v_mov_b32_e32 v2, s9 ; GFX8-NEXT: v_mad_i32_i24 v1, v10, v11, v1
; GFX8-NEXT: s_bfe_i32 s11, s3, 0x4000c ; GFX8-NEXT: v_bfe_i32 v14, v3, 24, 4
; GFX8-NEXT: v_mad_i32_i24 v0, s8, v2, v0 ; GFX8-NEXT: v_bfe_i32 v15, v0, 24, 4
; GFX8-NEXT: s_bfe_i32 s10, s2, 0x4000c ; GFX8-NEXT: v_mad_i32_i24 v1, v12, v13, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s11 ; GFX8-NEXT: v_ashrrev_i32_e32 v3, 28, v3
; GFX8-NEXT: s_bfe_i32 s13, s3, 0x40010 ; GFX8-NEXT: v_ashrrev_i32_e32 v0, 28, v0
; GFX8-NEXT: v_mad_i32_i24 v0, s10, v2, v0 ; GFX8-NEXT: v_mad_i32_i24 v1, v14, v15, v1
; GFX8-NEXT: s_bfe_i32 s12, s2, 0x40010 ; GFX8-NEXT: v_mad_i32_i24 v0, v3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s13 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v16, v0
; GFX8-NEXT: s_bfe_i32 s15, s3, 0x40014
; GFX8-NEXT: s_bfe_i32 s17, s3, 0x40018
; GFX8-NEXT: v_mad_i32_i24 v0, s12, v2, v0
; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40014
; GFX8-NEXT: v_mov_b32_e32 v2, s15
; GFX8-NEXT: s_bfe_i32 s16, s2, 0x40018
; GFX8-NEXT: v_mad_i32_i24 v0, s14, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v2, s17
; GFX8-NEXT: s_ashr_i32 s3, s3, 28
; GFX8-NEXT: v_mad_i32_i24 v0, s16, v2, v0
; GFX8-NEXT: s_ashr_i32 s2, s2, 28
; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: v_mad_i32_i24 v0, s2, v2, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: idot8_multiuses_mul1: ; GFX9-LABEL: idot8_multiuses_mul1:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0 ; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_bfe_i32 v3, v1, 0, 4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_bfe_i32 v4, v2, 0, 4
; GFX9-NEXT: v_bfe_i32 v5, v1, 4, 4
; GFX9-NEXT: v_bfe_i32 v6, v2, 4, 4
; GFX9-NEXT: v_bfe_i32 v7, v1, 8, 4
; GFX9-NEXT: v_bfe_i32 v8, v2, 8, 4
; GFX9-NEXT: v_bfe_i32 v9, v1, 12, 4
; GFX9-NEXT: v_bfe_i32 v10, v2, 12, 4
; GFX9-NEXT: v_bfe_i32 v11, v1, 16, 4
; GFX9-NEXT: v_bfe_i32 v12, v2, 16, 4
; GFX9-NEXT: v_bfe_i32 v13, v1, 20, 4
; GFX9-NEXT: v_bfe_i32 v15, v1, 24, 4
; GFX9-NEXT: v_bfe_i32 v14, v2, 20, 4
; GFX9-NEXT: v_bfe_i32 v16, v2, 24, 4
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 28, v1
; GFX9-NEXT: v_ashrrev_i32_e32 v2, 28, v2
; GFX9-NEXT: v_mul_i32_i24_e32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s4, s0, 0x40000 ; GFX9-NEXT: v_mad_i32_i24 v2, v3, v4, s0
; GFX9-NEXT: s_bfe_i32 s5, s1, 0x40000 ; GFX9-NEXT: v_mul_i32_i24_e32 v5, v5, v6
; GFX9-NEXT: v_mov_b32_e32 v1, s5 ; GFX9-NEXT: v_mul_i32_i24_e32 v6, v7, v8
; GFX9-NEXT: v_mov_b32_e32 v2, s18 ; GFX9-NEXT: v_mad_i32_i24 v3, v3, v4, v2
; GFX9-NEXT: v_mad_i32_i24 v2, s4, v1, v2 ; GFX9-NEXT: v_mul_i32_i24_e32 v7, v9, v10
; GFX9-NEXT: s_bfe_i32 s7, s1, 0x40004 ; GFX9-NEXT: v_mul_i32_i24_e32 v8, v11, v12
; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40004 ; GFX9-NEXT: v_add3_u32 v3, v3, v5, v6
; GFX9-NEXT: s_bfe_i32 s9, s1, 0x40008 ; GFX9-NEXT: v_mul_i32_i24_e32 v9, v13, v14
; GFX9-NEXT: v_mad_i32_i24 v1, s4, v1, v2 ; GFX9-NEXT: v_mul_i32_i24_e32 v10, v15, v16
; GFX9-NEXT: v_mov_b32_e32 v3, s7 ; GFX9-NEXT: v_add3_u32 v3, v3, v7, v8
; GFX9-NEXT: v_mad_i32_i24 v1, s6, v3, v1 ; GFX9-NEXT: v_add3_u32 v3, v3, v9, v10
; GFX9-NEXT: s_bfe_i32 s8, s0, 0x40008 ; GFX9-NEXT: v_add3_u32 v1, v3, v1, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s9
; GFX9-NEXT: s_bfe_i32 s11, s1, 0x4000c
; GFX9-NEXT: v_mad_i32_i24 v1, s8, v3, v1
; GFX9-NEXT: s_bfe_i32 s10, s0, 0x4000c
; GFX9-NEXT: v_mov_b32_e32 v3, s11
; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40010
; GFX9-NEXT: v_mad_i32_i24 v1, s10, v3, v1
; GFX9-NEXT: s_bfe_i32 s12, s0, 0x40010
; GFX9-NEXT: v_mov_b32_e32 v3, s13
; GFX9-NEXT: s_bfe_i32 s15, s1, 0x40014
; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40018
; GFX9-NEXT: v_mad_i32_i24 v1, s12, v3, v1
; GFX9-NEXT: s_bfe_i32 s14, s0, 0x40014
; GFX9-NEXT: v_mov_b32_e32 v3, s15
; GFX9-NEXT: s_bfe_i32 s16, s0, 0x40018
; GFX9-NEXT: v_mad_i32_i24 v1, s14, v3, v1
; GFX9-NEXT: v_mov_b32_e32 v3, s17
; GFX9-NEXT: s_ashr_i32 s1, s1, 28
; GFX9-NEXT: v_mad_i32_i24 v1, s16, v3, v1
; GFX9-NEXT: s_ashr_i32 s0, s0, 28
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mad_i32_i24 v1, s0, v3, v1
; GFX9-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot8_multiuses_mul1: ; GFX9-DL-LABEL: idot8_multiuses_mul1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s18, s[2:3], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 4
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 4
; GFX9-DL-NEXT: v_bfe_i32 v5, v1, 4, 4
; GFX9-DL-NEXT: v_bfe_i32 v6, v2, 4, 4
; GFX9-DL-NEXT: v_bfe_i32 v7, v1, 8, 4
; GFX9-DL-NEXT: v_bfe_i32 v8, v2, 8, 4
; GFX9-DL-NEXT: v_bfe_i32 v9, v1, 12, 4
; GFX9-DL-NEXT: v_bfe_i32 v10, v2, 12, 4
; GFX9-DL-NEXT: v_bfe_i32 v11, v1, 16, 4
; GFX9-DL-NEXT: v_bfe_i32 v12, v2, 16, 4
; GFX9-DL-NEXT: v_bfe_i32 v13, v1, 20, 4
; GFX9-DL-NEXT: v_bfe_i32 v15, v1, 24, 4
; GFX9-DL-NEXT: v_bfe_i32 v14, v2, 20, 4
; GFX9-DL-NEXT: v_bfe_i32 v16, v2, 24, 4
; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 28, v1
; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 28, v2
; GFX9-DL-NEXT: v_mul_i32_i24_e32 v1, v1, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_bfe_i32 s4, s0, 0x40000 ; GFX9-DL-NEXT: v_mad_i32_i24 v2, v3, v4, s0
; GFX9-DL-NEXT: s_bfe_i32 s5, s1, 0x40000 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v5, v5, v6
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v6, v7, v8
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s18 ; GFX9-DL-NEXT: v_mad_i32_i24 v3, v3, v4, v2
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s4, v1, v2 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v7, v9, v10
; GFX9-DL-NEXT: s_bfe_i32 s7, s1, 0x40004 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v8, v11, v12
; GFX9-DL-NEXT: s_bfe_i32 s6, s0, 0x40004 ; GFX9-DL-NEXT: v_add3_u32 v3, v3, v5, v6
; GFX9-DL-NEXT: s_bfe_i32 s9, s1, 0x40008 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v9, v13, v14
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v1, v2 ; GFX9-DL-NEXT: v_mul_i32_i24_e32 v10, v15, v16
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7 ; GFX9-DL-NEXT: v_add3_u32 v3, v3, v7, v8
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s6, v3, v1 ; GFX9-DL-NEXT: v_add3_u32 v3, v3, v9, v10
; GFX9-DL-NEXT: s_bfe_i32 s8, s0, 0x40008 ; GFX9-DL-NEXT: v_add3_u32 v1, v3, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s9
; GFX9-DL-NEXT: s_bfe_i32 s11, s1, 0x4000c
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s8, v3, v1
; GFX9-DL-NEXT: s_bfe_i32 s10, s0, 0x4000c
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s11
; GFX9-DL-NEXT: s_bfe_i32 s13, s1, 0x40010
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s10, v3, v1
; GFX9-DL-NEXT: s_bfe_i32 s12, s0, 0x40010
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s13
; GFX9-DL-NEXT: s_bfe_i32 s15, s1, 0x40014
; GFX9-DL-NEXT: s_bfe_i32 s17, s1, 0x40018
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s12, v3, v1
; GFX9-DL-NEXT: s_bfe_i32 s14, s0, 0x40014
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s15
; GFX9-DL-NEXT: s_bfe_i32 s16, s0, 0x40018
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s14, v3, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
; GFX9-DL-NEXT: s_ashr_i32 s1, s1, 28
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s16, v3, v1
; GFX9-DL-NEXT: s_ashr_i32 s0, s0, 28
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s0, v3, v1
; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-XNACK-LABEL: idot8_multiuses_mul1: ; GFX10-DL-XNACK-LABEL: idot8_multiuses_mul1:
; GFX10-DL-XNACK: ; %bb.0: ; %entry ; GFX10-DL-XNACK: ; %bb.0: ; %entry
; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-XNACK-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v0, v1, 0, 4
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v3, v1, 4, 4
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v4, v2, 4, 4
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v5, v1, 8, 4
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v6, v2, 8, 4
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v7, v2, 0, 4
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v8, v1, 12, 4
; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v3, v3, v4
; GFX10-DL-XNACK-NEXT: v_bfe_i32 v9, v2, 12, 4
; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v4, v5, v6
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v5, v0, v7, s2
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40000 ; GFX10-DL-XNACK-NEXT: v_bfe_i32 v15, v1, 16, 4
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40000 ; GFX10-DL-XNACK-NEXT: v_bfe_i32 v10, v2, 16, 4
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v0, s4, s5, v0 ; GFX10-DL-XNACK-NEXT: v_bfe_i32 v11, v1, 20, 4
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v0 ; GFX10-DL-XNACK-NEXT: v_bfe_i32 v12, v2, 20, 4
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40004 ; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v14, v0, v7, v5
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40004 ; GFX10-DL-XNACK-NEXT: v_bfe_i32 v7, v1, 24, 4
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1 ; GFX10-DL-XNACK-NEXT: v_bfe_i32 v13, v2, 24, 4
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40008 ; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v6, v15, v10
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40008 ; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v8, v8, v9
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1 ; GFX10-DL-XNACK-NEXT: v_add3_u32 v15, v14, v3, v4
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x4000c ; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v3, v11, v12
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x4000c ; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v4, v7, v13
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i32_e32 v1, 28, v1
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40010 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i32_e32 v2, 28, v2
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40010 ; GFX10-DL-XNACK-NEXT: v_add3_u32 v6, v15, v8, v6
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1 ; GFX10-DL-XNACK-NEXT: v_mul_i32_i24_e32 v7, v1, v2
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40014 ; GFX10-DL-XNACK-NEXT: v_add3_u32 v0, v6, v3, v4
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40014 ; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1 ; GFX10-DL-XNACK-NEXT: v_add3_u32 v0, v0, v7, v5
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s4, s0, 0x40018 ; GFX10-DL-XNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-XNACK-NEXT: s_bfe_i32 s5, s1, 0x40018
; GFX10-DL-XNACK-NEXT: s_ashr_i32 s0, s0, 28
; GFX10-DL-XNACK-NEXT: s_ashr_i32 s1, s1, 28
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s4, s5, v1
; GFX10-DL-XNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX10-DL-XNACK-NEXT: global_store_dword v2, v0, s[2:3]
; GFX10-DL-XNACK-NEXT: s_endpgm ; GFX10-DL-XNACK-NEXT: s_endpgm
; ;
; GFX10-DL-NOXNACK-LABEL: idot8_multiuses_mul1: ; GFX10-DL-NOXNACK-LABEL: idot8_multiuses_mul1:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry ; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: s_load_dword s6, s[4:5], 0x0 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-DL-NOXNACK-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v2, v1, 0, 4
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v3, v1, 4, 4
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v4, v0, 4, 4
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v5, v1, 8, 4
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v6, v0, 8, 4
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v7, v0, 0, 4
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v8, v1, 12, 4
; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v3, v3, v4
; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v9, v0, 12, 4
; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v4, v5, v6
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, s6 ; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v5, v2, v7, s2
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40000 ; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v15, v1, 16, 4
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40000 ; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v10, v0, 16, 4
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v0, s2, s3, v0 ; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v11, v1, 20, 4
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v0 ; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v12, v0, 20, 4
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v2, v2, v7, v5
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v7, v1, 24, 4
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_bfe_i32 v13, v0, 24, 4
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40008 ; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v6, v15, v10
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40008 ; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v8, v8, v9
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v15, v2, v3, v4
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x4000c ; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v3, v11, v12
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x4000c ; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v4, v7, v13
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i32_e32 v1, 28, v1
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40010 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i32_e32 v0, 28, v0
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40010 ; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v2, v15, v8, v6
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_mul_i32_i24_e32 v0, v1, v0
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40014 ; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v3, v2, v3, v4
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40014 ; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NOXNACK-NEXT: v_add3_u32 v0, v3, v0, v5
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s2, s0, 0x40018 ; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_bfe_i32 s3, s1, 0x40018
; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s0, s0, 28
; GFX10-DL-NOXNACK-NEXT: s_ashr_i32 s1, s1, 28
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s2, s3, v1
; GFX10-DL-NOXNACK-NEXT: v_mad_i32_i24 v1, s0, s1, v1
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_endpgm ; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_multiuses_mul1: ; GFX10-DL-LABEL: idot8_multiuses_mul1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
Show All 34 Lines
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s2, s3, v1 ; GFX10-DL-NEXT: v_mad_i32_i24 v1, s2, s3, v1
; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_i32_i24 v1, s0, s1, v1
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v1 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX10-DL-NEXT: global_store_dword v2, v0, s[4:5] ; GFX10-DL-NEXT: global_store_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = sext i4 %v1e0 to i32 %cv1e0 = sext i4 %v1e0 to i32
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = sext i4 %v2e0 to i32 %cv2e0 = sext i4 %v2e0 to i32
%mul0 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Linesentry:
store i32 %res, i32 addrspace(1)* %dst, align 4 store i32 %res, i32 addrspace(1)* %dst, align 4
ret void ret void
} }
; TODO: Support this pattern. ; TODO: Support this pattern.
define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc32_vecMul: ; GFX7-LABEL: idot8_acc32_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_ashr_i32 s13, s5, 28
; GFX7-NEXT: s_bfe_i32 s14, s5, 0x40018
; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40014
; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40010
; GFX7-NEXT: s_bfe_i32 s17, s5, 0x4000c
; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40008
; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40004
; GFX7-NEXT: s_bfe_i32 s5, s5, 0x40000
; GFX7-NEXT: s_ashr_i32 s6, s4, 28
; GFX7-NEXT: s_bfe_i32 s7, s4, 0x40018
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x40014
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40010
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x4000c
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008
; GFX7-NEXT: s_bfe_i32 s12, s4, 0x40004
; GFX7-NEXT: s_bfe_i32 s4, s4, 0x40000
; GFX7-NEXT: v_mov_b32_e32 v0, s5
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s19
; GFX7-NEXT: v_mad_i32_i24 v0, s12, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s18
; GFX7-NEXT: v_mad_i32_i24 v0, s11, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s17
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s16
; GFX7-NEXT: v_mad_i32_i24 v0, s9, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s15
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s14
; GFX7-NEXT: v_mad_i32_i24 v0, s7, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s13
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 28, v2
; GFX7-NEXT: v_bfe_i32 v3, v2, 24, 4
; GFX7-NEXT: v_bfe_i32 v4, v2, 20, 4
; GFX7-NEXT: v_bfe_i32 v5, v2, 16, 4
; GFX7-NEXT: v_bfe_i32 v6, v2, 12, 4
; GFX7-NEXT: v_bfe_i32 v7, v2, 8, 4
; GFX7-NEXT: v_bfe_i32 v8, v2, 4, 4
; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 4
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_ashrrev_i32_e32 v9, 28, v0
; GFX7-NEXT: v_bfe_i32 v10, v0, 24, 4
; GFX7-NEXT: v_bfe_i32 v11, v0, 20, 4
; GFX7-NEXT: v_bfe_i32 v12, v0, 16, 4
; GFX7-NEXT: v_bfe_i32 v13, v0, 12, 4
; GFX7-NEXT: v_bfe_i32 v14, v0, 8, 4
; GFX7-NEXT: v_bfe_i32 v15, v0, 4, 4
; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, s4
; GFX7-NEXT: v_mad_i32_i24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_i32_i24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_i32_i24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_i32_i24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_i32_i24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_i32_i24 v0, v3, v10, v0
; GFX7-NEXT: v_mad_i32_i24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot8_acc32_vecMul: ; GFX8-LABEL: idot8_acc32_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s23, 0xe80000 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_add_u32 s20, s20, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_ashrrev_i32_e32 v2, 28, v0
; GFX8-NEXT: v_bfe_i32 v3, v0, 24, 4
; GFX8-NEXT: v_bfe_i32 v4, v0, 20, 4
; GFX8-NEXT: v_bfe_i32 v5, v0, 16, 4
; GFX8-NEXT: v_bfe_i32 v6, v0, 12, 4
; GFX8-NEXT: v_bfe_i32 v7, v0, 8, 4
; GFX8-NEXT: v_bfe_i32 v8, v0, 4, 4
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 4
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_ashrrev_i32_e32 v9, 28, v1
; GFX8-NEXT: v_bfe_i32 v10, v1, 24, 4
; GFX8-NEXT: v_bfe_i32 v11, v1, 20, 4
; GFX8-NEXT: v_bfe_i32 v12, v1, 16, 4
; GFX8-NEXT: v_bfe_i32 v13, v1, 12, 4
; GFX8-NEXT: v_bfe_i32 v14, v1, 8, 4
; GFX8-NEXT: v_bfe_i32 v15, v1, 4, 4
; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 4
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_ashr_i32 s4, s2, 28 ; GFX8-NEXT: v_mad_i32_i24 v0, v0, v1, s2
; GFX8-NEXT: s_ashr_i32 s11, s3, 28 ; GFX8-NEXT: v_mad_i32_i24 v0, v8, v15, v0
; GFX8-NEXT: s_bfe_i32 s12, s3, 0x40018 ; GFX8-NEXT: v_mad_i32_i24 v0, v7, v14, v0
; GFX8-NEXT: s_bfe_i32 s13, s3, 0x40014 ; GFX8-NEXT: v_mad_i32_i24 v0, v6, v13, v0
; GFX8-NEXT: s_bfe_i32 s14, s3, 0x40010 ; GFX8-NEXT: v_mad_i32_i24 v0, v5, v12, v0
; GFX8-NEXT: s_bfe_i32 s15, s3, 0x4000c ; GFX8-NEXT: v_mad_i32_i24 v0, v4, v11, v0
; GFX8-NEXT: s_bfe_i32 s16, s3, 0x40008 ; GFX8-NEXT: v_mad_i32_i24 v0, v3, v10, v0
; GFX8-NEXT: s_bfe_i32 s17, s3, 0x40004 ; GFX8-NEXT: v_mad_i32_i24 v2, v2, v9, v0
; GFX8-NEXT: s_bfe_i32 s3, s3, 0x40000
; GFX8-NEXT: s_bfe_i32 s5, s2, 0x40018
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x40014
; GFX8-NEXT: s_bfe_i32 s7, s2, 0x40010
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x4000c
; GFX8-NEXT: s_bfe_i32 s9, s2, 0x40008
; GFX8-NEXT: s_bfe_i32 s10, s2, 0x40004
; GFX8-NEXT: s_bfe_i32 s2, s2, 0x40000
; GFX8-NEXT: v_mov_b32_e32 v0, s3
; GFX8-NEXT: v_mov_b32_e32 v1, s18
; GFX8-NEXT: v_mad_i32_i24 v0, s2, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s17
; GFX8-NEXT: v_mad_i32_i24 v0, s10, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s16
; GFX8-NEXT: v_mad_i32_i24 v0, s9, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s15
; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s14
; GFX8-NEXT: v_mad_i32_i24 v0, s7, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s13
; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s12
; GFX8-NEXT: v_mad_i32_i24 v0, s5, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s11
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: idot8_acc32_vecMul: ; GFX9-LABEL: idot8_acc32_vecMul:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0 ; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_ashrrev_i32_e32 v3, 28, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_ashrrev_i32_e32 v10, 28, v2
; GFX9-NEXT: v_bfe_i32 v4, v1, 24, 4
; GFX9-NEXT: v_bfe_i32 v11, v2, 24, 4
; GFX9-NEXT: v_bfe_i32 v5, v1, 20, 4
; GFX9-NEXT: v_bfe_i32 v12, v2, 20, 4
; GFX9-NEXT: v_bfe_i32 v6, v1, 16, 4
; GFX9-NEXT: v_bfe_i32 v13, v2, 16, 4
; GFX9-NEXT: v_bfe_i32 v7, v1, 12, 4
; GFX9-NEXT: v_bfe_i32 v14, v2, 12, 4
; GFX9-NEXT: v_bfe_i32 v8, v1, 8, 4
; GFX9-NEXT: v_bfe_i32 v9, v1, 4, 4
; GFX9-NEXT: v_bfe_i32 v15, v2, 8, 4
; GFX9-NEXT: v_bfe_i32 v16, v2, 4, 4
; GFX9-NEXT: v_bfe_i32 v1, v1, 0, 4
; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 4
; GFX9-NEXT: v_mul_i32_i24_e32 v1, v1, v2
; GFX9-NEXT: v_mul_i32_i24_e32 v2, v9, v16
; GFX9-NEXT: v_mul_i32_i24_e32 v8, v8, v15
; GFX9-NEXT: v_mul_i32_i24_e32 v7, v7, v14
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_ashr_i32 s4, s0, 28 ; GFX9-NEXT: v_add3_u32 v1, v1, s0, v2
; GFX9-NEXT: s_ashr_i32 s11, s1, 28 ; GFX9-NEXT: v_mul_i32_i24_e32 v6, v6, v13
; GFX9-NEXT: s_bfe_i32 s12, s1, 0x40018 ; GFX9-NEXT: v_mul_i32_i24_e32 v5, v5, v12
; GFX9-NEXT: s_bfe_i32 s13, s1, 0x40014 ; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7
; GFX9-NEXT: s_bfe_i32 s14, s1, 0x40010 ; GFX9-NEXT: v_mul_i32_i24_e32 v4, v4, v11
; GFX9-NEXT: s_bfe_i32 s15, s1, 0x4000c ; GFX9-NEXT: v_mul_i32_i24_e32 v3, v3, v10
; GFX9-NEXT: s_bfe_i32 s16, s1, 0x40008 ; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
; GFX9-NEXT: s_bfe_i32 s17, s1, 0x40004 ; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX9-NEXT: s_bfe_i32 s1, s1, 0x40000
; GFX9-NEXT: s_bfe_i32 s5, s0, 0x40018
; GFX9-NEXT: s_bfe_i32 s6, s0, 0x40014
; GFX9-NEXT: s_bfe_i32 s7, s0, 0x40010
; GFX9-NEXT: s_bfe_i32 s8, s0, 0x4000c
; GFX9-NEXT: s_bfe_i32 s9, s0, 0x40008
; GFX9-NEXT: s_bfe_i32 s10, s0, 0x40004
; GFX9-NEXT: s_bfe_i32 s0, s0, 0x40000
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mad_i32_i24 v1, s0, v1, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s17
; GFX9-NEXT: v_mad_i32_i24 v1, s10, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s16
; GFX9-NEXT: v_mad_i32_i24 v1, s9, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s15
; GFX9-NEXT: v_mad_i32_i24 v1, s8, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s14
; GFX9-NEXT: v_mad_i32_i24 v1, s7, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s13
; GFX9-NEXT: v_mad_i32_i24 v1, s6, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s12
; GFX9-NEXT: v_mad_i32_i24 v1, s5, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s11
; GFX9-NEXT: v_mad_i32_i24 v1, s4, v2, v1
; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot8_acc32_vecMul: ; GFX9-DL-LABEL: idot8_acc32_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s14, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s12, s12, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: v_dot8_i32_i4 v0, v2, v3, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot8_i32_i4 v1, s0, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-XNACK-LABEL: idot8_acc32_vecMul: ; GFX10-DL-XNACK-LABEL: idot8_acc32_vecMul:
; GFX10-DL-XNACK: ; %bb.0: ; %entry ; GFX10-DL-XNACK: ; %bb.0: ; %entry
; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1
; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3
; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-XNACK-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0 ; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-XNACK-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: v_dot8_i32_i4 v1, v1, v2, s2
; GFX10-DL-XNACK-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: s_endpgm ; GFX10-DL-XNACK-NEXT: s_endpgm
; ;
; GFX10-DL-NOXNACK-LABEL: idot8_acc32_vecMul: ; GFX10-DL-NOXNACK-LABEL: idot8_acc32_vecMul:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry ; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: s_load_dword s6, s[4:5], 0x0 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, s6 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, s0, s1, v0 ; GFX10-DL-NOXNACK-NEXT: v_dot8_i32_i4 v0, v1, v0, s2
; GFX10-DL-NOXNACK-NEXT: global_store_dword v1, v0, s[4:5] ; GFX10-DL-NOXNACK-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm ; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc32_vecMul: ; GFX10-DL-LABEL: idot8_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
Show All 9 Lines
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s6 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s6
; GFX10-DL-NEXT: v_dot8_i32_i4 v0, s0, s1, v0 ; GFX10-DL-NEXT: v_dot8_i32_i4 v0, s0, s1, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[4:5] ; GFX10-DL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = sext <8 x i4> %vec1 to <8 x i32> %cvec1 = sext <8 x i4> %vec1 to <8 x i32>
%cvec2 = sext <8 x i4> %vec2 to <8 x i32> %cvec2 = sext <8 x i4> %vec2 to <8 x i32>
%mul = mul <8 x i32> %cvec1, %cvec2 %mul = mul <8 x i32> %cvec1, %cvec2
%mul0 = extractelement <8 x i32> %mul, i64 0 %mul0 = extractelement <8 x i32> %mul, i64 0
%mul1 = extractelement <8 x i32> %mul, i64 1 %mul1 = extractelement <8 x i32> %mul, i64 1
%mul2 = extractelement <8 x i32> %mul, i64 2 %mul2 = extractelement <8 x i32> %mul, i64 2
Show All 16 Linesentry:
store i32 %add8, i32 addrspace(1)* %dst, align 4 store i32 %add8, i32 addrspace(1)* %dst, align 4
ret void ret void
} }
; TODO: Support this pattern. ; TODO: Support this pattern.
define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc16_vecMul: ; GFX7-LABEL: idot8_acc16_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_mov_b32 s8, 0xffff ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_ashr_i32 s6, s4, 28 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40018 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40014 ; GFX7-NEXT: s_mov_b32 s4, 0xffff
; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40010 ; GFX7-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX7-NEXT: s_bfe_i32 s18, s5, 0x40008 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_i32 s19, s5, 0x4000c ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40000 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_ashr_i32 s14, s5, 28 ; GFX7-NEXT: v_bfe_i32 v1, v3, 20, 4
; GFX7-NEXT: s_bfe_i32 s5, s5, 0x40004 ; GFX7-NEXT: v_bfe_i32 v4, v3, 16, 4
; GFX7-NEXT: s_bfe_i32 s7, s4, 0x40018 ; GFX7-NEXT: v_bfe_i32 v5, v3, 4, 4
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x40014 ; GFX7-NEXT: v_bfe_i32 v6, v3, 0, 4
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40010
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x40008
; GFX7-NEXT: v_mov_b32_e32 v4, s18
; GFX7-NEXT: s_bfe_i32 s12, s4, 0x4000c
; GFX7-NEXT: v_mov_b32_e32 v3, s19
; GFX7-NEXT: s_bfe_i32 s13, s4, 0x40000
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: s_bfe_i32 s4, s4, 0x40004
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mul_i32_i24_e32 v1, s4, v1
; GFX7-NEXT: v_mul_i32_i24_e32 v2, s13, v2
; GFX7-NEXT: v_mul_i32_i24_e32 v3, s12, v3
; GFX7-NEXT: v_mul_i32_i24_e32 v4, s11, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_and_b32_e32 v2, s8, v2 ; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_and_b32_e32 v4, s8, v4 ; GFX7-NEXT: v_and_b32_e32 v6, s4, v6
; GFX7-NEXT: v_or_b32_e32 v3, v4, v3
; GFX7-NEXT: v_or_b32_e32 v2, v2, v1
; GFX7-NEXT: v_alignbit_b32 v1, v3, v1, 16
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: v_mov_b32_e32 v5, s17
; GFX7-NEXT: v_mov_b32_e32 v6, s16
; GFX7-NEXT: v_mov_b32_e32 v7, s15
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GFX7-NEXT: v_bfe_i32 v10, v0, 20, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0 ; GFX7-NEXT: v_bfe_i32 v11, v0, 16, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0 ; GFX7-NEXT: v_bfe_i32 v12, v0, 4, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0 ; GFX7-NEXT: v_bfe_i32 v13, v0, 0, 4
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v5, v0 ; GFX7-NEXT: v_or_b32_e32 v1, v4, v1
; GFX7-NEXT: v_mad_i32_i24 v0, s9, v6, v0 ; GFX7-NEXT: v_or_b32_e32 v4, v6, v5
; GFX7-NEXT: v_mad_i32_i24 v0, s7, v7, v0 ; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v10
; GFX7-NEXT: v_mov_b32_e32 v1, s14 ; GFX7-NEXT: v_and_b32_e32 v6, s4, v11
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0 ; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v12
; GFX7-NEXT: v_and_b32_e32 v11, v2, v13
; GFX7-NEXT: v_bfe_i32 v7, v3, 24, 4
; GFX7-NEXT: v_bfe_i32 v8, v3, 8, 4
; GFX7-NEXT: v_ashrrev_i32_e32 v9, 28, v3
; GFX7-NEXT: v_bfe_i32 v3, v3, 12, 4
; GFX7-NEXT: v_bfe_i32 v14, v0, 24, 4
; GFX7-NEXT: v_bfe_i32 v15, v0, 8, 4
; GFX7-NEXT: v_ashrrev_i32_e32 v16, 28, v0
; GFX7-NEXT: v_bfe_i32 v0, v0, 12, 4
; GFX7-NEXT: v_or_b32_e32 v5, v6, v5
; GFX7-NEXT: v_or_b32_e32 v6, v11, v10
; GFX7-NEXT: v_and_b32_e32 v12, v2, v14
; GFX7-NEXT: v_and_b32_e32 v13, v2, v15
; GFX7-NEXT: v_and_b32_e32 v14, v2, v16
; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v6
; GFX7-NEXT: v_and_b32_e32 v3, v2, v3
; GFX7-NEXT: v_and_b32_e32 v9, v2, v9
; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
; GFX7-NEXT: v_and_b32_e32 v4, v2, v4
; GFX7-NEXT: v_and_b32_e32 v1, v2, v1
; GFX7-NEXT: v_and_b32_e32 v6, v2, v6
; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; GFX7-NEXT: v_and_b32_e32 v2, v2, v5
; GFX7-NEXT: buffer_load_ushort v5, off, s[0:3], 0
; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v4, v4, v6, v5
; GFX7-NEXT: v_mad_u32_u24 v4, v16, v11, v4
; GFX7-NEXT: v_mad_u32_u24 v4, v8, v13, v4
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v4
; GFX7-NEXT: v_mad_u32_u24 v0, v1, v2, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v15, v10, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v7, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v9, v14, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot8_acc16_vecMul: ; GFX8-LABEL: idot8_acc16_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: v_mov_b32_e32 v5, 12
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s19, 0xe80000
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_addc_u32 s17, s17, 0 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_bfe_i32 s8, s0, 0x40000 ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_bfe_i32 s15, s1, 0x40000 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 4, v3
; GFX8-NEXT: s_bfe_i32 s10, s1, 0x40018 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3
; GFX8-NEXT: s_bfe_i32 s11, s1, 0x40014 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
; GFX8-NEXT: s_bfe_i32 s12, s1, 0x40010 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 20, v3
; GFX8-NEXT: s_bfe_i32 s13, s1, 0x4000c ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX8-NEXT: s_bfe_i32 s14, s1, 0x40004 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_ashr_i32 s9, s1, 28 ; GFX8-NEXT: v_lshrrev_b32_e32 v11, 4, v2
; GFX8-NEXT: s_bfe_i32 s1, s1, 0x40008 ; GFX8-NEXT: v_lshrrev_b32_e32 v12, 8, v2
; GFX8-NEXT: v_mov_b32_e32 v4, s15 ; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
; GFX8-NEXT: s_ashr_i32 s2, s0, 28 ; GFX8-NEXT: v_lshrrev_b32_e32 v14, 20, v2
; GFX8-NEXT: s_bfe_i32 s3, s0, 0x40018 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 28, v2
; GFX8-NEXT: s_bfe_i32 s4, s0, 0x40014 ; GFX8-NEXT: v_lshlrev_b16_sdwa v16, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: s_bfe_i32 s5, s0, 0x40010 ; GFX8-NEXT: v_lshlrev_b16_sdwa v17, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_bfe_i32 s6, s0, 0x4000c ; GFX8-NEXT: v_lshlrev_b16_sdwa v18, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: s_bfe_i32 s7, s0, 0x40004 ; GFX8-NEXT: v_lshlrev_b16_sdwa v5, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_bfe_i32 s0, s0, 0x40008 ; GFX8-NEXT: v_lshlrev_b16_e32 v3, 12, v3
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_lshlrev_b16_e32 v2, 12, v2
; GFX8-NEXT: v_mov_b32_e32 v5, s14 ; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX8-NEXT: v_mul_i32_i24_e32 v3, s0, v3 ; GFX8-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
; GFX8-NEXT: v_mov_b32_e32 v7, s12 ; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v2, s8, v4, v2 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
; GFX8-NEXT: v_mad_i32_i24 v2, s7, v5, v2 ; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX8-NEXT: v_mad_i32_i24 v2, s6, v6, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX8-NEXT: v_mad_i32_i24 v2, s5, v7, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v8, v2 ; GFX8-NEXT: v_mad_u16 v2, v6, v11, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s3, v9, v2 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2 ; GFX8-NEXT: v_mad_u16 v2, v7, v12, v2
; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v17
; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX8-NEXT: v_mad_u16 v2, v8, v13, v2
; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX8-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX8-NEXT: v_mad_u16 v2, v17, v5, v2
; GFX8-NEXT: v_lshlrev_b16_e32 v10, 12, v10
; GFX8-NEXT: v_lshlrev_b16_e32 v15, 12, v15
; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v16
; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v18
; GFX8-NEXT: v_mad_u16 v2, v9, v14, v2
; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v15
; GFX8-NEXT: v_mad_u16 v2, v16, v18, v2
; GFX8-NEXT: v_mad_u16 v2, v10, v15, v2
; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: idot8_acc16_vecMul: ; GFX9-LABEL: idot8_acc16_vecMul:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_u32 s1, s0, 0x40018 ; GFX9-NEXT: global_load_dword v3, v0, s[4:5]
; GFX9-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-NEXT: global_load_dword v4, v0, s[6:7]
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40010 ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40014 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-NEXT: v_and_b32_e32 v10, 15, v3
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x4000c ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_and_b32 s11, s0, 15 ; GFX9-NEXT: v_and_b32_e32 v17, 15, v4
; GFX9-NEXT: s_bfe_u32 s0, s0, 0x40004 ; GFX9-NEXT: v_bfe_u32 v0, v3, 24, 4
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s11, s0 ; GFX9-NEXT: v_bfe_u32 v6, v3, 16, 4
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 12, s0 op_sel_hi:[0,1] ; GFX9-NEXT: v_bfe_u32 v8, v3, 8, 4
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s9, s10 ; GFX9-NEXT: v_bfe_u32 v13, v4, 16, 4
; GFX9-NEXT: v_pk_lshlrev_b16 v2, 12, s0 op_sel_hi:[0,1] ; GFX9-NEXT: v_bfe_u32 v15, v4, 8, 4
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s5, s8 ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v3
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1] ; GFX9-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s4 ; GFX9-NEXT: v_bfe_u32 v9, v3, 12, 4
; GFX9-NEXT: s_bfe_u32 s7, s6, 0x40018 ; GFX9-NEXT: v_bfe_u32 v3, v3, 4, 4
; GFX9-NEXT: s_lshr_b32 s12, s6, 28 ; GFX9-NEXT: v_and_b32_e32 v10, v2, v10
; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40010 ; GFX9-NEXT: v_bfe_u32 v11, v4, 24, 4
; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40014 ; GFX9-NEXT: v_lshrrev_b32_e32 v12, 28, v4
; GFX9-NEXT: s_bfe_u32 s15, s6, 0x40008 ; GFX9-NEXT: v_bfe_u32 v14, v4, 20, 4
; GFX9-NEXT: s_bfe_u32 s16, s6, 0x4000c ; GFX9-NEXT: v_bfe_u32 v16, v4, 12, 4
; GFX9-NEXT: s_and_b32 s17, s6, 15 ; GFX9-NEXT: v_bfe_u32 v4, v4, 4, 4
; GFX9-NEXT: s_bfe_u32 s6, s6, 0x40004 ; GFX9-NEXT: v_and_b32_e32 v17, v2, v17
; GFX9-NEXT: v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1] ; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v10
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s17, s6 ; GFX9-NEXT: v_lshl_or_b32 v4, v4, 16, v17
; GFX9-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1] ; GFX9-NEXT: v_pk_lshlrev_b16 v3, 12, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1] ; GFX9-NEXT: v_pk_lshlrev_b16 v4, 12, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-NEXT: global_load_ushort v5, v0, s[2:3]
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s15, s16
; GFX9-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s13, s14
; GFX9-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v7, 12, s0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s7, s12
; GFX9-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1] ; GFX9-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX9-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1] ; GFX9-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_mul_lo_u16 v3, v3, v4
; GFX9-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-NEXT: v_and_b32_e32 v8, v2, v8
; GFX9-NEXT: v_and_b32_e32 v0, v2, v0
; GFX9-NEXT: v_and_b32_e32 v15, v2, v15
; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
; GFX9-NEXT: v_lshl_or_b32 v5, v16, 16, v15
; GFX9-NEXT: v_and_b32_e32 v6, v2, v6
; GFX9-NEXT: v_pk_lshlrev_b16 v8, 12, v8 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v5, 12, v5 op_sel_hi:[0,1]
; GFX9-NEXT: v_and_b32_e32 v13, v2, v13
; GFX9-NEXT: v_and_b32_e32 v2, v2, v11
; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-NEXT: v_lshl_or_b32 v7, v14, 16, v13
; GFX9-NEXT: v_lshl_or_b32 v2, v12, 16, v2
; GFX9-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1] ; GFX9-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_mul_lo_u16 v4, v4, v8 ; GFX9-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v6, 12, v6 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 12, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v2, 12, v2 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_mul_lo_u16 v5, v8, v5
; GFX9-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-NEXT: v_pk_mul_lo_u16 v2, v6, v7
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v5, v1, v5 ; GFX9-NEXT: v_add_u16_e32 v4, v3, v4
; GFX9-NEXT: v_add_u32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX9-NEXT: v_add_u16_e32 v3, v3, v5
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NEXT: v_add_u16_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v4 ; GFX9-NEXT: v_add_u16_e32 v2, v2, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot8_acc16_vecMul: ; GFX9-DL-LABEL: idot8_acc16_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[4:5]
; GFX9-DL-NEXT: s_nop 0 ; GFX9-DL-NEXT: global_load_dword v4, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_bfe_u32 s1, s0, 0x40018 ; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v3
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40010 ; GFX9-DL-NEXT: v_and_b32_e32 v17, 15, v4
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v0, v3, 24, 4
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-DL-NEXT: v_bfe_u32 v6, v3, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x4000c ; GFX9-DL-NEXT: v_bfe_u32 v8, v3, 8, 4
; GFX9-DL-NEXT: s_and_b32 s11, s0, 15 ; GFX9-DL-NEXT: v_bfe_u32 v13, v4, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x40004 ; GFX9-DL-NEXT: v_bfe_u32 v15, v4, 8, 4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s11, s0 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v3
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v1, 12, s0 op_sel_hi:[0,1] ; GFX9-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s9, s10 ; GFX9-DL-NEXT: v_bfe_u32 v9, v3, 12, 4
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v2, 12, s0 op_sel_hi:[0,1] ; GFX9-DL-NEXT: v_bfe_u32 v3, v3, 4, 4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s5, s8 ; GFX9-DL-NEXT: v_and_b32_e32 v10, v2, v10
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1] ; GFX9-DL-NEXT: v_bfe_u32 v11, v4, 24, 4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s1, s4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 28, v4
; GFX9-DL-NEXT: s_bfe_u32 s7, s6, 0x40018 ; GFX9-DL-NEXT: v_bfe_u32 v14, v4, 20, 4
; GFX9-DL-NEXT: s_lshr_b32 s12, s6, 28 ; GFX9-DL-NEXT: v_bfe_u32 v16, v4, 12, 4
; GFX9-DL-NEXT: s_bfe_u32 s13, s6, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v4, v4, 4, 4
; GFX9-DL-NEXT: s_bfe_u32 s14, s6, 0x40014 ; GFX9-DL-NEXT: v_and_b32_e32 v17, v2, v17
; GFX9-DL-NEXT: s_bfe_u32 s15, s6, 0x40008 ; GFX9-DL-NEXT: v_lshl_or_b32 v3, v3, 16, v10
; GFX9-DL-NEXT: s_bfe_u32 s16, s6, 0x4000c ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v17
; GFX9-DL-NEXT: s_and_b32 s17, s6, 15 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v3, 12, v3 op_sel_hi:[0,1]
; GFX9-DL-NEXT: s_bfe_u32 s6, s6, 0x40004 ; GFX9-DL-NEXT: v_pk_lshlrev_b16 v4, 12, v4 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s17, s6
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-DL-NEXT: global_load_ushort v5, v0, s[2:3]
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s15, s16
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s13, s14
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v7, 12, s0 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s7, s12
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1] ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1] ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v3, v4
; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-DL-NEXT: v_and_b32_e32 v8, v2, v8
; GFX9-DL-NEXT: v_and_b32_e32 v0, v2, v0
; GFX9-DL-NEXT: v_and_b32_e32 v15, v2, v15
; GFX9-DL-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-DL-NEXT: v_lshl_or_b32 v8, v9, 16, v8
; GFX9-DL-NEXT: v_lshl_or_b32 v5, v16, 16, v15
; GFX9-DL-NEXT: v_and_b32_e32 v6, v2, v6
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v8, 12, v8 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v5, 12, v5 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_and_b32_e32 v13, v2, v13
; GFX9-DL-NEXT: v_and_b32_e32 v2, v2, v11
; GFX9-DL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-DL-NEXT: v_lshl_or_b32 v7, v14, 16, v13
; GFX9-DL-NEXT: v_lshl_or_b32 v2, v12, 16, v2
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1] ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v8 ; GFX9-DL-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v6, 12, v6 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v0, 12, v0 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_lshlrev_b16 v2, 12, v2 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v8, v5
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v6, v7
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u32_e32 v5, v1, v5 ; GFX9-DL-NEXT: v_add_u16_e32 v4, v3, v4
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v5
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v2
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v4 ; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v0
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-XNACK-LABEL: idot8_acc16_vecMul: ; GFX10-DL-XNACK-LABEL: idot8_acc16_vecMul:
; GFX10-DL-XNACK: ; %bb.0: ; %entry ; GFX10-DL-XNACK: ; %bb.0: ; %entry
; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX10-DL-XNACK-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s15, 0x31c16000 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_add_u32 s12, s12, s3 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: global_load_ushort v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s0, s6, 0x40018 ; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s6, 28 ; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s2, s6, 0x40010 ; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s3, s6, 0x40014 ; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s8, s6, 0x40008 ; GFX10-DL-XNACK-NEXT: global_load_ushort v3, v0, s[0:1]
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s9, s6, 0x4000c ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-XNACK-NEXT: s_and_b32 s10, s6, 15 ; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v11, 15, v1
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s6, s6, 0x40004 ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-XNACK-NEXT: s_and_b32 s11, s7, 15 ; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v13, 15, v2
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s6, s10, s6 ; GFX10-DL-XNACK-NEXT: v_bfe_u32 v7, v1, 16, 4
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s10, s7, 0x40004 ; GFX10-DL-XNACK-NEXT: v_bfe_u32 v9, v1, 8, 4
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v2, 12, s6 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_bfe_u32 v5, v1, 24, 4
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s6, s11, s10 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s11, s7, 0x4000c ; GFX10-DL-XNACK-NEXT: v_bfe_u32 v8, v1, 20, 4
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v3, 12, s6 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_bfe_u32 v10, v1, 12, 4
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s6, s7, 0x40008 ; GFX10-DL-XNACK-NEXT: v_bfe_u32 v1, v1, 4, 4
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v11, v4, v11
; GFX10-DL-XNACK-NEXT: v_bfe_u32 v16, v2, 4, 4
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v13, v4, v13
; GFX10-DL-XNACK-NEXT: v_bfe_u32 v18, v2, 8, 4
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v9, v4, v9
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v1, v1, 16, v11
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v7, v4, v7
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v11, v16, 16, v13
; GFX10-DL-XNACK-NEXT: v_bfe_u32 v19, v2, 24, 4
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 28, v2
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v1, 12, v1 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_bfe_u32 v15, v2, 16, 4
; GFX10-DL-XNACK-NEXT: v_bfe_u32 v17, v2, 20, 4
; GFX10-DL-XNACK-NEXT: v_bfe_u32 v2, v2, 12, 4
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v13, v4, v18
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v11, 12, v11 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v9, v10, 16, v9
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v2, v2, 16, v13
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v8, 12, v11 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v9, 12, v9 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v10, v4, v15
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v2, 12, v2 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v1, v1, v8
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v8, v17, 16, v10
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 16, v1
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v3
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, v4, v5
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, v8 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v4, v4, v19
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v2, v9, v2
; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v10
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v3, v6, 16, v3
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v4, v14, 16, v4
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v2
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v2, 12, v3 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v3, 12, v4 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v4, v7, v5
; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v6
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s8, s8, s9
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s6, s6, s11
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v4, 12, s8 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s6 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v7, v1, v4
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s8, s7, 0x40010
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s6, s7, 0x40014
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v3 ; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v3
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v4 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v7, v5
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v5 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s2, s2, s3 ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v2
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s3, s8, s6 ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v1, v1, v3
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s2 op_sel_hi:[0,1] ; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s3 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4
; GFX10-DL-XNACK-NEXT: s_bfe_u32 s10, s7, 0x40018
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s2, s7, 28
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: s_pack_ll_b32_b16 s1, s10, s2
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s1 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v5 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v4
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-XNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v5 op_sel_hi:[0,1]
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX10-DL-XNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v3
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-XNACK-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DL-XNACK-NEXT: s_endpgm ; GFX10-DL-XNACK-NEXT: s_endpgm
; ;
; GFX10-DL-NOXNACK-LABEL: idot8_acc16_vecMul: ; GFX10-DL-NOXNACK-LABEL: idot8_acc16_vecMul:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry ; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s15, 0x31c16000 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s12, s12, s3 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NOXNACK-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s2, s0, 0x40018 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s0, 28 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s6, s0, 0x40010 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s7, s0, 0x40014 ; GFX10-DL-NOXNACK-NEXT: global_load_ushort v3, v2, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s8, s0, 0x40008 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s9, s0, 0x4000c ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v11, 15, v1
; GFX10-DL-NOXNACK-NEXT: s_and_b32 s10, s0, 15 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s0, s0, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v13, 15, v0
; GFX10-DL-NOXNACK-NEXT: s_and_b32 s11, s1, 15 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v7, v1, 16, 4
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s10, s0 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v9, v1, 8, 4
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s10, s1, 0x40004 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v5, v1, 24, 4
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v2, 12, s0 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s11, s10 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v8, v1, 20, 4
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s11, s1, 0x4000c ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v10, v1, 12, 4
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v1, v1, 4, 4
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s0, s1, 0x40008 ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v11, v4, v11
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v2 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v16, v0, 4, 4
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s8, s8, s9 ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v13, v4, v13
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s0, s11 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v18, v0, 8, 4
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v9, v4, v9
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v4, 12, s8 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v1, v1, 16, v11
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v7, v4, v7
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s8, s1, 0x40010 ; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v11, v16, 16, v13
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s0, s1, 0x40014 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v19, v0, 24, 4
; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v3 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 28, v0
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v4 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v1, 12, v1 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v5 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v15, v0, 16, 4
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s6, s6, s7 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v17, v0, 20, 4
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s8, s0 ; GFX10-DL-NOXNACK-NEXT: v_bfe_u32 v0, v0, 12, 4
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s6 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v13, v4, v18
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v11, 12, v11 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4 ; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v9, v10, 16, v9
; GFX10-DL-NOXNACK-NEXT: s_bfe_u32 s10, s1, 0x40018 ; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s0, s1, 28 ; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s1, s2, s3 ; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v0, v0, 16, v13
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v8, 12, v11 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: s_pack_ll_b32_b16 s0, s10, s0 ; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v9, 12, v9 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v10, v4, v15
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v7, 12, v7 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v0, 12, v0 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v1, v1, v8
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v8, v17, 16, v10
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v0, 12, v0 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 16, v1
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v1, v1, v3
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, v4, v5
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v2, 12, v5 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, v8 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v5, 12, s1 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v4, v4, v19
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v0, v9, v0
; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v2, v2, v4 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v1, v1, v10
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v4, 12, v6 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v3, v6, 16, v3
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v5 op_sel_hi:[0,1] ; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v4, v14, 16, v4
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v3, v3, v4 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v1, v0
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v1, 12, v3 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-DL-NOXNACK-NEXT: v_pk_lshlrev_b16 v3, 12, v4 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v4, v7, v5
; GFX10-DL-NOXNACK-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v7, v0, v6
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v1, 12, v1 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v7, v4
; GFX10-DL-NOXNACK-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v5
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v3
; GFX10-DL-NOXNACK-NEXT: global_store_short v2, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_endpgm ; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc16_vecMul: ; GFX10-DL-LABEL: idot8_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-NEXT: s_mov_b32 s14, -1
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v3
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = sext <8 x i4> %vec1 to <8 x i16> %cvec1 = sext <8 x i4> %vec1 to <8 x i16>
%cvec2 = sext <8 x i4> %vec2 to <8 x i16> %cvec2 = sext <8 x i4> %vec2 to <8 x i16>
%mul = mul <8 x i16> %cvec1, %cvec2 %mul = mul <8 x i16> %cvec1, %cvec2
%mul0 = extractelement <8 x i16> %mul, i64 0 %mul0 = extractelement <8 x i16> %mul, i64 0
%mul1 = extractelement <8 x i16> %mul, i64 1 %mul1 = extractelement <8 x i16> %mul, i64 1
%mul2 = extractelement <8 x i16> %mul, i64 2 %mul2 = extractelement <8 x i16> %mul, i64 2
Show All 16 Linesentry:
store i16 %add8, i16 addrspace(1)* %dst, align 4 store i16 %add8, i16 addrspace(1)* %dst, align 4
ret void ret void
} }
; TODO: Support this pattern. ; TODO: Support this pattern.
define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: idot8_acc8_vecMul: ; GFX7-LABEL: idot8_acc8_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_addc_u32 s25, s25, 0
; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_mov_b32 s9, 0xffff
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_bfe_i32 s6, s4, 0x40000 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_bfe_i32 s15, s5, 0x40000 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_bfe_i32 s16, s5, 0x40004 ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_bfe_i32 s17, s5, 0x40008 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_i32 s18, s5, 0x4000c ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_bfe_i32 s19, s5, 0x40010 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_i32 s20, s5, 0x40014 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_i32 s21, s5, 0x40018 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_ashr_i32 s5, s5, 28 ; GFX7-NEXT: s_movk_i32 s4, 0xff
; GFX7-NEXT: v_mov_b32_e32 v8, s15 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_i32 s7, s4, 0x40004 ; GFX7-NEXT: v_mov_b32_e32 v2, 0xff
; GFX7-NEXT: v_mov_b32_e32 v7, s16 ; GFX7-NEXT: s_mov_b32 s5, 0xffff
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x40008 ; GFX7-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX7-NEXT: v_mov_b32_e32 v6, s17 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x4000c ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mov_b32_e32 v5, s18 ; GFX7-NEXT: v_bfe_i32 v6, v4, 20, 4
; GFX7-NEXT: s_bfe_i32 s12, s4, 0x40010 ; GFX7-NEXT: v_bfe_i32 v7, v4, 16, 4
; GFX7-NEXT: v_mov_b32_e32 v4, s19 ; GFX7-NEXT: v_bfe_i32 v8, v4, 12, 4
; GFX7-NEXT: s_bfe_i32 s13, s4, 0x40014 ; GFX7-NEXT: v_bfe_i32 v9, v4, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v3, s20 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6
; GFX7-NEXT: s_bfe_i32 s14, s4, 0x40018 ; GFX7-NEXT: v_and_b32_e32 v7, s4, v7
; GFX7-NEXT: v_mov_b32_e32 v2, s21 ; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX7-NEXT: s_ashr_i32 s4, s4, 28 ; GFX7-NEXT: v_and_b32_e32 v9, s4, v9
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mul_i32_i24_e32 v1, s4, v1
; GFX7-NEXT: v_mul_i32_i24_e32 v2, s14, v2
; GFX7-NEXT: v_mul_i32_i24_e32 v3, s13, v3
; GFX7-NEXT: v_mul_i32_i24_e32 v9, s12, v4
; GFX7-NEXT: v_mul_i32_i24_e32 v5, s11, v5
; GFX7-NEXT: v_mul_i32_i24_e32 v6, s10, v6
; GFX7-NEXT: v_mul_i32_i24_e32 v7, s7, v7
; GFX7-NEXT: v_mul_i32_i24_e32 v8, s6, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; GFX7-NEXT: v_and_b32_e32 v2, s8, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX7-NEXT: v_and_b32_e32 v9, s8, v9
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX7-NEXT: v_and_b32_e32 v6, s8, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; GFX7-NEXT: v_and_b32_e32 v8, s8, v8
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
; GFX7-NEXT: v_or_b32_e32 v2, v9, v3
; GFX7-NEXT: v_or_b32_e32 v3, v6, v5
; GFX7-NEXT: v_or_b32_e32 v5, v8, v7
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_and_b32_e32 v2, s9, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_and_b32_e32 v5, s9, v5
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
; GFX7-NEXT: v_or_b32_e32 v2, v5, v3
; GFX7-NEXT: v_alignbit_b32 v3, v1, v2, 8
; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16
; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GFX7-NEXT: v_bfe_i32 v13, v0, 24, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0 ; GFX7-NEXT: v_bfe_i32 v16, v0, 12, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0 ; GFX7-NEXT: v_or_b32_e32 v6, v7, v6
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0 ; GFX7-NEXT: v_or_b32_e32 v7, v9, v8
; GFX7-NEXT: v_mad_i32_i24 v0, s12, v4, v0 ; GFX7-NEXT: v_and_b32_e32 v9, v2, v13
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v7 ; GFX7-NEXT: v_lshlrev_b32_e32 v13, 8, v16
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v8 ; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ; GFX7-NEXT: v_bfe_i32 v5, v4, 24, 4
; GFX7-NEXT: v_bfe_i32 v10, v4, 4, 4
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 28, v4
; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 4
; GFX7-NEXT: v_lshlrev_b32_e32 v11, 8, v1
; GFX7-NEXT: v_and_b32_e32 v5, s4, v5
; GFX7-NEXT: v_ashrrev_i32_e32 v12, 28, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v10, 8, v10
; GFX7-NEXT: v_and_b32_e32 v4, v2, v4
; GFX7-NEXT: v_bfe_i32 v14, v0, 20, 4
; GFX7-NEXT: v_bfe_i32 v15, v0, 16, 4
; GFX7-NEXT: v_bfe_i32 v17, v0, 8, 4
; GFX7-NEXT: v_bfe_i32 v18, v0, 4, 4
; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 4
; GFX7-NEXT: v_or_b32_e32 v5, v5, v11
; GFX7-NEXT: v_or_b32_e32 v4, v4, v10
; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v12
; GFX7-NEXT: v_lshlrev_b32_e32 v10, 8, v14
; GFX7-NEXT: v_and_b32_e32 v11, v2, v15
; GFX7-NEXT: v_and_b32_e32 v14, v2, v17
; GFX7-NEXT: v_lshlrev_b32_e32 v15, 8, v18
; GFX7-NEXT: v_and_b32_e32 v0, v2, v0
; GFX7-NEXT: v_or_b32_e32 v0, v0, v15
; GFX7-NEXT: v_or_b32_e32 v8, v9, v8
; GFX7-NEXT: v_or_b32_e32 v9, v11, v10
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_and_b32_e32 v6, s5, v6
; GFX7-NEXT: v_or_b32_e32 v10, v14, v13
; GFX7-NEXT: v_or_b32_e32 v5, v6, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
; GFX7-NEXT: v_and_b32_e32 v0, v3, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX7-NEXT: v_and_b32_e32 v4, s5, v4
; GFX7-NEXT: v_or_b32_e32 v0, v0, v8
; GFX7-NEXT: v_or_b32_e32 v4, v4, v7
; GFX7-NEXT: v_and_b32_e32 v7, v3, v9
; GFX7-NEXT: v_or_b32_e32 v3, v7, v6
; GFX7-NEXT: v_and_b32_e32 v7, v2, v4
; GFX7-NEXT: v_and_b32_e32 v13, v2, v0
; GFX7-NEXT: v_bfe_u32 v8, v4, 8, 8
; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 8
; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v11, 24, v0
; GFX7-NEXT: v_bfe_u32 v4, v4, 16, 8
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX7-NEXT: v_and_b32_e32 v1, v2, v1
; GFX7-NEXT: v_and_b32_e32 v12, v2, v12
; GFX7-NEXT: v_and_b32_e32 v9, v2, v5
; GFX7-NEXT: v_and_b32_e32 v2, v2, v3
; GFX7-NEXT: v_bfe_u32 v10, v5, 8, 8
; GFX7-NEXT: v_bfe_u32 v15, v3, 8, 8
; GFX7-NEXT: v_bfe_u32 v5, v5, 16, 8
; GFX7-NEXT: v_bfe_u32 v3, v3, 16, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v7, v7, v13, v16
; GFX7-NEXT: v_mad_u32_u24 v7, v8, v14, v7
; GFX7-NEXT: v_mad_u32_u24 v0, v4, v0, v7
; GFX7-NEXT: v_mad_u32_u24 v0, v6, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v9, v2, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v10, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v5, v3, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v1, v12, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: idot8_acc8_vecMul: ; GFX8-LABEL: idot8_acc8_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: v_mov_b32_e32 v5, 12
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s23, 0xe80000
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_add_u32 s20, s20, s3 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_mov_b32 s0, 0xffff ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_bfe_i32 s7, s1, 0x40004 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 28, v3
; GFX8-NEXT: s_bfe_i32 s9, s1, 0x4000c ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 12, v3
; GFX8-NEXT: s_bfe_i32 s14, s2, 0x40004 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
; GFX8-NEXT: s_bfe_i32 s15, s2, 0x40000 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 20, v3
; GFX8-NEXT: s_bfe_i32 s16, s2, 0x4000c ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_bfe_i32 s3, s1, 0x40014 ; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v2
; GFX8-NEXT: s_ashr_i32 s5, s1, 28 ; GFX8-NEXT: v_lshrrev_b32_e32 v13, 12, v2
; GFX8-NEXT: s_bfe_i32 s10, s2, 0x40014 ; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v2
; GFX8-NEXT: s_bfe_i32 s11, s2, 0x40010 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 4, v3
; GFX8-NEXT: s_ashr_i32 s12, s2, 28 ; GFX8-NEXT: v_lshrrev_b32_e32 v11, 20, v2
; GFX8-NEXT: s_bfe_i32 s13, s2, 0x40018 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 4, v2
; GFX8-NEXT: s_bfe_i32 s2, s2, 0x40008 ; GFX8-NEXT: v_lshlrev_b16_e32 v16, 12, v3
; GFX8-NEXT: s_bfe_i32 s8, s1, 0x40000 ; GFX8-NEXT: v_lshlrev_b16_sdwa v17, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: v_mov_b32_e32 v4, s16 ; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_mov_b32_e32 v5, s9 ; GFX8-NEXT: v_lshlrev_b16_e32 v18, 12, v2
; GFX8-NEXT: v_mov_b32_e32 v6, s15 ; GFX8-NEXT: v_lshlrev_b16_sdwa v19, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: v_mov_b32_e32 v7, s14 ; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_mov_b32_e32 v8, s7 ; GFX8-NEXT: v_lshlrev_b16_e32 v5, 12, v10
; GFX8-NEXT: v_mul_i32_i24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX8-NEXT: v_mul_i32_i24_e32 v5, s8, v6 ; GFX8-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX8-NEXT: v_mul_i32_i24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_ashrrev_i16_e32 v10, 12, v16
; GFX8-NEXT: s_bfe_i32 s4, s1, 0x40010 ; GFX8-NEXT: v_ashrrev_i16_e32 v16, 12, v17
; GFX8-NEXT: s_bfe_i32 s6, s1, 0x40018 ; GFX8-NEXT: v_ashrrev_i16_e32 v17, 12, v3
; GFX8-NEXT: v_mov_b32_e32 v9, s13 ; GFX8-NEXT: v_lshlrev_b16_e32 v3, 12, v6
; GFX8-NEXT: s_bfe_i32 s1, s1, 0x40008 ; GFX8-NEXT: v_lshlrev_b16_e32 v6, 12, v15
; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_ashrrev_i16_e32 v15, 12, v18
; GFX8-NEXT: v_mov_b32_e32 v10, s12 ; GFX8-NEXT: v_ashrrev_i16_e32 v18, 12, v19
; GFX8-NEXT: v_mov_b32_e32 v11, s5 ; GFX8-NEXT: v_ashrrev_i16_e32 v19, 12, v2
; GFX8-NEXT: v_mov_b32_e32 v12, s11 ; GFX8-NEXT: v_lshlrev_b16_e32 v2, 12, v11
; GFX8-NEXT: v_mov_b32_e32 v13, s10 ; GFX8-NEXT: v_lshlrev_b16_e32 v9, 12, v9
; GFX8-NEXT: v_mov_b32_e32 v14, s3 ; GFX8-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX8-NEXT: v_mul_i32_i24_e32 v3, s1, v3 ; GFX8-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX8-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX8-NEXT: v_lshlrev_b16_e32 v14, 12, v14
; GFX8-NEXT: v_mul_i32_i24_e32 v7, s6, v9 ; GFX8-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX8-NEXT: v_mul_i32_i24_sdwa v8, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_ashrrev_i16_e32 v3, 12, v3
; GFX8-NEXT: v_mul_i32_i24_e32 v9, s4, v12 ; GFX8-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX8-NEXT: v_mul_i32_i24_sdwa v10, v14, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX8-NEXT: v_and_b32_e32 v5, s0, v5 ; GFX8-NEXT: v_ashrrev_i16_e32 v11, 12, v14
; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX8-NEXT: v_mul_lo_u16_e32 v10, v10, v15
; GFX8-NEXT: v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX8-NEXT: v_mul_lo_u16_sdwa v2, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX8-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX8-NEXT: v_and_b32_e32 v4, s0, v9 ; GFX8-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX8-NEXT: v_or_b32_e32 v3, v5, v3 ; GFX8-NEXT: v_ashrrev_i16_e32 v13, 12, v13
; GFX8-NEXT: v_or_b32_e32 v6, v4, v7 ; GFX8-NEXT: v_mul_lo_u16_sdwa v3, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3 ; GFX8-NEXT: v_mul_lo_u16_e32 v15, v16, v18
; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v6 ; GFX8-NEXT: v_mul_lo_u16_sdwa v7, v8, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_mul_lo_u16_e32 v8, v9, v11
; GFX8-NEXT: v_or_b32_sdwa v3, v15, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX8-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX8-NEXT: v_mul_lo_u16_e32 v14, v17, v19
; GFX8-NEXT: v_mul_lo_u16_sdwa v5, v5, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: v_or_b32_sdwa v6, v14, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3
; GFX8-NEXT: v_or_b32_sdwa v8, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX8-NEXT: v_lshrrev_b32_e32 v10, 8, v3
; GFX8-NEXT: v_or_b32_sdwa v3, v6, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_or_b32_e32 v5, v5, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v3
; GFX8-NEXT: v_lshrrev_b64 v[2:3], 24, v[2:3]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v5
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5 ; GFX8-NEXT: v_add_u16_e32 v3, v8, v4
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v7, v2 ; GFX8-NEXT: v_add_u16_e32 v3, v3, v5
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0 ; GFX8-NEXT: v_add_u16_e32 v3, v3, v7
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD ; GFX8-NEXT: v_add_u16_e32 v2, v3, v2
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; GFX8-NEXT: v_mad_u16 v2, v17, v19, v2
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v8, v2 ; GFX8-NEXT: v_add_u16_e32 v2, v2, v6
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD ; GFX8-NEXT: v_mad_u16 v2, v16, v18, v2
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD ; GFX8-NEXT: v_add_u16_e32 v2, v2, v10
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: idot8_acc8_vecMul: ; GFX9-LABEL: idot8_acc8_vecMul:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b32 s0, 0xffff ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s9, s1, 4 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_lshr_b32 s16, s8, 4 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: v_lshlrev_b16_e64 v2, 12, s1 ; GFX9-NEXT: global_load_ubyte v4, v3, s[2:3]
; GFX9-NEXT: v_lshlrev_b16_e64 v3, 12, s8 ; GFX9-NEXT: v_mov_b32_e32 v0, 12
; GFX9-NEXT: v_lshlrev_b16_e64 v6, 12, s9 ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e64 v13, 12, s16 ; GFX9-NEXT: v_lshrrev_b32_e32 v6, 28, v1
; GFX9-NEXT: s_lshr_b32 s10, s1, 12 ; GFX9-NEXT: v_lshrrev_b32_e32 v7, 12, v1
; GFX9-NEXT: s_lshr_b32 s11, s1, 8 ; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
; GFX9-NEXT: s_lshr_b32 s17, s8, 12 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_lshr_b32 s18, s8, 8 ; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX9-NEXT: v_lshlrev_b16_e64 v4, 12, s11 ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 20, v1
; GFX9-NEXT: v_lshlrev_b16_e64 v5, 12, s10 ; GFX9-NEXT: v_lshrrev_b32_e32 v9, 4, v1
; GFX9-NEXT: v_lshlrev_b16_e64 v11, 12, s18 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 20, v2
; GFX9-NEXT: v_lshlrev_b16_e64 v12, 12, s17 ; GFX9-NEXT: v_lshrrev_b32_e32 v14, 4, v2
; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2 ; GFX9-NEXT: v_lshlrev_b16_e32 v15, 12, v1
; GFX9-NEXT: v_ashrrev_i16_e32 v3, 12, v3 ; GFX9-NEXT: v_lshlrev_b16_sdwa v16, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshlrev_b16_sdwa v18, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: v_lshlrev_b16_e32 v17, 12, v2
; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v12, 12, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v2, 12, v9
; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v15
; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 12, v5
; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v14
; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v17
; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-NEXT: v_ashrrev_i16_e32 v18, 12, v0
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 12, v10
; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6 ; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX9-NEXT: v_ashrrev_i16_e32 v13, 12, v13 ; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX9-NEXT: v_ashrrev_i16_e32 v0, 12, v0
; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11 ; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5 ; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v13
; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12 ; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v1, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v2, v2, v3
; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s1, 20
; GFX9-NEXT: s_lshr_b32 s5, s1, 16
; GFX9-NEXT: s_lshr_b32 s12, s8, 20
; GFX9-NEXT: s_lshr_b32 s13, s8, 16
; GFX9-NEXT: v_mul_lo_u16_sdwa v5, v5, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v4, v4, v11
; GFX9-NEXT: v_lshlrev_b16_e64 v9, 12, s5
; GFX9-NEXT: v_lshlrev_b16_e64 v10, 12, s4
; GFX9-NEXT: v_lshlrev_b16_e64 v16, 12, s13
; GFX9-NEXT: v_lshlrev_b16_e64 v17, 12, s12
; GFX9-NEXT: s_lshr_b32 s6, s1, 28
; GFX9-NEXT: s_lshr_b32 s7, s1, 24
; GFX9-NEXT: s_lshr_b32 s14, s8, 28
; GFX9-NEXT: s_lshr_b32 s15, s8, 24
; GFX9-NEXT: v_and_b32_e32 v2, s0, v2
; GFX9-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b16_e64 v7, 12, s7
; GFX9-NEXT: v_lshlrev_b16_e64 v8, 12, s6
; GFX9-NEXT: v_lshlrev_b16_e64 v14, 12, s15
; GFX9-NEXT: v_lshlrev_b16_e64 v15, 12, s14
; GFX9-NEXT: v_or_b32_e32 v4, v2, v4
; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v16
; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v17
; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8 ; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v15 ; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-NEXT: v_mul_lo_u16_sdwa v3, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v16 ; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_lshrrev_b32_e32 v6, 8, v4 ; GFX9-NEXT: v_mul_lo_u16_e32 v19, v15, v17
; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_mul_lo_u16_e32 v7, v8, v10
; GFX9-NEXT: v_mul_lo_u16_e32 v7, v7, v14 ; GFX9-NEXT: v_or_b32_sdwa v1, v19, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-NEXT: v_and_b32_e32 v3, s0, v3 ; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-NEXT: v_or_b32_e32 v5, v3, v7 ; GFX9-NEXT: v_mul_lo_u16_e32 v13, v16, v18
; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v2, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v5, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX9-NEXT: v_or_b32_sdwa v7, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v1, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, v2, v1 ; GFX9-NEXT: v_add_u16_e32 v1, v7, v4
; GFX9-NEXT: v_add_u32_e32 v1, v1, v6 ; GFX9-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 ; GFX9-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NEXT: v_mad_legacy_u16 v0, v16, v18, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v5 ; GFX9-NEXT: v_add_u16_e32 v0, v0, v5
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-NEXT: v_mad_legacy_u16 v0, v15, v17, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_e32 v0, v0, v9
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: idot8_acc8_vecMul: ; GFX9-DL-LABEL: idot8_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DL-NEXT: s_mov_b32 s0, 0xffff ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_lshr_b32 s9, s1, 4 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_lshr_b32 s16, s8, 4 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v2, 12, s1 ; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[2:3]
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v3, 12, s8 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 12
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v6, 12, s9 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v13, 12, s16 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 28, v1
; GFX9-DL-NEXT: s_lshr_b32 s10, s1, 12 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 12, v1
; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 8 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
; GFX9-DL-NEXT: s_lshr_b32 s17, s8, 12 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_lshr_b32 s18, s8, 8 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v4, 12, s11 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 20, v1
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v5, 12, s10 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 4, v1
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v11, 12, s18 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 20, v2
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v12, 12, s17 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 4, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2 ; GFX9-DL-NEXT: v_lshlrev_b16_e32 v15, 12, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v3, 12, v3 ; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v16, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v18, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v17, 12, v2
; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 12, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v13, 8, v2
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v2, 12, v9
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v15
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v1
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v1, 12, v5
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v14
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v17
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v18, 12, v0
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v0, 12, v10
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v13, 12, v13 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v0, 12, v0
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v13
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v0, v1, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v2, v2, v3
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: s_lshr_b32 s4, s1, 20
; GFX9-DL-NEXT: s_lshr_b32 s5, s1, 16
; GFX9-DL-NEXT: s_lshr_b32 s12, s8, 20
; GFX9-DL-NEXT: s_lshr_b32 s13, s8, 16
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v5, v5, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v4, v4, v11
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v9, 12, s5
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v10, 12, s4
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v16, 12, s13
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v17, 12, s12
; GFX9-DL-NEXT: s_lshr_b32 s6, s1, 28
; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 24
; GFX9-DL-NEXT: s_lshr_b32 s14, s8, 28
; GFX9-DL-NEXT: s_lshr_b32 s15, s8, 24
; GFX9-DL-NEXT: v_and_b32_e32 v2, s0, v2
; GFX9-DL-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v7, 12, s7
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v8, 12, s6
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v14, 12, s15
; GFX9-DL-NEXT: v_lshlrev_b16_e64 v15, 12, s14
; GFX9-DL-NEXT: v_or_b32_e32 v4, v2, v4
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v9
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v16
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v10
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v17
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v14
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v15 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v3, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v16 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v4 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v19, v15, v17
; GFX9-DL-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v8, v10
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v7, v14 ; GFX9-DL-NEXT: v_or_b32_sdwa v1, v19, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-DL-NEXT: v_and_b32_e32 v3, s0, v3 ; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-DL-NEXT: v_or_b32_e32 v5, v3, v7 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v13, v16, v18
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v2, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-DL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_sdwa v5, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX9-DL-NEXT: v_or_b32_sdwa v7, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX9-DL-NEXT: v_or_b32_sdwa v1, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_e32 v2, v2, v0
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v7, v4
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v6 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v16, v18, v0
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v5 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v5
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v15, v17, v0
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v9
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-DL-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-XNACK-LABEL: idot8_acc8_vecMul: ; GFX10-DL-XNACK-LABEL: idot8_acc8_vecMul:
; GFX10-DL-XNACK: ; %bb.0: ; %entry ; GFX10-DL-XNACK: ; %bb.0: ; %entry
; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX10-DL-XNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-XNACK-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX10-DL-XNACK-NEXT: v_mov_b32_e32 v19, 0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s22, -1 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s23, 0x31c16000 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-XNACK-NEXT: s_add_u32 s20, s20, s3 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-XNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-XNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-XNACK-NEXT: s_addc_u32 s21, s21, 0 ; GFX10-DL-XNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-XNACK-NEXT: global_load_ubyte v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-XNACK-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-XNACK-NEXT: s_mov_b32 s0, 0xffff
; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s9, s6, 4 ; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s16, s7, 4 ; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s9 ; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s16 ; GFX10-DL-XNACK-NEXT: global_load_ubyte v3, v19, s[0:1]
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s10, s6, 12 ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s17, s7, 12 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s6 ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s7 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v13, 12, s17 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, s10 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v15, 12, v15
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 28, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v15
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v0, 20, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 24, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 20, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 24, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v13, 12, v13
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v8, v8, v15
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v17
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v16, 12, v16
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 16, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v14, 12, v14
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v11, 12, v11
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v12, 12, v12
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s11, s6, 8
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s18, s7, 8
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v4, 12, s11
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v11, 12, s18
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v19, 12, v5
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v12
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v13, 12, v13 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v13, 12, v13
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 12, v2
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v17
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v9, v9, v16
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 8, v8
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v13
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v12, 12, v12
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v14, 12, v14
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v11 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v11
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v2, v2, v3 ; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v10, v10, v15
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v6 ; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v3, v19, v13 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v1
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s1, s6, 20 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s2, s6, 16 ; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v9, v0, v11
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s3, s6, 28 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s8, s6, 24 ; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v23, 12, v12
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s12, s7, 20 ; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v11, v7, v14
; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 8, v6
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v7, 12, s8 ; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v1, v1, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v8, 12, s3 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 8, v10
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 12, s2 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v10, 12, s1 ; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v2, v5, v23
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s12 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v9, 8, v9
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v11 ; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v6, v11, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v3, 8, v3 ; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s13, s7, 16 ; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v11, v10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s14, s7, 28 ; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s13 ; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v9, 16, v6
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v7
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v8
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v9
; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v15, 12, s14
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v10
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v12
; GFX10-DL-XNACK-NEXT: s_lshr_b32 s15, s7, 24
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v15
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v14, 12, s15
; GFX10-DL-XNACK-NEXT: v_or_b32_e32 v3, v2, v3
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v9
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v15, v8, v6
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v7, v7, v10
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v14
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 8, v3
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1 ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v3, v1, v3
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v2, 8, v4 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 8, v11
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16_e64 v4, v5, v11 ; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v7 ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v9, v3, v10
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v8 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v15, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v9, v8
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v0, v2
; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v2, s0, v2 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v0, v5, v23, v0
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
; GFX10-DL-XNACK-NEXT: v_or_b32_e32 v3, v2, v4 ; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v6
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-XNACK-NEXT: v_mad_u16 v0, v7, v14, v0
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v3 ; GFX10-DL-XNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-XNACK-NEXT: global_store_byte v19, v0, s[0:1]
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-XNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX10-DL-XNACK-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-XNACK-NEXT: s_endpgm ; GFX10-DL-XNACK-NEXT: s_endpgm
; ;
; GFX10-DL-NOXNACK-LABEL: idot8_acc8_vecMul: ; GFX10-DL-NOXNACK-LABEL: idot8_acc8_vecMul:
; GFX10-DL-NOXNACK: ; %bb.0: ; %entry ; GFX10-DL-NOXNACK: ; %bb.0: ; %entry
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NOXNACK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NOXNACK-NEXT: v_mov_b32_e32 v19, 0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s22, -1 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s23, 0x31c16000 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NOXNACK-NEXT: s_add_u32 s20, s20, s3 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NOXNACK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s21, s21, 0 ; GFX10-DL-NOXNACK-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_load_dword s0, s[0:1], 0x0
; GFX10-DL-NOXNACK-NEXT: s_load_dword s1, s[2:3], 0x0
; GFX10-DL-NOXNACK-NEXT: s_mov_b32 s2, 0xffff
; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s9, s0, 4 ; GFX10-DL-NOXNACK-NEXT: s_clause 0x1
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s16, s1, 4 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s9 ; GFX10-DL-NOXNACK-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s16 ; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v2, v19, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s10, s0, 12 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s17, s1, 12 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 12, s0 ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, s1 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v13, 12, s17 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, s10 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 20, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 16, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 28, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 24, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, v8
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v15, 12, v15
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 28, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v8
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v18, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v0, 12, v16
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v15
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 20, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 24, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v8, v8, v15
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v17, 12, v17
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v0, 12, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 12, v3
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v14, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v11, 12, v11
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v12, 12, v12
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s11, s0, 8
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s18, s1, 8
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v4, 12, s11
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v11, 12, s18
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v2, 12, v2
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v19, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v12
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v13, 12, v13 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v4 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v11 ; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v23, v9, v0
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v2, v2, v3 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 8, v8
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v6 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v3, v19, v13 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v15, 12, v17
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s3, s0, 20 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 12, v5
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s6, s0, 16 ; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v6, v6, v13
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s7, s0, 28 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v12, 12, v12
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s8, s0, 24 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v11
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s12, s1, 20 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v3, 12, v3
; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v7, 12, s8 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v14, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v8, 12, s7 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v9, 12, s6 ; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v10, v10, v15
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 12, s3 ; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v12, 12, s12 ; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v3, v3, v9
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v11 ; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v5
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v12
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v9, v7, v14
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 8, v6
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v10, 8, v10
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v1, v1, v18
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v12, v5, v11
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 8, v3 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v3, 8, v3
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s13, s1, 16 ; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v6, v9, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s14, s1, 28 ; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v6, 12, s13 ; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v9, v10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v5, 12, v7 ; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v12, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v7, 12, v8 ; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v8, 12, v9
; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v15, 12, s14
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v4, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v9, 12, v12
; GFX10-DL-NOXNACK-NEXT: s_lshr_b32 s15, s1, 24
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v10, 12, v15
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v14, 12, s15
; GFX10-DL-NOXNACK-NEXT: v_or_b32_e32 v3, v2, v3
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v4, v4, v9
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v15, v8, v6
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v7, v7, v10
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16_e64 v11, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 8, v3
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v2, v1 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v2, v1, v2
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v2, 8, v4 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v9
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16_e64 v4, v5, v11 ; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v3, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16_e64 v5, 8, v7 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v9, v2, v9
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v8 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v2, v15, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v9, v8
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v2
; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v2, s2, v2 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v5, v11, v0
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
; GFX10-DL-NOXNACK-NEXT: v_or_b32_e32 v3, v2, v4 ; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v6
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NOXNACK-NEXT: v_mad_u16 v0, v7, v14, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v2, 8, v3 ; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16_e64 v0, v0, v1
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NOXNACK-NEXT: global_store_byte v19, v0, s[0:1]
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX10-DL-NOXNACK-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NOXNACK-NEXT: s_endpgm ; GFX10-DL-NOXNACK-NEXT: s_endpgm
; GFX10-DL-LABEL: idot8_acc8_vecMul: ; GFX10-DL-LABEL: idot8_acc8_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s22, -1 ; GFX10-DL-NEXT: s_mov_b32 s22, -1
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = sext <8 x i4> %vec1 to <8 x i8> %cvec1 = sext <8 x i4> %vec1 to <8 x i8>
%cvec2 = sext <8 x i4> %vec2 to <8 x i8> %cvec2 = sext <8 x i4> %vec2 to <8 x i8>
%mul = mul <8 x i8> %cvec1, %cvec2 %mul = mul <8 x i8> %cvec1, %cvec2
%mul0 = extractelement <8 x i8> %mul, i64 0 %mul0 = extractelement <8 x i8> %mul, i64 0
%mul1 = extractelement <8 x i8> %mul, i64 1 %mul1 = extractelement <8 x i8> %mul, i64 1
%mul2 = extractelement <8 x i8> %mul, i64 2 %mul2 = extractelement <8 x i8> %mul, i64 2
Show All 11 Linesentry:
%add5 = add i8 %add4, %mul4 %add5 = add i8 %add4, %mul4
%add6 = add i8 %add5, %mul5 %add6 = add i8 %add5, %mul5
%add7 = add i8 %add6, %mul6 %add7 = add i8 %add6, %mul6
%add8 = add i8 %add7, %mul7 %add8 = add i8 %add7, %mul7
store i8 %add8, i8 addrspace(1)* %dst, align 4 store i8 %add8, i8 addrspace(1)* %dst, align 4
ret void ret void
} }
declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/idot8u.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc32: ; GFX7-LABEL: udot8_acc32:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s7, s6, 28
; GFX7-NEXT: s_bfe_u32 s14, s6, 0x40018
; GFX7-NEXT: s_bfe_u32 s15, s6, 0x40014
; GFX7-NEXT: s_bfe_u32 s16, s6, 0x40010
; GFX7-NEXT: s_bfe_u32 s17, s6, 0x4000c
; GFX7-NEXT: s_bfe_u32 s18, s6, 0x40008
; GFX7-NEXT: s_bfe_u32 s19, s6, 0x40004
; GFX7-NEXT: s_and_b32 s6, s6, 15
; GFX7-NEXT: s_lshr_b32 s5, s4, 28
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40018
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40014
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40010
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x4000c
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40008
; GFX7-NEXT: s_bfe_u32 s13, s4, 0x40004
; GFX7-NEXT: s_and_b32 s4, s4, 15
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s19
; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s18
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s17
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s16
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s15
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s14
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s4
; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc32: ; GFX8-LABEL: udot8_acc32:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_mov_b32 s23, 0xe80000 ; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: s_add_u32 s20, s20, s3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_lshr_b32 s7, s6, 28 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_bfe_u32 s12, s6, 0x40018 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_bfe_u32 s13, s6, 0x40014 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_bfe_u32 s14, s6, 0x40010 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_bfe_u32 s15, s6, 0x4000c ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_bfe_u32 s16, s6, 0x40008 ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_bfe_u32 s17, s6, 0x40004 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_and_b32 s6, s6, 15 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 28, v0
; GFX8-NEXT: s_lshr_b32 s3, s2, 28 ; GFX8-NEXT: v_bfe_u32 v3, v0, 24, 4
; GFX8-NEXT: s_bfe_u32 s4, s2, 0x40018 ; GFX8-NEXT: v_bfe_u32 v4, v0, 20, 4
; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40014 ; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 4
; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010 ; GFX8-NEXT: v_bfe_u32 v6, v0, 12, 4
; GFX8-NEXT: s_bfe_u32 s9, s2, 0x4000c ; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40008 ; GFX8-NEXT: v_bfe_u32 v8, v0, 4, 4
; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40004 ; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_and_b32 s2, s2, 15 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s18 ; GFX8-NEXT: v_bfe_u32 v10, v1, 24, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s2, v0, v1 ; GFX8-NEXT: v_bfe_u32 v11, v1, 20, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s17 ; GFX8-NEXT: v_bfe_u32 v12, v1, 16, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s11, v1, v0 ; GFX8-NEXT: v_bfe_u32 v13, v1, 12, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s16 ; GFX8-NEXT: v_bfe_u32 v14, v1, 8, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX8-NEXT: v_bfe_u32 v15, v1, 4, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s15 ; GFX8-NEXT: v_and_b32_e32 v1, 15, v1
; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s14 ; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, s2
; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s13 ; GFX8-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s12 ; GFX8-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v2, v2, v9, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc32: ; GFX9-LABEL: udot8_acc32:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s7, s6, 28 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_bfe_u32 s12, s6, 0x40018 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40014 ; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40010 ; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_bfe_u32 s15, s6, 0x4000c ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_u32 s16, s6, 0x40008 ; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v1
; GFX9-NEXT: s_bfe_u32 s17, s6, 0x40004 ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_and_b32 s6, s6, 15 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
; GFX9-NEXT: s_lshr_b32 s1, s0, 28 ; GFX9-NEXT: v_bfe_u32 v4, v1, 24, 4
; GFX9-NEXT: s_bfe_u32 s4, s0, 0x40018 ; GFX9-NEXT: v_bfe_u32 v11, v2, 24, 4
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40014 ; GFX9-NEXT: v_bfe_u32 v5, v1, 20, 4
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40010 ; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 4
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x4000c ; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 4
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40008 ; GFX9-NEXT: v_bfe_u32 v13, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s11, s0, 0x40004 ; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 4
; GFX9-NEXT: s_and_b32 s0, s0, 15 ; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 4
; GFX9-NEXT: v_mov_b32_e32 v1, s6 ; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 4
; GFX9-NEXT: v_mov_b32_e32 v2, s18 ; GFX9-NEXT: v_bfe_u32 v9, v1, 4, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s0, v1, v2 ; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 4
; GFX9-NEXT: v_mov_b32_e32 v2, s17 ; GFX9-NEXT: v_bfe_u32 v16, v2, 4, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s11, v2, v1 ; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s16 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s10, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v1, v1, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s15 ; GFX9-NEXT: v_mul_u32_u24_e32 v2, v9, v16
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
; GFX9-NEXT: v_mov_b32_e32 v2, s14 ; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s13 ; GFX9-NEXT: v_add3_u32 v1, v1, s0, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
; GFX9-NEXT: v_mov_b32_e32 v2, s12 ; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7
; GFX9-NEXT: v_mov_b32_e32 v2, s7 ; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
; GFX9-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v3, v3, v10
; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc32: ; GFX9-DL-LABEL: udot8_acc32:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s14, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s12, s12, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: v_dot8_u32_u4 v0, v2, v3, s0
; GFX9-DL-NEXT: v_dot8_u32_u4 v1, s0, v1, v2 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc32: ; GFX10-DL-LABEL: udot8_acc32:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s14, -1
; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s12, s12, s3
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_dot8_u32_u4 v0, s0, s1, v0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i32 %cv1e0 = zext i4 %v1e0 to i32
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = zext i4 %v2e0 to i32 %cv2e0 = zext i4 %v2e0 to i32
%mul0 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Linesentry:
ret void ret void
} }
; TODO: Remove the unnecessary instruction(that is zero-extending the ; TODO: Remove the unnecessary instruction(that is zero-extending the
; 2nd MAD) to have the pattern-recognizer to kick in. ; 2nd MAD) to have the pattern-recognizer to kick in.
define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc16: ; GFX7-LABEL: udot8_acc16:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s22, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s23, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s20, s20, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_addc_u32 s21, s21, 0 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_lshr_b32 s6, s4, 28 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014 ; GFX7-NEXT: buffer_load_ushort v16, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004 ; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: s_lshr_b32 s13, s5, 28 ; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: s_and_b32 s5, s5, 15 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014 ; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010 ; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c ; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004 ; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: s_and_b32 s4, s4, 15 ; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_mov_b32_e32 v2, s19 ; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_mov_b32_e32 v3, s18 ; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_mov_b32_e32 v4, s17 ; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v5, s16 ; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_mov_b32_e32 v6, s15 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: v_mov_b32_e32 v7, s14
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s13 ; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc16: ; GFX8-LABEL: udot8_acc16:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_mov_b32 s19, 0xe80000 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s17, s17, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshr_b32 s2, s0, 28
; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
; GFX8-NEXT: s_bfe_u32 s13, s1, 0x4000c
; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40008
; GFX8-NEXT: s_bfe_u32 s15, s1, 0x40004
; GFX8-NEXT: s_lshr_b32 s9, s1, 28
; GFX8-NEXT: s_and_b32 s1, s1, 15
; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
; GFX8-NEXT: s_bfe_u32 s6, s0, 0x4000c
; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40008
; GFX8-NEXT: s_bfe_u32 s8, s0, 0x40004
; GFX8-NEXT: s_and_b32 s0, s0, 15
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v4, s15 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, s14 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: flat_load_ushort v18, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v7, s12 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
; GFX8-NEXT: v_bfe_u32 v5, v4, 24, 4
; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
; GFX8-NEXT: v_bfe_u32 v12, v0, 24, 4
; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2 ; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2 ; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 ; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2 ; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2 ; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2 ; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc16: ; GFX9-LABEL: udot8_acc16:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ushort v17, v1, s[2:3]
; GFX9-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-NEXT: v_bfe_u32 v4, v2, 24, 4
; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-NEXT: s_bfe_u32 s15, s1, 0x4000c ; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40008 ; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-NEXT: s_bfe_u32 s17, s1, 0x40004 ; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-NEXT: s_lshr_b32 s11, s1, 28 ; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-NEXT: s_and_b32 s1, s1, 15 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010 ; GFX9-NEXT: v_bfe_u32 v11, v3, 24, 4
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x4000c ; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40004 ; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-NEXT: s_and_b32 s0, s0, 15 ; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s16
; GFX9-NEXT: v_mov_b32_e32 v5, s15
; GFX9-NEXT: v_mov_b32_e32 v6, s14
; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_mov_b32_e32 v8, s12
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc16: ; GFX9-DL-LABEL: udot8_acc16:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: global_load_ushort v1, v0, s[2:3] ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: global_load_ushort v17, v1, s[2:3]
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v4, v2, 24, 4
; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x4000c ; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40008 ; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-DL-NEXT: s_bfe_u32 s17, s1, 0x40004 ; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28 ; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-DL-NEXT: s_and_b32 s1, s1, 15 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v11, v3, 24, 4
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x4000c ; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x40004 ; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-DL-NEXT: s_and_b32 s0, s0, 15 ; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc16: ; GFX10-DL-LABEL: udot8_acc16:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX10-DL-NEXT: s_and_b32 s0, s6, 15 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_and_b32 s1, s7, 15 ; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40008 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
; GFX10-DL-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x4000c ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 24, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28 ; GFX10-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i16 %cv1e0 = zext i4 %v1e0 to i16
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = zext i4 %v2e0 to i16 %cv2e0 = zext i4 %v2e0 to i16
%mul0 = mul nuw nsw i16 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Linesentry:
ret void ret void
} }
; TODO: Remove the unnecessary instruction(that is zero-extending the ; TODO: Remove the unnecessary instruction(that is zero-extending the
; 2nd MAD) to have the pattern-recognizer to kick in. ; 2nd MAD) to have the pattern-recognizer to kick in.
define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc8: ; GFX7-LABEL: udot8_acc8:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s22, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s23, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s20, s20, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_addc_u32 s21, s21, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s6, s4, 28 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_lshr_b32 s13, s5, 28 ; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
; GFX7-NEXT: s_and_b32 s5, s5, 15 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010 ; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c ; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004 ; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: s_and_b32 s4, s4, 15 ; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: v_mov_b32_e32 v2, s19 ; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: v_mov_b32_e32 v3, s18 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mov_b32_e32 v4, s17 ; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: v_mov_b32_e32 v5, s16 ; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v6, s15 ; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_mov_b32_e32 v7, s14 ; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s13 ; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc8: ; GFX8-LABEL: udot8_acc8:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_mov_b32 s19, 0xe80000 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s17, s17, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshr_b32 s2, s0, 28
; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
; GFX8-NEXT: s_bfe_u32 s13, s1, 0x4000c
; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40008
; GFX8-NEXT: s_bfe_u32 s15, s1, 0x40004
; GFX8-NEXT: s_lshr_b32 s9, s1, 28
; GFX8-NEXT: s_and_b32 s1, s1, 15
; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
; GFX8-NEXT: s_bfe_u32 s6, s0, 0x4000c
; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40008
; GFX8-NEXT: s_bfe_u32 s8, s0, 0x40004
; GFX8-NEXT: s_and_b32 s0, s0, 15
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v4, s15 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, s14 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: flat_load_ubyte v18, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v7, s12 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
; GFX8-NEXT: v_bfe_u32 v5, v4, 24, 4
; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
; GFX8-NEXT: v_bfe_u32 v12, v0, 24, 4
; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2 ; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v2 ; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 ; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2 ; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2 ; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2 ; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc8: ; GFX9-LABEL: udot8_acc8:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ubyte v17, v1, s[2:3]
; GFX9-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-NEXT: v_bfe_u32 v4, v2, 24, 4
; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-NEXT: s_bfe_u32 s15, s1, 0x4000c ; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40008 ; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-NEXT: s_bfe_u32 s17, s1, 0x40004 ; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-NEXT: s_lshr_b32 s11, s1, 28 ; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-NEXT: s_and_b32 s1, s1, 15 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010 ; GFX9-NEXT: v_bfe_u32 v11, v3, 24, 4
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x4000c ; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40004 ; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-NEXT: s_and_b32 s0, s0, 15 ; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s16
; GFX9-NEXT: v_mov_b32_e32 v5, s15
; GFX9-NEXT: v_mov_b32_e32 v6, s14
; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_mov_b32_e32 v8, s12
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xff, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc8: ; GFX9-DL-LABEL: udot8_acc8:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: global_load_ubyte v17, v1, s[2:3]
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v4, v2, 24, 4
; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x4000c ; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40008 ; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-DL-NEXT: s_bfe_u32 s17, s1, 0x40004 ; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28 ; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-DL-NEXT: s_and_b32 s1, s1, 15 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v11, v3, 24, 4
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x4000c ; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x40004 ; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-DL-NEXT: s_and_b32 s0, s0, 15 ; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-DL-NEXT: v_and_b32_e32 v1, 0xff, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc8: ; GFX10-DL-LABEL: udot8_acc8:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: s_and_b32 s0, s6, 15 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_and_b32 s1, s7, 15 ; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40008 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
; GFX10-DL-NEXT: v_and_b32_e32 v1, 0xff, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x4000c ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 24, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i8 %cv1e0 = zext i4 %v1e0 to i8
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = zext i4 %v2e0 to i8 %cv2e0 = zext i4 %v2e0 to i8
%mul0 = mul nuw nsw i8 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Linesentry:
ret void ret void
} }
; TODO: Remove the two unnecessary instructions(and+add after 2nd MAD) ; TODO: Remove the two unnecessary instructions(and+add after 2nd MAD)
; to have the pattern-recognizer to kick in. ; to have the pattern-recognizer to kick in.
define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc4: ; GFX7-LABEL: udot8_acc4:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s22, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s23, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s20, s20, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_addc_u32 s21, s21, 0 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_lshr_b32 s6, s4, 28 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014 ; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004 ; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: s_lshr_b32 s13, s5, 28 ; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: s_and_b32 s5, s5, 15 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014 ; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010 ; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c ; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004 ; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: s_and_b32 s4, s4, 15 ; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_mov_b32_e32 v2, s19 ; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_mov_b32_e32 v3, s18 ; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_mov_b32_e32 v4, s17 ; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v5, s16 ; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_mov_b32_e32 v6, s15 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: v_mov_b32_e32 v7, s14
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s13 ; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc4: ; GFX8-LABEL: udot8_acc4:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_mov_b32 s19, 0xe80000 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s17, s17, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s8, s0, 15
; GFX8-NEXT: s_and_b32 s15, s1, 15
; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v4, s15
; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
; GFX8-NEXT: s_bfe_u32 s13, s1, 0x40008
; GFX8-NEXT: s_lshr_b32 s9, s1, 28
; GFX8-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v5, s14
; GFX8-NEXT: s_lshr_b32 s2, s0, 28
; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
; GFX8-NEXT: s_bfe_u32 s6, s0, 0x40008
; GFX8-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mul_u32_u24_e32 v3, s0, v3 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_and_b32_e32 v3, 15, v3 ; GFX8-NEXT: flat_load_ubyte v18, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v7, s12 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v4
; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v12, 24, v0
; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 ; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2 ; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2 ; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2 ; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2 ; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc4: ; GFX9-LABEL: udot8_acc4:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ubyte v17, v1, s[2:3]
; GFX9-NEXT: s_and_b32 s10, s0, 15 ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: s_and_b32 s17, s1, 15 ; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40004 ; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40008 ; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-NEXT: s_lshr_b32 s11, s1, 28
; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40004
; GFX9-NEXT: v_mov_b32_e32 v4, s16
; GFX9-NEXT: s_lshr_b32 s4, s0, 28
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008
; GFX9-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: v_mov_b32_e32 v5, s15
; GFX9-NEXT: v_mul_u32_u24_e32 v2, s0, v2
; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: v_mov_b32_e32 v6, s14 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-NEXT: v_mov_b32_e32 v8, s12 ; GFX9-NEXT: v_lshrrev_b32_e32 v11, 24, v3
; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc4: ; GFX9-DL-LABEL: udot8_acc4:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: global_load_ubyte v17, v1, s[2:3]
; GFX9-DL-NEXT: s_and_b32 s10, s0, 15 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_and_b32 s17, s1, 15 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40004 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x40008 ; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40004
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40008
; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, s0, v2
; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v3
; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc4: ; GFX10-DL-LABEL: udot8_acc4:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: s_and_b32 s0, s6, 15 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_and_b32 s1, s7, 15 ; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40008 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
; GFX10-DL-NEXT: v_mul_u32_u24_e64 v2, s1, s3 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s2, v1 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v2
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v3
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v0
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i4 addrspace(1)* nocapture %dst) { i4 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%mul0 = mul nuw nsw i4 %v1e0, %v2e0 %mul0 = mul nuw nsw i4 %v1e0, %v2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
%v2e1 = extractelement <8 x i4> %vec2, i64 1 %v2e1 = extractelement <8 x i4> %vec2, i64 1
%mul1 = mul nuw nsw i4 %v1e1, %v2e1 %mul1 = mul nuw nsw i4 %v1e1, %v2e1
Show All 36 Linesentry:
ret void ret void
} }
; TODO: Currently, permutation of udot8 is turned off due to a huge increase ; TODO: Currently, permutation of udot8 is turned off due to a huge increase
; in the compile time. ; in the compile time.
define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_CommutationInsideMAD: ; GFX7-LABEL: udot8_CommutationInsideMAD:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s22, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s23, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s20, s20, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_addc_u32 s21, s21, 0 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_lshr_b32 s6, s4, 28 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014 ; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004 ; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: s_lshr_b32 s13, s5, 28 ; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: s_and_b32 s5, s5, 15 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014 ; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010 ; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c ; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004 ; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: s_and_b32 s4, s4, 15 ; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_mov_b32_e32 v2, s19 ; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_mov_b32_e32 v3, s18 ; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_mov_b32_e32 v4, s17 ; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v5, s16 ; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_mov_b32_e32 v6, s15 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: v_mov_b32_e32 v7, s14
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s13 ; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_CommutationInsideMAD: ; GFX8-LABEL: udot8_CommutationInsideMAD:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_mov_b32 s19, 0xe80000 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s17, s17, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s8, s0, 15
; GFX8-NEXT: s_and_b32 s15, s1, 15
; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v4, s15
; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
; GFX8-NEXT: s_bfe_u32 s13, s1, 0x40008
; GFX8-NEXT: s_lshr_b32 s9, s1, 28
; GFX8-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v5, s14
; GFX8-NEXT: s_lshr_b32 s2, s0, 28
; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
; GFX8-NEXT: s_bfe_u32 s6, s0, 0x40008
; GFX8-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mul_u32_u24_e32 v3, s0, v3 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_and_b32_e32 v3, 15, v3 ; GFX8-NEXT: flat_load_ubyte v18, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v7, s12 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v4
; GFX8-NEXT: v_bfe_u32 v6, v4, 20, 4
; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 4
; GFX8-NEXT: v_bfe_u32 v8, v4, 12, 4
; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
; GFX8-NEXT: v_bfe_u32 v10, v4, 4, 4
; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshrrev_b32_e32 v11, 28, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v12, 24, v0
; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
; GFX8-NEXT: v_bfe_u32 v14, v0, 16, 4
; GFX8-NEXT: v_bfe_u32 v15, v0, 12, 4
; GFX8-NEXT: v_bfe_u32 v16, v0, 8, 4
; GFX8-NEXT: v_bfe_u32 v17, v0, 4, 4
; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2 ; GFX8-NEXT: v_mad_u16 v0, v4, v0, v18
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_mad_u16 v0, v10, v17, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 ; GFX8-NEXT: v_mad_u16 v0, v9, v16, v0
; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX8-NEXT: v_mad_u16 v0, v8, v15, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_mad_u16 v0, v7, v14, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2 ; GFX8-NEXT: v_mad_u16 v0, v6, v13, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2 ; GFX8-NEXT: v_mad_u16 v0, v5, v12, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2 ; GFX8-NEXT: v_mad_u16 v0, v1, v11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_CommutationInsideMAD: ; GFX9-LABEL: udot8_CommutationInsideMAD:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ubyte v17, v1, s[2:3]
; GFX9-NEXT: s_and_b32 s10, s0, 15 ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: s_and_b32 s17, s1, 15 ; GFX9-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40004 ; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40008 ; GFX9-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-NEXT: s_lshr_b32 s11, s1, 28
; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40004
; GFX9-NEXT: v_mov_b32_e32 v4, s16
; GFX9-NEXT: s_lshr_b32 s4, s0, 28
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008
; GFX9-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: v_mov_b32_e32 v5, s15
; GFX9-NEXT: v_mul_u32_u24_e32 v2, s0, v2
; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: v_mov_b32_e32 v6, s14 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-NEXT: v_mov_b32_e32 v8, s12 ; GFX9-NEXT: v_lshrrev_b32_e32 v11, 24, v3
; GFX9-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-NEXT: v_add_u32_e32 v1, v2, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_CommutationInsideMAD: ; GFX9-DL-LABEL: udot8_CommutationInsideMAD:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: global_load_ubyte v17, v1, s[2:3]
; GFX9-DL-NEXT: s_and_b32 s10, s0, 15 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_and_b32 s17, s1, 15 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 28, v2
; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40004 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17 ; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018 ; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x40008 ; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40004
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40008
; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, s0, v2
; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v3
; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 16, 4
; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 8, 4
; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v3, 15, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v2, v3, v17
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v9, v16, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v8, v15, v2
; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v7, v14, v2
; GFX9-DL-NEXT: v_add_u32_e32 v1, v2, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v6, v13, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v5, v12, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v2, v4, v11, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v10, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v0
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_CommutationInsideMAD: ; GFX10-DL-LABEL: udot8_CommutationInsideMAD:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
; GFX10-DL-NEXT: s_and_b32 s0, s6, 15 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_and_b32 s1, s7, 15 ; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v2
; GFX10-DL-NEXT: s_bfe_u32 s2, s6, 0x40008 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x40008 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3
; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4
; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4
; GFX10-DL-NEXT: v_mul_u32_u24_e64 v2, s0, s1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s2, s3, v1 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 24, v2
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v3
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018 ; GFX10-DL-NEXT: v_mad_u16 v0, v11, v5, v0
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v0
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i4 addrspace(1)* nocapture %dst) { i4 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%mul0 = mul nuw nsw i4 %v1e0, %v2e0 %mul0 = mul nuw nsw i4 %v1e0, %v2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
%v2e1 = extractelement <8 x i4> %vec2, i64 1 %v2e1 = extractelement <8 x i4> %vec2, i64 1
%mul1 = mul nuw nsw i4 %v1e1, %v2e1 %mul1 = mul nuw nsw i4 %v1e1, %v2e1
Show All 34 Linesentry:
store i4 %add8, i4 addrspace(1)* %dst, align 4 store i4 %add8, i4 addrspace(1)* %dst, align 4
ret void ret void
} }
define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_multiuses_mul1: ; GFX7-LABEL: udot8_multiuses_mul1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_bfe_u32 s19, s6, 0x40004
; GFX7-NEXT: s_lshr_b32 s7, s6, 28
; GFX7-NEXT: s_bfe_u32 s14, s6, 0x40018
; GFX7-NEXT: s_bfe_u32 s15, s6, 0x40014
; GFX7-NEXT: s_bfe_u32 s16, s6, 0x40010
; GFX7-NEXT: s_bfe_u32 s17, s6, 0x4000c
; GFX7-NEXT: s_bfe_u32 s18, s6, 0x40008
; GFX7-NEXT: s_and_b32 s6, s6, 15
; GFX7-NEXT: s_lshr_b32 s5, s4, 28
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40018
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40014
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40010
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x4000c
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40008
; GFX7-NEXT: s_bfe_u32 s13, s4, 0x40004
; GFX7-NEXT: s_and_b32 s4, s4, 15
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mad_u32_u24 v1, s4, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s19
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1
; GFX7-NEXT: v_mad_u32_u24 v1, s13, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s18
; GFX7-NEXT: v_mad_u32_u24 v1, s12, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s17
; GFX7-NEXT: v_mad_u32_u24 v1, s11, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s16
; GFX7-NEXT: v_mad_u32_u24 v1, s10, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s15
; GFX7-NEXT: v_mad_u32_u24 v1, s9, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s14
; GFX7-NEXT: v_mad_u32_u24 v1, s8, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s7
; GFX7-NEXT: v_mad_u32_u24 v1, s5, v2, v1
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v0 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v16, v2, v0, s4
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
; GFX7-NEXT: v_mad_u32_u24 v2, v8, v15, v16
; GFX7-NEXT: v_mad_u32_u24 v2, v7, v14, v2
; GFX7-NEXT: v_mad_u32_u24 v2, v6, v13, v2
; GFX7-NEXT: v_mad_u32_u24 v2, v5, v12, v2
; GFX7-NEXT: v_mad_u32_u24 v2, v4, v11, v2
; GFX7-NEXT: v_mad_u32_u24 v2, v3, v10, v2
; GFX7-NEXT: v_mad_u32_u24 v1, v1, v9, v2
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_multiuses_mul1: ; GFX8-LABEL: udot8_multiuses_mul1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_mov_b32 s23, 0xe80000 ; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: s_add_u32 s20, s20, s3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_bfe_u32 s17, s6, 0x40004 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_lshr_b32 s7, s6, 28 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_bfe_u32 s12, s6, 0x40018 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_bfe_u32 s13, s6, 0x40014 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_bfe_u32 s14, s6, 0x40010 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_bfe_u32 s15, s6, 0x4000c ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_bfe_u32 s16, s6, 0x40008 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_and_b32 s6, s6, 15 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 28, v0
; GFX8-NEXT: s_lshr_b32 s3, s2, 28 ; GFX8-NEXT: v_bfe_u32 v3, v0, 24, 4
; GFX8-NEXT: s_bfe_u32 s4, s2, 0x40018 ; GFX8-NEXT: v_bfe_u32 v4, v0, 20, 4
; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40014 ; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 4
; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010 ; GFX8-NEXT: v_bfe_u32 v6, v0, 12, 4
; GFX8-NEXT: s_bfe_u32 s9, s2, 0x4000c ; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40008 ; GFX8-NEXT: v_bfe_u32 v8, v0, 4, 4
; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40004 ; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_and_b32 s2, s2, 15 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s18 ; GFX8-NEXT: v_bfe_u32 v10, v1, 24, 4
; GFX8-NEXT: v_mad_u32_u24 v1, s2, v0, v1 ; GFX8-NEXT: v_bfe_u32 v11, v1, 20, 4
; GFX8-NEXT: v_mov_b32_e32 v2, s17 ; GFX8-NEXT: v_bfe_u32 v12, v1, 16, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s2, v0, v1 ; GFX8-NEXT: v_bfe_u32 v13, v1, 12, 4
; GFX8-NEXT: v_mad_u32_u24 v1, s11, v2, v1 ; GFX8-NEXT: v_bfe_u32 v14, v1, 8, 4
; GFX8-NEXT: v_mov_b32_e32 v2, s16 ; GFX8-NEXT: v_bfe_u32 v15, v1, 4, 4
; GFX8-NEXT: v_mad_u32_u24 v1, s10, v2, v1 ; GFX8-NEXT: v_and_b32_e32 v1, 15, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s15 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v1, s9, v2, v1 ; GFX8-NEXT: v_mad_u32_u24 v16, v0, v1, s2
; GFX8-NEXT: v_mov_b32_e32 v2, s14 ; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, v16
; GFX8-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX8-NEXT: v_mad_u32_u24 v1, v8, v15, v16
; GFX8-NEXT: v_mov_b32_e32 v2, s13 ; GFX8-NEXT: v_mad_u32_u24 v1, v7, v14, v1
; GFX8-NEXT: v_mad_u32_u24 v1, s5, v2, v1 ; GFX8-NEXT: v_mad_u32_u24 v1, v6, v13, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s12 ; GFX8-NEXT: v_mad_u32_u24 v1, v5, v12, v1
; GFX8-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX8-NEXT: v_mad_u32_u24 v1, v4, v11, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s7 ; GFX8-NEXT: v_mad_u32_u24 v1, v3, v10, v1
; GFX8-NEXT: v_mad_u32_u24 v1, s3, v2, v1 ; GFX8-NEXT: v_mad_u32_u24 v1, v2, v9, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_multiuses_mul1: ; GFX9-LABEL: udot8_multiuses_mul1:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_nop 0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0 ; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_bfe_u32 v3, v1, 4, 4
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 28, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX9-NEXT: v_bfe_u32 v5, v1, 24, 4
; GFX9-NEXT: v_bfe_u32 v12, v2, 24, 4
; GFX9-NEXT: v_bfe_u32 v6, v1, 20, 4
; GFX9-NEXT: v_bfe_u32 v13, v2, 20, 4
; GFX9-NEXT: v_bfe_u32 v7, v1, 16, 4
; GFX9-NEXT: v_bfe_u32 v14, v2, 16, 4
; GFX9-NEXT: v_bfe_u32 v8, v1, 12, 4
; GFX9-NEXT: v_bfe_u32 v9, v1, 8, 4
; GFX9-NEXT: v_bfe_u32 v15, v2, 12, 4
; GFX9-NEXT: v_bfe_u32 v16, v2, 8, 4
; GFX9-NEXT: v_bfe_u32 v10, v2, 4, 4
; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: v_mul_u32_u24_e32 v17, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_u32 s17, s6, 0x40004 ; GFX9-NEXT: v_mad_u32_u24 v1, v1, v2, s0
; GFX9-NEXT: s_lshr_b32 s7, s6, 28 ; GFX9-NEXT: v_mul_u32_u24_e32 v9, v9, v16
; GFX9-NEXT: s_bfe_u32 s12, s6, 0x40018 ; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40014 ; GFX9-NEXT: v_mad_u32_u24 v2, v3, v10, v1
; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40010 ; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-NEXT: s_bfe_u32 s15, s6, 0x4000c ; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
; GFX9-NEXT: s_bfe_u32 s16, s6, 0x40008 ; GFX9-NEXT: v_add3_u32 v2, v2, v9, v8
; GFX9-NEXT: s_and_b32 s6, s6, 15 ; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
; GFX9-NEXT: s_lshr_b32 s1, s0, 28 ; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
; GFX9-NEXT: s_bfe_u32 s4, s0, 0x40018 ; GFX9-NEXT: v_add3_u32 v2, v2, v7, v6
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40014 ; GFX9-NEXT: v_add3_u32 v2, v2, v5, v4
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40010 ; GFX9-NEXT: v_add3_u32 v1, v17, v1, v2
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x4000c
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40008
; GFX9-NEXT: s_bfe_u32 s11, s0, 0x40004
; GFX9-NEXT: s_and_b32 s0, s0, 15
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mad_u32_u24 v2, s0, v1, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s17
; GFX9-NEXT: v_mad_u32_u24 v1, s0, v1, v2
; GFX9-NEXT: v_mad_u32_u24 v2, s11, v3, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: v_mad_u32_u24 v2, s10, v3, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s15
; GFX9-NEXT: v_mad_u32_u24 v2, s9, v3, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s14
; GFX9-NEXT: v_mad_u32_u24 v2, s8, v3, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s13
; GFX9-NEXT: v_mad_u32_u24 v2, s5, v3, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: v_mad_u32_u24 v2, s4, v3, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: v_mad_u32_u24 v2, s1, v3, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_multiuses_mul1: ; GFX9-DL-LABEL: udot8_multiuses_mul1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_nop 0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_load_dword s18, s[2:3], 0x0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_bfe_u32 v3, v1, 4, 4
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2
; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 4
; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 4
; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4
; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4
; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4
; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4
; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4
; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4
; GFX9-DL-NEXT: v_bfe_u32 v10, v2, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v17, v1, v2
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s17, s6, 0x40004 ; GFX9-DL-NEXT: v_mad_u32_u24 v1, v1, v2, s0
; GFX9-DL-NEXT: s_lshr_b32 s7, s6, 28 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v9, v9, v16
; GFX9-DL-NEXT: s_bfe_u32 s12, s6, 0x40018 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v8, v8, v15
; GFX9-DL-NEXT: s_bfe_u32 s13, s6, 0x40014 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, v3, v10, v1
; GFX9-DL-NEXT: s_bfe_u32 s14, s6, 0x40010 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-DL-NEXT: s_bfe_u32 s15, s6, 0x4000c ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v13
; GFX9-DL-NEXT: s_bfe_u32 s16, s6, 0x40008 ; GFX9-DL-NEXT: v_add3_u32 v2, v2, v9, v8
; GFX9-DL-NEXT: s_and_b32 s6, s6, 15 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v12
; GFX9-DL-NEXT: s_lshr_b32 s1, s0, 28 ; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v11
; GFX9-DL-NEXT: s_bfe_u32 s4, s0, 0x40018 ; GFX9-DL-NEXT: v_add3_u32 v2, v2, v7, v6
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40014 ; GFX9-DL-NEXT: v_add3_u32 v2, v2, v5, v4
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40010 ; GFX9-DL-NEXT: v_add3_u32 v1, v17, v1, v2
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x4000c
; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x40008
; GFX9-DL-NEXT: s_bfe_u32 s11, s0, 0x40004
; GFX9-DL-NEXT: s_and_b32 s0, s0, 15
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s6
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s18
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s0, v1, v2
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s11, v3, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s16
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s10, v3, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s15
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v3, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s14
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s8, v3, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s13
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v3, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s12
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s4, v3, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v3, v2
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_multiuses_mul1: ; GFX10-DL-LABEL: udot8_multiuses_mul1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s14, -1
; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s12, s12, s3
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_and_b32 s4, s0, 15 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_and_b32 s5, s1, 15 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40004 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40004 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0 ; GFX10-DL-NEXT: v_and_b32_e32 v8, 15, v1
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40008 ; GFX10-DL-NEXT: v_and_b32_e32 v9, 15, v2
; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40008 ; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
; GFX10-DL-NEXT: v_mad_u32_u24 v0, s4, s5, v0 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v1
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1 ; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 24, 4
; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 16, 4
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1 ; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v12, v2, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 8, 4
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1 ; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 4
; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40014 ; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 4, 4
; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40014 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1 ; GFX10-DL-NEXT: v_mad_u32_u24 v13, v8, v9, s2
; GFX10-DL-NEXT: s_bfe_u32 s6, s0, 0x40018 ; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s7, s1, 0x40018 ; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 16, 4
; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 28 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v1, v11
; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 28 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v12
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v10, v13
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v2
; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v1 ; GFX10-DL-NEXT: v_bfe_u32 v2, v2, 24, 4
; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3] ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v15
; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v14
; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v7
; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v4, v2
; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v10
; GFX10-DL-NEXT: v_mul_u32_u24_e32 v3, v8, v9
; GFX10-DL-NEXT: v_add3_u32 v0, v0, v6, v5
; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: v_add3_u32 v0, v3, v13, v0
; GFX10-DL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%v1e0 = extractelement <8 x i4> %vec1, i64 0 %v1e0 = extractelement <8 x i4> %vec1, i64 0
%cv1e0 = zext i4 %v1e0 to i32 %cv1e0 = zext i4 %v1e0 to i32
%v2e0 = extractelement <8 x i4> %vec2, i64 0 %v2e0 = extractelement <8 x i4> %vec2, i64 0
%cv2e0 = zext i4 %v2e0 to i32 %cv2e0 = zext i4 %v2e0 to i32
%mul0 = mul nuw nsw i32 %cv1e0, %cv2e0 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
%v1e1 = extractelement <8 x i4> %vec1, i64 1 %v1e1 = extractelement <8 x i4> %vec1, i64 1
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Linesentry:
%res = add i32 %add, %add8 %res = add i32 %add, %add8
store i32 %res, i32 addrspace(1)* %dst, align 4 store i32 %res, i32 addrspace(1)* %dst, align 4
ret void ret void
} }
define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc32_vecMul: ; GFX7-LABEL: udot8_acc32_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s24, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s25, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s26, -1 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_mov_b32 s27, 0xe8f000
; GFX7-NEXT: s_add_u32 s24, s24, s3
; GFX7-NEXT: s_addc_u32 s25, s25, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshr_b32 s7, s6, 28
; GFX7-NEXT: s_bfe_u32 s14, s6, 0x40018
; GFX7-NEXT: s_bfe_u32 s15, s6, 0x40014
; GFX7-NEXT: s_bfe_u32 s16, s6, 0x40010
; GFX7-NEXT: s_bfe_u32 s17, s6, 0x4000c
; GFX7-NEXT: s_bfe_u32 s18, s6, 0x40008
; GFX7-NEXT: s_bfe_u32 s19, s6, 0x40004
; GFX7-NEXT: s_and_b32 s6, s6, 15
; GFX7-NEXT: s_lshr_b32 s5, s4, 28
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40018
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40014
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40010
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x4000c
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40008
; GFX7-NEXT: s_bfe_u32 s13, s4, 0x40004
; GFX7-NEXT: s_and_b32 s4, s4, 15
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s19
; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s18
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s17
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s16
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s15
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s14
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s4
; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc32_vecMul: ; GFX8-LABEL: udot8_acc32_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_mov_b32 s23, 0xe80000 ; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: s_add_u32 s20, s20, s3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_lshr_b32 s7, s6, 28 ; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_bfe_u32 s12, s6, 0x40018 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_bfe_u32 s13, s6, 0x40014 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_bfe_u32 s14, s6, 0x40010 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_bfe_u32 s15, s6, 0x4000c ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_bfe_u32 s16, s6, 0x40008 ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_bfe_u32 s17, s6, 0x40004 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_and_b32 s6, s6, 15 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 28, v0
; GFX8-NEXT: s_lshr_b32 s3, s2, 28 ; GFX8-NEXT: v_bfe_u32 v3, v0, 24, 4
; GFX8-NEXT: s_bfe_u32 s4, s2, 0x40018 ; GFX8-NEXT: v_bfe_u32 v4, v0, 20, 4
; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40014 ; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 4
; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010 ; GFX8-NEXT: v_bfe_u32 v6, v0, 12, 4
; GFX8-NEXT: s_bfe_u32 s9, s2, 0x4000c ; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40008 ; GFX8-NEXT: v_bfe_u32 v8, v0, 4, 4
; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40004 ; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: s_and_b32 s2, s2, 15 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s18 ; GFX8-NEXT: v_bfe_u32 v10, v1, 24, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s2, v0, v1 ; GFX8-NEXT: v_bfe_u32 v11, v1, 20, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s17 ; GFX8-NEXT: v_bfe_u32 v12, v1, 16, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s11, v1, v0 ; GFX8-NEXT: v_bfe_u32 v13, v1, 12, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s16 ; GFX8-NEXT: v_bfe_u32 v14, v1, 8, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX8-NEXT: v_bfe_u32 v15, v1, 4, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s15 ; GFX8-NEXT: v_and_b32_e32 v1, 15, v1
; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s14 ; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, s2
; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s13 ; GFX8-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s12 ; GFX8-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v2, v2, v9, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc32_vecMul: ; GFX9-LABEL: udot8_acc32_vecMul:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_nop 0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0 ; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_lshr_b32 s7, s6, 28 ; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v1
; GFX9-NEXT: s_bfe_u32 s12, s6, 0x40018 ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40014 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2
; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40010 ; GFX9-NEXT: v_bfe_u32 v4, v1, 24, 4
; GFX9-NEXT: s_bfe_u32 s15, s6, 0x4000c ; GFX9-NEXT: v_bfe_u32 v11, v2, 24, 4
; GFX9-NEXT: s_bfe_u32 s16, s6, 0x40008 ; GFX9-NEXT: v_bfe_u32 v5, v1, 20, 4
; GFX9-NEXT: s_bfe_u32 s17, s6, 0x40004 ; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 4
; GFX9-NEXT: s_and_b32 s6, s6, 15 ; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 4
; GFX9-NEXT: s_lshr_b32 s1, s0, 28 ; GFX9-NEXT: v_bfe_u32 v13, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s4, s0, 0x40018 ; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 4
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40014 ; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 4
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40010 ; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 4
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x4000c ; GFX9-NEXT: v_bfe_u32 v9, v1, 4, 4
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x40008 ; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 4
; GFX9-NEXT: s_bfe_u32 s11, s0, 0x40004 ; GFX9-NEXT: v_bfe_u32 v16, v2, 4, 4
; GFX9-NEXT: s_and_b32 s0, s0, 15 ; GFX9-NEXT: v_and_b32_e32 v1, 15, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s6 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s18 ; GFX9-NEXT: v_mul_u32_u24_e32 v1, v1, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s0, v1, v2 ; GFX9-NEXT: v_mul_u32_u24_e32 v2, v9, v16
; GFX9-NEXT: v_mov_b32_e32 v2, s17 ; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
; GFX9-NEXT: v_mad_u32_u24 v1, s11, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-NEXT: v_mov_b32_e32 v2, s16 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mad_u32_u24 v1, s10, v2, v1 ; GFX9-NEXT: v_add3_u32 v1, v1, s0, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s15 ; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
; GFX9-NEXT: v_mov_b32_e32 v2, s14 ; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7
; GFX9-NEXT: v_mad_u32_u24 v1, s8, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
; GFX9-NEXT: v_mov_b32_e32 v2, s13 ; GFX9-NEXT: v_mul_u32_u24_e32 v3, v3, v10
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v2, v1 ; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
; GFX9-NEXT: v_mov_b32_e32 v2, s12 ; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NEXT: v_mad_u32_u24 v1, s1, v2, v1
; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc32_vecMul: ; GFX9-DL-LABEL: udot8_acc32_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s14, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s12, s12, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: v_dot8_u32_u4 v0, v2, v3, s0
; GFX9-DL-NEXT: v_dot8_u32_u4 v1, s0, v1, v2 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc32_vecMul: ; GFX10-DL-LABEL: udot8_acc32_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s14, -1
; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s12, s12, s3
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: v_dot8_u32_u4 v0, s0, s1, v0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i32 addrspace(1)* nocapture %dst) { i32 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = zext <8 x i4> %vec1 to <8 x i32> %cvec1 = zext <8 x i4> %vec1 to <8 x i32>
%cvec2 = zext <8 x i4> %vec2 to <8 x i32> %cvec2 = zext <8 x i4> %vec2 to <8 x i32>
%mul = mul <8 x i32> %cvec1, %cvec2 %mul = mul <8 x i32> %cvec1, %cvec2
%mul0 = extractelement <8 x i32> %mul, i64 0 %mul0 = extractelement <8 x i32> %mul, i64 0
%mul1 = extractelement <8 x i32> %mul, i64 1 %mul1 = extractelement <8 x i32> %mul, i64 1
%mul2 = extractelement <8 x i32> %mul, i64 2 %mul2 = extractelement <8 x i32> %mul, i64 2
Show All 17 Linesentry:
ret void ret void
} }
; TODO: Clean up the code(by default pk_mad_I16 should be generated), then ; TODO: Clean up the code(by default pk_mad_I16 should be generated), then
; support the pattern. ; support the pattern.
define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc16_vecMul: ; GFX7-LABEL: udot8_acc16_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s22, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s23, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s20, s20, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_addc_u32 s21, s21, 0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v4, s17 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018 ; GFX7-NEXT: buffer_load_ushort v16, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014 ; GFX7-NEXT: s_mov_b32 s4, 0xf0000
; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_lshr_b32 s13, s5, 28 ; GFX7-NEXT: v_bfe_u32 v7, v2, 20, 4
; GFX7-NEXT: s_and_b32 s5, s5, 15 ; GFX7-NEXT: v_lshlrev_b32_e32 v8, 12, v2
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: v_mov_b32_e32 v2, s19 ; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: v_mul_u32_u24_e32 v2, s12, v2 ; GFX7-NEXT: v_bfe_u32 v4, v2, 12, 4
; GFX7-NEXT: v_mul_u32_u24_e32 v4, s10, v4 ; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 4
; GFX7-NEXT: s_lshr_b32 s6, s4, 28 ; GFX7-NEXT: v_and_b32_e32 v6, 15, v2
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX7-NEXT: v_alignbit_b32 v2, v7, v2, 16
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014 ; GFX7-NEXT: v_and_b32_e32 v7, s4, v8
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008 ; GFX7-NEXT: v_lshlrev_b32_e32 v8, 12, v0
; GFX7-NEXT: v_mov_b32_e32 v3, s18 ; GFX7-NEXT: v_or_b32_e32 v6, v6, v7
; GFX7-NEXT: s_and_b32 s4, s4, 15 ; GFX7-NEXT: v_and_b32_e32 v7, s4, v8
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_and_b32_e32 v13, 15, v0
; GFX7-NEXT: v_mul_u32_u24_e32 v1, s4, v1 ; GFX7-NEXT: v_or_b32_e32 v7, v13, v7
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v6
; GFX7-NEXT: v_mul_u32_u24_e32 v3, s11, v3 ; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v7
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4 ; GFX7-NEXT: v_and_b32_e32 v6, 15, v6
; GFX7-NEXT: v_or_b32_e32 v3, v3, v4 ; GFX7-NEXT: v_and_b32_e32 v7, 15, v7
; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
; GFX7-NEXT: v_alignbit_b32 v2, v3, v2, 16
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; GFX7-NEXT: v_mov_b32_e32 v5, s16
; GFX7-NEXT: v_mov_b32_e32 v6, s15
; GFX7-NEXT: v_mov_b32_e32 v7, s14
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ; GFX7-NEXT: v_mad_u32_u24 v6, v6, v7, v16
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v2, v0 ; GFX7-NEXT: v_bfe_u32 v12, v0, 8, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0 ; GFX7-NEXT: v_mad_u32_u24 v6, v8, v13, v6
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0 ; GFX7-NEXT: v_bfe_u32 v14, v0, 20, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0 ; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0 ; GFX7-NEXT: v_bfe_u32 v11, v0, 12, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s13 ; GFX7-NEXT: v_alignbit_b32 v0, v14, v0, 16
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v5, v5, v12, v6
; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v0
; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: v_mad_u32_u24 v4, v4, v11, v5
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v4
; GFX7-NEXT: v_mad_u32_u24 v0, v15, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc16_vecMul: ; GFX8-LABEL: udot8_acc16_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_mov_b32 s19, 0xe80000 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s17, s17, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshr_b32 s2, s0, 28
; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
; GFX8-NEXT: s_bfe_u32 s13, s1, 0x4000c
; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40008
; GFX8-NEXT: s_bfe_u32 s15, s1, 0x40004
; GFX8-NEXT: s_lshr_b32 s9, s1, 28
; GFX8-NEXT: s_and_b32 s1, s1, 15
; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
; GFX8-NEXT: s_bfe_u32 s6, s0, 0x4000c
; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40008
; GFX8-NEXT: s_bfe_u32 s8, s0, 0x40004
; GFX8-NEXT: s_and_b32 s0, s0, 15
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v4, s15 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, s14 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: flat_load_ushort v18, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v7, s12 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_and_b32_e32 v1, 15, v4
; GFX8-NEXT: v_bfe_u32 v5, v4, 4, 4
; GFX8-NEXT: v_bfe_u32 v6, v4, 8, 4
; GFX8-NEXT: v_bfe_u32 v7, v4, 12, 4
; GFX8-NEXT: v_bfe_u32 v8, v4, 16, 4
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_and_b32_e32 v11, 15, v0
; GFX8-NEXT: v_bfe_u32 v12, v0, 4, 4
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u16 v1, v1, v11, v18
; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2 ; GFX8-NEXT: v_bfe_u32 v13, v0, 8, 4
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2 ; GFX8-NEXT: v_mad_u16 v1, v5, v12, v1
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_bfe_u32 v14, v0, 12, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 ; GFX8-NEXT: v_mad_u16 v1, v6, v13, v1
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2 ; GFX8-NEXT: v_bfe_u32 v15, v0, 16, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2 ; GFX8-NEXT: v_mad_u16 v1, v7, v14, v1
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2 ; GFX8-NEXT: v_bfe_u32 v9, v4, 20, 4
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: v_bfe_u32 v16, v0, 20, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: v_mad_u16 v1, v8, v15, v1
; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: v_bfe_u32 v10, v4, 24, 4
; GFX8-NEXT: v_bfe_u32 v17, v0, 24, 4
; GFX8-NEXT: v_mad_u16 v1, v9, v16, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 28, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 28, v0
; GFX8-NEXT: v_mad_u16 v1, v10, v17, v1
; GFX8-NEXT: v_mad_u16 v0, v4, v0, v1
; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc16_vecMul: ; GFX9-LABEL: udot8_acc16_vecMul:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v5, v0, s[2:3] ; GFX9-NEXT: global_load_dword v3, v0, s[4:5]
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v4, v0, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_bfe_u32 s7, s6, 0x40018 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_lshr_b32 s12, s6, 28 ; GFX9-NEXT: v_bfe_u32 v0, v3, 24, 4
; GFX9-NEXT: s_pack_ll_b32_b16 s7, s7, s12 ; GFX9-NEXT: v_bfe_u32 v6, v3, 16, 4
; GFX9-NEXT: s_bfe_u32 s1, s0, 0x40018 ; GFX9-NEXT: v_bfe_u32 v8, v3, 8, 4
; GFX9-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_bfe_u32 s13, s6, 0x40010 ; GFX9-NEXT: v_bfe_u32 v11, v4, 24, 4
; GFX9-NEXT: s_bfe_u32 s14, s6, 0x40014 ; GFX9-NEXT: v_bfe_u32 v13, v4, 16, 4
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4 ; GFX9-NEXT: v_bfe_u32 v15, v4, 8, 4
; GFX9-NEXT: v_mov_b32_e32 v1, s7 ; GFX9-NEXT: v_and_b32_e32 v17, 15, v4
; GFX9-NEXT: v_pk_mul_lo_u16 v1, s1, v1 ; GFX9-NEXT: v_and_b32_e32 v10, 15, v3
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s13, s14 ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 28, v3
; GFX9-NEXT: s_bfe_u32 s15, s6, 0x40008 ; GFX9-NEXT: v_and_b32_e32 v0, v2, v0
; GFX9-NEXT: s_bfe_u32 s16, s6, 0x4000c ; GFX9-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX9-NEXT: s_and_b32 s17, s6, 15 ; GFX9-NEXT: v_and_b32_e32 v6, v2, v6
; GFX9-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NEXT: v_bfe_u32 v9, v3, 12, 4
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s15, s16 ; GFX9-NEXT: v_and_b32_e32 v8, v2, v8
; GFX9-NEXT: s_bfe_u32 s6, s6, 0x40004 ; GFX9-NEXT: v_bfe_u32 v3, v3, 4, 4
; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: v_lshrrev_b32_e32 v12, 28, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s17, s6 ; GFX9-NEXT: v_bfe_u32 v14, v4, 20, 4
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40010 ; GFX9-NEXT: v_bfe_u32 v16, v4, 12, 4
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40014 ; GFX9-NEXT: v_bfe_u32 v4, v4, 4, 4
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-NEXT: v_and_b32_e32 v17, v2, v17
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x4000c ; GFX9-NEXT: v_and_b32_e32 v11, v2, v11
; GFX9-NEXT: s_and_b32 s11, s0, 15 ; GFX9-NEXT: v_and_b32_e32 v13, v2, v13
; GFX9-NEXT: s_bfe_u32 s0, s0, 0x40004 ; GFX9-NEXT: v_and_b32_e32 v15, v2, v15
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s11, s0 ; GFX9-NEXT: v_and_b32_e32 v2, v2, v10
; GFX9-NEXT: v_mov_b32_e32 v4, s1 ; GFX9-NEXT: v_lshl_or_b32 v4, v4, 16, v17
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s5, s8 ; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX9-NEXT: v_pk_mul_lo_u16 v4, s0, v4 ; GFX9-NEXT: v_pk_mul_lo_u16 v2, v2, v4
; GFX9-NEXT: v_pk_mul_lo_u16 v2, s4, v2 ; GFX9-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s9, s10 ; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: v_pk_mul_lo_u16 v3, s4, v3 ; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-NEXT: v_lshl_or_b32 v5, v14, 16, v13
; GFX9-NEXT: v_lshl_or_b32 v7, v16, 16, v15
; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
; GFX9-NEXT: v_pk_mul_lo_u16 v3, v6, v5
; GFX9-NEXT: v_pk_mul_lo_u16 v5, v8, v7
; GFX9-NEXT: v_lshl_or_b32 v10, v12, 16, v11
; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v10
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v5, v4, v5 ; GFX9-NEXT: v_add_u16_e32 v4, v2, v4
; GFX9-NEXT: v_add_u32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_sdwa v4, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0 ; GFX9-NEXT: v_add_u16_e32 v2, v2, v5
; GFX9-NEXT: v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2 ; GFX9-NEXT: v_add_u16_e32 v2, v2, v3
; GFX9-NEXT: v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1 ; GFX9-NEXT: v_add_u16_e32 v2, v2, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc16_vecMul: ; GFX9-DL-LABEL: udot8_acc16_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ushort v5, v0, s[2:3] ; GFX9-DL-NEXT: global_load_dword v3, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v4, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_bfe_u32 s7, s6, 0x40018 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_lshr_b32 s12, s6, 28 ; GFX9-DL-NEXT: v_bfe_u32 v0, v3, 24, 4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s7, s7, s12 ; GFX9-DL-NEXT: v_bfe_u32 v6, v3, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s1, s0, 0x40018 ; GFX9-DL-NEXT: v_bfe_u32 v8, v3, 8, 4
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: s_bfe_u32 s13, s6, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v11, v4, 24, 4
; GFX9-DL-NEXT: s_bfe_u32 s14, s6, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v13, v4, 16, 4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s1, s4 ; GFX9-DL-NEXT: v_bfe_u32 v15, v4, 8, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s7 ; GFX9-DL-NEXT: v_and_b32_e32 v17, 15, v4
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, s1, v1 ; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v3
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s13, s14 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 28, v3
; GFX9-DL-NEXT: s_bfe_u32 s15, s6, 0x40008 ; GFX9-DL-NEXT: v_and_b32_e32 v0, v2, v0
; GFX9-DL-NEXT: s_bfe_u32 s16, s6, 0x4000c ; GFX9-DL-NEXT: v_bfe_u32 v7, v3, 20, 4
; GFX9-DL-NEXT: s_and_b32 s17, s6, 15 ; GFX9-DL-NEXT: v_and_b32_e32 v6, v2, v6
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-DL-NEXT: v_bfe_u32 v9, v3, 12, 4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s15, s16 ; GFX9-DL-NEXT: v_and_b32_e32 v8, v2, v8
; GFX9-DL-NEXT: s_bfe_u32 s6, s6, 0x40004 ; GFX9-DL-NEXT: v_bfe_u32 v3, v3, 4, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 28, v4
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s1, s17, s6 ; GFX9-DL-NEXT: v_bfe_u32 v14, v4, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v16, v4, 12, 4
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v4, v4, 4, 4
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40008 ; GFX9-DL-NEXT: v_and_b32_e32 v17, v2, v17
; GFX9-DL-NEXT: s_bfe_u32 s10, s0, 0x4000c ; GFX9-DL-NEXT: v_and_b32_e32 v11, v2, v11
; GFX9-DL-NEXT: s_and_b32 s11, s0, 15 ; GFX9-DL-NEXT: v_and_b32_e32 v13, v2, v13
; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x40004 ; GFX9-DL-NEXT: v_and_b32_e32 v15, v2, v15
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s0, s11, s0 ; GFX9-DL-NEXT: v_and_b32_e32 v2, v2, v10
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s1 ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v17
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s4, s5, s8 ; GFX9-DL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, s0, v4 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v4
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, s4, v2 ; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
; GFX9-DL-NEXT: s_pack_ll_b32_b16 s4, s9, s10 ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, s4, v3 ; GFX9-DL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-DL-NEXT: v_lshl_or_b32 v5, v14, 16, v13
; GFX9-DL-NEXT: v_lshl_or_b32 v7, v16, 16, v15
; GFX9-DL-NEXT: v_lshl_or_b32 v8, v9, 16, v8
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v3, v6, v5
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v8, v7
; GFX9-DL-NEXT: v_lshl_or_b32 v10, v12, 16, v11
; GFX9-DL-NEXT: v_pk_mul_lo_u16 v0, v0, v10
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u32_e32 v5, v4, v5 ; GFX9-DL-NEXT: v_add_u16_e32 v4, v2, v4
; GFX9-DL-NEXT: v_add_u32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u32_sdwa v4, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v5
; GFX9-DL-NEXT: v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u32_e32 v3, v3, v2 ; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v3
; GFX9-DL-NEXT: v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v1 ; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v0
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc16_vecMul: ; GFX10-DL-LABEL: udot8_acc16_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ushort v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_and_b32 s0, s6, 15 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
; GFX10-DL-NEXT: s_bfe_u32 s3, s6, 0x40004 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_and_b32 s1, s7, 15 ; GFX10-DL-NEXT: v_and_b32_e32 v7, 15, v1
; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40004 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s3 ; GFX10-DL-NEXT: v_and_b32_e32 v6, 15, v2
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s1, s1, s2 ; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 4, 4
; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40008 ; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 4, 4
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, s0, s1 ; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 8, 4
; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c ; GFX10-DL-NEXT: v_and_b32_e32 v7, v4, v7
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008 ; GFX10-DL-NEXT: v_and_b32_e32 v6, v4, v6
; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v19, v1, 8, 4
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s2, s2, s3 ; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 12, 4
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s1 ; GFX10-DL-NEXT: v_and_b32_e32 v13, v4, v13
; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x40014 ; GFX10-DL-NEXT: v_lshl_or_b32 v7, v9, 16, v7
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, s0, s2 ; GFX10-DL-NEXT: v_lshl_or_b32 v6, v15, 16, v6
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40010 ; GFX10-DL-NEXT: v_and_b32_e32 v12, v4, v19
; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x40014 ; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 16, 4
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s1 ; GFX10-DL-NEXT: v_lshl_or_b32 v10, v10, 16, v13
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s2, s2, s3 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v6, v7, v6
; GFX10-DL-NEXT: s_lshr_b32 s1, s6, 28 ; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 16, 4
; GFX10-DL-NEXT: s_lshr_b32 s3, s7, 28 ; GFX10-DL-NEXT: v_lshl_or_b32 v9, v9, 16, v12
; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 24, 4
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v23, 28, v1
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v12, 16, v6
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v6, v3
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v9, v9, v10
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, s0, s2 ; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018 ; GFX10-DL-NEXT: v_and_b32_e32 v11, v4, v11
; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40018 ; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s0, s0, s1 ; GFX10-DL-NEXT: v_and_b32_e32 v7, v4, v7
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v12
; GFX10-DL-NEXT: s_pack_ll_b32_b16 s2, s2, s3 ; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 24, 4
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v1, 16, v11
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, s0, s2 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NEXT: v_lshl_or_b32 v6, v6, 16, v7
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v9
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v14, v3, v9
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_and_b32_e32 v9, v4, v10
; GFX10-DL-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-DL-NEXT: v_and_b32_e32 v4, v4, v5
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v6
; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v14, v7
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v2, 16, v9
; GFX10-DL-NEXT: v_lshl_or_b32 v4, v23, 16, v4
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v1
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v2
; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v3, v5
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v2
; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v1, v3
; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i16 addrspace(1)* nocapture %dst) { i16 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = zext <8 x i4> %vec1 to <8 x i16> %cvec1 = zext <8 x i4> %vec1 to <8 x i16>
%cvec2 = zext <8 x i4> %vec2 to <8 x i16> %cvec2 = zext <8 x i4> %vec2 to <8 x i16>
%mul = mul <8 x i16> %cvec1, %cvec2 %mul = mul <8 x i16> %cvec1, %cvec2
%mul0 = extractelement <8 x i16> %mul, i64 0 %mul0 = extractelement <8 x i16> %mul, i64 0
%mul1 = extractelement <8 x i16> %mul, i64 1 %mul1 = extractelement <8 x i16> %mul, i64 1
%mul2 = extractelement <8 x i16> %mul, i64 2 %mul2 = extractelement <8 x i16> %mul, i64 2
Show All 16 Linesentry:
store i16 %add8, i16 addrspace(1)* %dst, align 4 store i16 %add8, i16 addrspace(1)* %dst, align 4
ret void ret void
} }
; TODO: Cleanup the code to generate MAD; pattern should be recognized then. ; TODO: Cleanup the code to generate MAD; pattern should be recognized then.
define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc8_vecMul: ; GFX7-LABEL: udot8_acc8_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s22, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s23, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s20, s20, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_addc_u32 s21, s21, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_bfe_u32 s6, s4, 0x4000c ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_bfe_u32 s13, s5, 0x4000c ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40004 ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_lshr_b32 s17, s5, 28 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v8, s13 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40008 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_and_b32 s16, s5, 15 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40018 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40014 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40004 ; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
; GFX7-NEXT: v_mov_b32_e32 v6, s15 ; GFX7-NEXT: s_movk_i32 s4, 0xf00
; GFX7-NEXT: s_lshr_b32 s10, s4, 28 ; GFX7-NEXT: v_mov_b32_e32 v3, 0xf00
; GFX7-NEXT: v_mov_b32_e32 v4, s17 ; GFX7-NEXT: s_movk_i32 s5, 0xf0f
; GFX7-NEXT: v_mul_u32_u24_e32 v4, s10, v4 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: v_mul_u32_u24_e32 v6, s8, v6 ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_mul_u32_u24_e32 v8, s6, v8 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 28, v2
; GFX7-NEXT: s_bfe_u32 s5, s5, 0x40010 ; GFX7-NEXT: v_lshlrev_b32_e32 v9, 4, v2
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40008 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 4, v2
; GFX7-NEXT: v_mov_b32_e32 v7, s14 ; GFX7-NEXT: v_lshrrev_b32_e32 v8, 12, v2
; GFX7-NEXT: s_and_b32 s9, s4, 15 ; GFX7-NEXT: v_bfe_u32 v1, v2, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v5, s16 ; GFX7-NEXT: v_and_b32_e32 v5, 15, v2
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40018 ; GFX7-NEXT: v_bfe_u32 v7, v2, 16, 4
; GFX7-NEXT: v_mov_b32_e32 v3, s18 ; GFX7-NEXT: v_alignbit_b32 v2, v6, v2, 24
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40014 ; GFX7-NEXT: v_and_b32_e32 v6, s4, v9
; GFX7-NEXT: v_mov_b32_e32 v2, s19 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mul_u32_u24_e32 v2, s12, v2 ; GFX7-NEXT: v_lshrrev_b32_e32 v11, 4, v0
; GFX7-NEXT: s_bfe_u32 s4, s4, 0x40010 ; GFX7-NEXT: v_lshlrev_b32_e32 v9, 4, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_and_b32_e32 v4, s4, v4
; GFX7-NEXT: v_mul_u32_u24_e32 v3, s11, v3 ; GFX7-NEXT: v_or_b32_e32 v5, v5, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4 ; GFX7-NEXT: v_and_b32_e32 v6, v3, v9
; GFX7-NEXT: v_mul_u32_u24_e32 v5, s9, v5 ; GFX7-NEXT: v_bfe_u32 v10, v0, 8, 4
; GFX7-NEXT: v_mul_u32_u24_e32 v7, s7, v7 ; GFX7-NEXT: v_and_b32_e32 v3, v3, v11
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6 ; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v8 ; GFX7-NEXT: v_or_b32_e32 v3, v10, v3
; GFX7-NEXT: v_or_b32_e32 v3, v3, v4 ; GFX7-NEXT: v_and_b32_e32 v12, 15, v0
; GFX7-NEXT: v_or_b32_e32 v4, v5, v6 ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_or_b32_e32 v5, v7, v8 ; GFX7-NEXT: v_lshrrev_b32_e32 v13, 28, v0
; GFX7-NEXT: v_mul_u32_u24_e32 v9, s4, v1 ; GFX7-NEXT: v_or_b32_e32 v6, v12, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
; GFX7-NEXT: v_or_b32_e32 v2, v9, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5 ; GFX7-NEXT: v_and_b32_e32 v2, s5, v2
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX7-NEXT: v_or_b32_e32 v3, v6, v3
; GFX7-NEXT: v_or_b32_e32 v3, v4, v5 ; GFX7-NEXT: v_lshrrev_b32_e32 v15, 12, v0
; GFX7-NEXT: v_alignbit_b32 v4, v2, v3, 8 ; GFX7-NEXT: v_bfe_u32 v14, v0, 16, 4
; GFX7-NEXT: v_alignbit_b32 v5, v2, v3, 16 ; GFX7-NEXT: v_alignbit_b32 v0, v13, v0, 24
; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v3 ; GFX7-NEXT: v_and_b32_e32 v8, s4, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v2 ; GFX7-NEXT: v_or_b32_e32 v1, v5, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v2 ; GFX7-NEXT: v_and_b32_e32 v4, s4, v15
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2 ; GFX7-NEXT: v_and_b32_e32 v0, s5, v0
; GFX7-NEXT: v_or_b32_e32 v7, v7, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_and_b32_e32 v6, 15, v1
; GFX7-NEXT: v_and_b32_e32 v12, 15, v3
; GFX7-NEXT: v_or_b32_e32 v2, v7, v2
; GFX7-NEXT: v_or_b32_e32 v4, v14, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_bfe_u32 v7, v1, 8, 4
; GFX7-NEXT: v_bfe_u32 v13, v3, 8, 4
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v3 ; GFX7-NEXT: v_mad_u32_u24 v6, v6, v12, v16
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0 ; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 24, v1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0 ; GFX7-NEXT: v_lshrrev_b32_e32 v10, 24, v3
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_bfe_u32 v1, v1, 16, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v7 ; GFX7-NEXT: v_bfe_u32 v3, v3, 16, 4
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v8 ; GFX7-NEXT: v_mad_u32_u24 v6, v7, v13, v6
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v6
; GFX7-NEXT: v_and_b32_e32 v8, 15, v2
; GFX7-NEXT: v_and_b32_e32 v14, 15, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v10, v1
; GFX7-NEXT: v_bfe_u32 v9, v2, 8, 4
; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 4
; GFX7-NEXT: v_mad_u32_u24 v1, v8, v14, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v11, 24, v0
; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 4
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 4
; GFX7-NEXT: v_mad_u32_u24 v1, v9, v15, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v5, v11, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc8_vecMul: ; GFX8-LABEL: udot8_acc8_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s22, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_mov_b32 s23, 0xe80000 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_ubyte v5, v[0:1]
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: s_add_u32 s20, s20, s3 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: s_addc_u32 s21, s21, 0 ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: s_mov_b32 s0, 0xffff ; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_bfe_u32 v3, v4, 20, 4
; GFX8-NEXT: s_bfe_u32 s7, s1, 0x40004 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_bfe_u32 s9, s1, 0x4000c ; GFX8-NEXT: v_bfe_u32 v13, v2, 20, 4
; GFX8-NEXT: s_bfe_u32 s14, s2, 0x40004 ; GFX8-NEXT: v_bfe_u32 v7, v4, 24, 4
; GFX8-NEXT: s_and_b32 s15, s2, 15 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 28, v4
; GFX8-NEXT: s_bfe_u32 s16, s2, 0x4000c ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 28, v2
; GFX8-NEXT: s_bfe_u32 s3, s1, 0x40014 ; GFX8-NEXT: v_bfe_u32 v14, v2, 24, 4
; GFX8-NEXT: s_lshr_b32 s5, s1, 28 ; GFX8-NEXT: v_mul_lo_u16_sdwa v3, v3, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40014 ; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 4
; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40010 ; GFX8-NEXT: v_bfe_u32 v12, v2, 16, 4
; GFX8-NEXT: s_lshr_b32 s12, s2, 28 ; GFX8-NEXT: v_bfe_u32 v9, v4, 8, 4
; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018 ; GFX8-NEXT: v_bfe_u32 v16, v2, 8, 4
; GFX8-NEXT: s_bfe_u32 s2, s2, 0x40008 ; GFX8-NEXT: v_bfe_u32 v10, v4, 12, 4
; GFX8-NEXT: s_and_b32 s8, s1, 15 ; GFX8-NEXT: v_and_b32_e32 v11, 15, v4
; GFX8-NEXT: v_mov_b32_e32 v4, s16 ; GFX8-NEXT: v_bfe_u32 v17, v2, 12, 4
; GFX8-NEXT: v_mov_b32_e32 v5, s9 ; GFX8-NEXT: v_and_b32_e32 v18, 15, v2
; GFX8-NEXT: v_mov_b32_e32 v6, s15 ; GFX8-NEXT: v_bfe_u32 v4, v4, 4, 4
; GFX8-NEXT: v_mov_b32_e32 v7, s14 ; GFX8-NEXT: v_bfe_u32 v2, v2, 4, 4
; GFX8-NEXT: v_mov_b32_e32 v8, s7 ; GFX8-NEXT: v_mul_lo_u16_e32 v13, v7, v14
; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_mul_u32_u24_e32 v5, s8, v6 ; GFX8-NEXT: v_mul_lo_u16_e32 v19, v6, v12
; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_mul_lo_u16_e32 v9, v9, v16
; GFX8-NEXT: s_bfe_u32 s4, s1, 0x40010 ; GFX8-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: s_bfe_u32 s6, s1, 0x40018 ; GFX8-NEXT: v_mul_lo_u16_e32 v11, v11, v18
; GFX8-NEXT: v_mov_b32_e32 v9, s13 ; GFX8-NEXT: v_mul_lo_u16_sdwa v4, v4, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: s_bfe_u32 s1, s1, 0x40008 ; GFX8-NEXT: v_or_b32_e32 v8, v13, v8
; GFX8-NEXT: v_mov_b32_e32 v3, s2
; GFX8-NEXT: v_mov_b32_e32 v10, s12
; GFX8-NEXT: v_mov_b32_e32 v11, s5
; GFX8-NEXT: v_mov_b32_e32 v12, s11
; GFX8-NEXT: v_mov_b32_e32 v13, s10
; GFX8-NEXT: v_mov_b32_e32 v14, s3
; GFX8-NEXT: v_mul_u32_u24_e32 v3, s1, v3
; GFX8-NEXT: v_or_b32_e32 v5, v5, v6
; GFX8-NEXT: v_mul_u32_u24_e32 v7, s6, v9
; GFX8-NEXT: v_mul_u32_u24_sdwa v8, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_mul_u32_u24_e32 v9, s4, v12
; GFX8-NEXT: v_mul_u32_u24_sdwa v10, v14, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_and_b32_e32 v5, s0, v5
; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_or_b32_e32 v9, v9, v10 ; GFX8-NEXT: v_or_b32_e32 v9, v9, v10
; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_or_b32_e32 v10, v11, v4
; GFX8-NEXT: v_and_b32_e32 v4, s0, v9 ; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v8
; GFX8-NEXT: v_or_b32_e32 v3, v5, v3 ; GFX8-NEXT: v_or_b32_e32 v3, v19, v3
; GFX8-NEXT: v_or_b32_e32 v6, v4, v7 ; GFX8-NEXT: v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v9
; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v6 ; GFX8-NEXT: v_or_b32_e32 v4, v4, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v11, 8, v3
; GFX8-NEXT: v_lshrrev_b64 v[2:3], 24, v[2:3]
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 8, v4
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5 ; GFX8-NEXT: v_add_u16_e32 v3, v10, v5
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v7, v2 ; GFX8-NEXT: v_add_u16_e32 v3, v3, v4
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_0 ; GFX8-NEXT: v_add_u16_e32 v3, v3, v9
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD ; GFX8-NEXT: v_add_u16_e32 v2, v3, v2
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; GFX8-NEXT: v_mad_u16 v2, v6, v12, v2
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v8, v2 ; GFX8-NEXT: v_add_u16_e32 v2, v2, v11
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v8
; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD ; GFX8-NEXT: v_mad_u16 v2, v7, v14, v2
; GFX8-NEXT: v_add_u16_e32 v2, v2, v8
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc8_vecMul: ; GFX9-LABEL: udot8_acc8_vecMul:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b32 s0, 0xffff ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_load_dword s8, s[6:7], 0x0 ; GFX9-NEXT: global_load_ubyte v4, v3, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: s_bfe_u32 s4, s1, 0x40010 ; GFX9-NEXT: v_bfe_u32 v0, v1, 20, 4
; GFX9-NEXT: s_bfe_u32 s12, s8, 0x40010 ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_u32 s13, s8, 0x40014 ; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 4
; GFX9-NEXT: s_bfe_u32 s14, s8, 0x40018 ; GFX9-NEXT: v_bfe_u32 v6, v1, 24, 4
; GFX9-NEXT: s_lshr_b32 s15, s8, 28 ; GFX9-NEXT: v_lshrrev_b32_e32 v7, 28, v1
; GFX9-NEXT: s_and_b32 s16, s8, 15 ; GFX9-NEXT: v_lshrrev_b32_e32 v14, 28, v2
; GFX9-NEXT: s_bfe_u32 s17, s8, 0x40004 ; GFX9-NEXT: v_bfe_u32 v13, v2, 24, 4
; GFX9-NEXT: s_bfe_u32 s18, s8, 0x40008 ; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v0, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v2, s12 ; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 4
; GFX9-NEXT: s_bfe_u32 s8, s8, 0x4000c ; GFX9-NEXT: v_bfe_u32 v11, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s5, s1, 0x40014 ; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 4
; GFX9-NEXT: v_mov_b32_e32 v3, s13 ; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 4
; GFX9-NEXT: s_bfe_u32 s6, s1, 0x40018 ; GFX9-NEXT: v_bfe_u32 v9, v1, 12, 4
; GFX9-NEXT: v_mov_b32_e32 v4, s14 ; GFX9-NEXT: v_and_b32_e32 v10, 15, v1
; GFX9-NEXT: s_lshr_b32 s7, s1, 28 ; GFX9-NEXT: v_bfe_u32 v16, v2, 12, 4
; GFX9-NEXT: v_mov_b32_e32 v5, s15 ; GFX9-NEXT: v_and_b32_e32 v17, 15, v2
; GFX9-NEXT: s_and_b32 s9, s1, 15 ; GFX9-NEXT: v_bfe_u32 v1, v1, 4, 4
; GFX9-NEXT: v_mov_b32_e32 v6, s16 ; GFX9-NEXT: v_bfe_u32 v2, v2, 4, 4
; GFX9-NEXT: s_bfe_u32 s10, s1, 0x40004 ; GFX9-NEXT: v_mul_lo_u16_e32 v12, v6, v13
; GFX9-NEXT: v_mov_b32_e32 v7, s17 ; GFX9-NEXT: v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_bfe_u32 s11, s1, 0x40008 ; GFX9-NEXT: v_mul_lo_u16_e32 v18, v5, v11
; GFX9-NEXT: v_mov_b32_e32 v8, s18 ; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c ; GFX9-NEXT: v_mul_lo_u16_e32 v8, v8, v15
; GFX9-NEXT: v_mov_b32_e32 v9, s8 ; GFX9-NEXT: v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v2, s4, v2 ; GFX9-NEXT: v_mul_lo_u16_e32 v10, v10, v17
; GFX9-NEXT: v_mul_lo_u16_sdwa v3, s5, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_or_b32_e32 v7, v12, v7
; GFX9-NEXT: v_mul_lo_u16_e32 v4, s6, v4 ; GFX9-NEXT: v_or_b32_e32 v8, v8, v9
; GFX9-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_or_b32_e32 v1, v18, v0
; GFX9-NEXT: v_mul_lo_u16_e32 v6, s9, v6 ; GFX9-NEXT: v_or_b32_e32 v9, v10, v2
; GFX9-NEXT: v_mul_lo_u16_sdwa v7, s10, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v7
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX9-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v8
; GFX9-NEXT: v_or_b32_e32 v4, v6, v7 ; GFX9-NEXT: v_or_b32_e32 v2, v2, v0
; GFX9-NEXT: v_mul_lo_u16_e32 v8, s11, v8 ; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v1
; GFX9-NEXT: v_mul_lo_u16_sdwa v9, s1, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
; GFX9-NEXT: v_and_b32_e32 v4, s0, v4 ; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-NEXT: v_or_b32_sdwa v5, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v5, v4, v5
; GFX9-NEXT: v_lshrrev_b32_e32 v6, 8, v5
; GFX9-NEXT: v_and_b32_e32 v2, s0, v2
; GFX9-NEXT: v_or_b32_e32 v3, v2, v3
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, v4, v1 ; GFX9-NEXT: v_add_u16_e32 v1, v9, v4
; GFX9-NEXT: v_add_u32_e32 v1, v1, v6 ; GFX9-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 ; GFX9-NEXT: v_add_u16_e32 v1, v1, v8
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-NEXT: v_mad_legacy_u16 v0, v5, v11, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v3 ; GFX9-NEXT: v_add_u16_e32 v0, v0, v10
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v7
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_mad_legacy_u16 v0, v6, v13, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-NEXT: v_add_u16_e32 v0, v0, v7
; GFX9-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc8_vecMul: ; GFX9-DL-LABEL: udot8_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DL-NEXT: s_mov_b32 s0, 0xffff ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s8, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[2:3]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: s_bfe_u32 s4, s1, 0x40010 ; GFX9-DL-NEXT: v_bfe_u32 v0, v1, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s12, s8, 0x40010 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_bfe_u32 s13, s8, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 20, 4
; GFX9-DL-NEXT: s_bfe_u32 s14, s8, 0x40018 ; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 24, 4
; GFX9-DL-NEXT: s_lshr_b32 s15, s8, 28 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 28, v1
; GFX9-DL-NEXT: s_and_b32 s16, s8, 15 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v14, 28, v2
; GFX9-DL-NEXT: s_bfe_u32 s17, s8, 0x40004 ; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 24, 4
; GFX9-DL-NEXT: s_bfe_u32 s18, s8, 0x40008 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v0, v0, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s12 ; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s8, s8, 0x4000c ; GFX9-DL-NEXT: v_bfe_u32 v11, v2, 16, 4
; GFX9-DL-NEXT: s_bfe_u32 s5, s1, 0x40014 ; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 8, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s13 ; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 8, 4
; GFX9-DL-NEXT: s_bfe_u32 s6, s1, 0x40018 ; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 12, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s14 ; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v1
; GFX9-DL-NEXT: s_lshr_b32 s7, s1, 28 ; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 12, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15 ; GFX9-DL-NEXT: v_and_b32_e32 v17, 15, v2
; GFX9-DL-NEXT: s_and_b32 s9, s1, 15 ; GFX9-DL-NEXT: v_bfe_u32 v1, v1, 4, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s16 ; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 4, 4
; GFX9-DL-NEXT: s_bfe_u32 s10, s1, 0x40004 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v12, v6, v13
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s17 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: s_bfe_u32 s11, s1, 0x40008 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v5, v11
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s18 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v8, v8, v15
; GFX9-DL-NEXT: v_mov_b32_e32 v9, s8 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v2, s4, v2 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v10, v10, v17
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v3, s5, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_or_b32_e32 v7, v12, v7
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v4, s6, v4 ; GFX9-DL-NEXT: v_or_b32_e32 v8, v8, v9
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_or_b32_e32 v1, v18, v0
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v6, s9, v6 ; GFX9-DL-NEXT: v_or_b32_e32 v9, v10, v2
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, s10, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v10, 16, v7
; GFX9-DL-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX9-DL-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_sdwa v3, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
; GFX9-DL-NEXT: v_or_b32_e32 v4, v6, v7 ; GFX9-DL-NEXT: v_or_b32_e32 v2, v2, v0
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v8, s11, v8 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 8, v1
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v9, s1, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
; GFX9-DL-NEXT: v_and_b32_e32 v4, s0, v4 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-DL-NEXT: v_or_b32_sdwa v5, v8, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_e32 v5, v4, v5
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v5
; GFX9-DL-NEXT: v_and_b32_e32 v2, s0, v2
; GFX9-DL-NEXT: v_or_b32_e32 v3, v2, v3
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v1 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v9, v4
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v6 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v2
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v8
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v5, v11, v0
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v3 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v10
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v7
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v6, v13, v0
; GFX9-DL-NEXT: v_add_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v7
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: global_store_byte v3, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc8_vecMul: ; GFX10-DL-LABEL: udot8_acc8_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: v_mov_b32_e32 v19, 0
; GFX10-DL-NEXT: s_mov_b32 s14, -1 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_add_u32 s12, s12, s3 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x40004 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x40004 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_and_b32 s0, s6, 15 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v2, s1, s3 ; GFX10-DL-NEXT: global_load_ubyte v3, v19, s[0:1]
; GFX10-DL-NEXT: s_and_b32 s1, s7, 15 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_bfe_u32 s8, s6, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v3, s0, s1 ; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 12, 4
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s8, s3 ; GFX10-DL-NEXT: v_bfe_u32 v8, v1, 8, 4
; GFX10-DL-NEXT: v_lshlrev_b16_e64 v2, 8, v2 ; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 8, 4
; GFX10-DL-NEXT: s_bfe_u32 s2, s6, 0x40008 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 28, v1
; GFX10-DL-NEXT: s_bfe_u32 s0, s7, 0x40008 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v14, 28, v2
; GFX10-DL-NEXT: s_mov_b32 s1, 0xffff ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, v9, v10
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s2, s0 ; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 16, 4
; GFX10-DL-NEXT: v_or_b32_e32 v2, v3, v2 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v8, v8, v13
; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v4 ; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s2, s6, 0x40014 ; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 24, 4
; GFX10-DL-NEXT: s_bfe_u32 s8, s7, 0x40014 ; GFX10-DL-NEXT: v_and_b32_e32 v11, 15, v1
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010 ; GFX10-DL-NEXT: v_lshlrev_b16_e64 v9, 8, v9
; GFX10-DL-NEXT: v_and_b32_e32 v2, s1, v2 ; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 4, 4
; GFX10-DL-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 4, 4
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s2, s8 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v7, v7, v14
; GFX10-DL-NEXT: s_bfe_u32 s3, s6, 0x40018 ; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s9, s7, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 24, 4
; GFX10-DL-NEXT: s_lshr_b32 s6, s6, 28 ; GFX10-DL-NEXT: v_bfe_u32 v23, v2, 16, 4
; GFX10-DL-NEXT: v_or_b32_e32 v3, v2, v3 ; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2
; GFX10-DL-NEXT: s_lshr_b32 s2, s7, 28 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v1, v1, v15
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s0, s9 ; GFX10-DL-NEXT: v_or_b32_e32 v8, v8, v9
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v6, s6, s2 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, v0, v10
; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v4 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v10, v6, v13
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v3 ; GFX10-DL-NEXT: v_lshlrev_b16_e64 v7, 8, v7
; GFX10-DL-NEXT: s_bfe_u32 s0, s7, 0x40018 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v2, v11, v2
; GFX10-DL-NEXT: v_mul_lo_u16_e64 v11, s3, s0 ; GFX10-DL-NEXT: v_lshlrev_b16_e64 v1, 8, v1
; GFX10-DL-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
; GFX10-DL-NEXT: v_lshlrev_b16_e64 v6, 8, v6 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v11, v5, v23
; GFX10-DL-NEXT: v_and_b32_e32 v4, s1, v4 ; GFX10-DL-NEXT: v_or_b32_e32 v7, v10, v7
; GFX10-DL-NEXT: v_lshlrev_b16_e64 v9, 8, v9
; GFX10-DL-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-DL-NEXT: v_or_b32_e32 v1, v2, v1
; GFX10-DL-NEXT: v_or_b32_e32 v2, v11, v9
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v9, 16, v7
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 8, v10
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v2, v1 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v1, v3
; GFX10-DL-NEXT: v_or_b32_sdwa v2, v11, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX10-DL-NEXT: v_or_b32_sdwa v1, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v7 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v9, v3, v10
; GFX10-DL-NEXT: v_or_b32_e32 v2, v4, v2 ; GFX10-DL-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v9, v8
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v2 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v0, v2
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v4 ; GFX10-DL-NEXT: v_mad_u16 v0, v5, v23, v0
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v0, v1
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v7
; GFX10-DL-NEXT: v_add_nc_u32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX10-DL-NEXT: v_mad_u16 v0, v6, v13, v0
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: v_add_nc_u16_e64 v0, v0, v1
; GFX10-DL-NEXT: global_store_byte v19, v0, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i8 addrspace(1)* nocapture %dst) { i8 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%cvec1 = zext <8 x i4> %vec1 to <8 x i8> %cvec1 = zext <8 x i4> %vec1 to <8 x i8>
%cvec2 = zext <8 x i4> %vec2 to <8 x i8> %cvec2 = zext <8 x i4> %vec2 to <8 x i8>
%mul = mul <8 x i8> %cvec1, %cvec2 %mul = mul <8 x i8> %cvec1, %cvec2
%mul0 = extractelement <8 x i8> %mul, i64 0 %mul0 = extractelement <8 x i8> %mul, i64 0
%mul1 = extractelement <8 x i8> %mul, i64 1 %mul1 = extractelement <8 x i8> %mul, i64 1
%mul2 = extractelement <8 x i8> %mul, i64 2 %mul2 = extractelement <8 x i8> %mul, i64 2
Show All 16 Linesentry:
store i8 %add8, i8 addrspace(1)* %dst, align 4 store i8 %add8, i8 addrspace(1)* %dst, align 4
ret void ret void
} }
; TODO: Once the adictional "and+add" are removed, the pattern will be recognized. ; TODO: Once the adictional "and+add" are removed, the pattern will be recognized.
define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1, define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
; GFX7-LABEL: udot8_acc4_vecMul: ; GFX7-LABEL: udot8_acc4_vecMul:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX7-NEXT: s_mov_b32 s14, -1
; GFX7-NEXT: s_mov_b32 s22, -1 ; GFX7-NEXT: s_mov_b32 s15, 0xe8f000
; GFX7-NEXT: s_mov_b32 s23, 0xe8f000 ; GFX7-NEXT: s_add_u32 s12, s12, s3
; GFX7-NEXT: s_add_u32 s20, s20, s3
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_addc_u32 s21, s21, 0 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_lshr_b32 s6, s4, 28 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s14, s5, 0x40018 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40014 ; GFX7-NEXT: buffer_load_ubyte v16, off, s[0:3], 0
; GFX7-NEXT: s_bfe_u32 s16, s5, 0x40010 ; GFX7-NEXT: s_addc_u32 s13, s13, 0
; GFX7-NEXT: s_bfe_u32 s17, s5, 0x4000c ; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: s_bfe_u32 s18, s5, 0x40008 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v2
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40004 ; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 4
; GFX7-NEXT: s_lshr_b32 s13, s5, 28 ; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 4
; GFX7-NEXT: s_and_b32 s5, s5, 15 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 4
; GFX7-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40014 ; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 4
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x40010 ; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 4
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x4000c ; GFX7-NEXT: v_and_b32_e32 v2, 15, v2
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x40008 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x40004 ; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v0
; GFX7-NEXT: s_and_b32 s4, s4, 15 ; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 4
; GFX7-NEXT: v_mov_b32_e32 v2, s19 ; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 4
; GFX7-NEXT: v_mov_b32_e32 v3, s18 ; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 4
; GFX7-NEXT: v_mov_b32_e32 v4, s17 ; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v5, s16 ; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 4
; GFX7-NEXT: v_mov_b32_e32 v6, s15 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: v_mov_b32_e32 v7, s14
; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16
; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s10, v4, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v5, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s8, v6, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v7, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s13 ; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0
; GFX7-NEXT: v_and_b32_e32 v0, 15, v0 ; GFX7-NEXT: v_and_b32_e32 v0, 15, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_acc4_vecMul: ; GFX8-LABEL: udot8_acc4_vecMul:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 ; GFX8-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX8-NEXT: s_mov_b32 s18, -1 ; GFX8-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX8-NEXT: s_mov_b32 s19, 0xe80000 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ubyte v2, v[0:1] ; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_add_u32 s16, s16, s3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_addc_u32 s17, s17, 0 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s8, s0, 15
; GFX8-NEXT: s_and_b32 s15, s1, 15
; GFX8-NEXT: s_bfe_u32 s14, s1, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v4, s15
; GFX8-NEXT: s_bfe_u32 s10, s1, 0x40018
; GFX8-NEXT: s_bfe_u32 s11, s1, 0x40014
; GFX8-NEXT: s_bfe_u32 s12, s1, 0x40010
; GFX8-NEXT: s_bfe_u32 s13, s1, 0x40008
; GFX8-NEXT: s_lshr_b32 s9, s1, 28
; GFX8-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX8-NEXT: s_bfe_u32 s7, s0, 0x40004
; GFX8-NEXT: v_mov_b32_e32 v5, s14
; GFX8-NEXT: s_lshr_b32 s2, s0, 28
; GFX8-NEXT: s_bfe_u32 s3, s0, 0x40018
; GFX8-NEXT: s_bfe_u32 s4, s0, 0x40014
; GFX8-NEXT: s_bfe_u32 s5, s0, 0x40010
; GFX8-NEXT: s_bfe_u32 s6, s0, 0x40008
; GFX8-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mul_u32_u24_e32 v3, s0, v3 ; GFX8-NEXT: s_mov_b32 s10, -1
; GFX8-NEXT: v_and_b32_e32 v3, 15, v3 ; GFX8-NEXT: s_mov_b32 s11, 0xe80000
; GFX8-NEXT: v_mov_b32_e32 v7, s12 ; GFX8-NEXT: s_add_u32 s8, s8, s3
; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: s_addc_u32 s9, s9, 0
; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_and_b32_e32 v1, 15, v4
; GFX8-NEXT: v_bfe_u32 v5, v4, 4, 4
; GFX8-NEXT: v_bfe_u32 v6, v4, 8, 4
; GFX8-NEXT: v_bfe_u32 v7, v4, 12, 4
; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s8, v4, v2 ; GFX8-NEXT: v_and_b32_e32 v11, 15, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_bfe_u32 v12, v0, 4, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 ; GFX8-NEXT: v_bfe_u32 v13, v0, 8, 4
; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX8-NEXT: v_bfe_u32 v14, v0, 12, 4
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2 ; GFX8-NEXT: v_bfe_u32 v8, v4, 16, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v7, v2 ; GFX8-NEXT: v_bfe_u32 v15, v0, 16, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v8, v2 ; GFX8-NEXT: v_bfe_u32 v9, v4, 20, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s3, v9, v2 ; GFX8-NEXT: v_bfe_u32 v10, v4, 24, 4
; GFX8-NEXT: v_mov_b32_e32 v3, s9 ; GFX8-NEXT: v_bfe_u32 v16, v0, 20, 4
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: v_bfe_u32 v17, v0, 24, 4
; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, 28, v4
; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 28, v0
; GFX8-NEXT: v_mul_u32_u24_e32 v0, v4, v0
; GFX8-NEXT: v_mul_u32_u24_e32 v4, v10, v17
; GFX8-NEXT: flat_load_ubyte v10, v[2:3]
; GFX8-NEXT: v_mul_u32_u24_e32 v1, v1, v11
; GFX8-NEXT: v_mul_u32_u24_e32 v5, v5, v12
; GFX8-NEXT: v_mul_u32_u24_e32 v6, v6, v13
; GFX8-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX8-NEXT: v_mul_u32_u24_e32 v8, v8, v15
; GFX8-NEXT: v_mul_u32_u24_e32 v9, v9, v16
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_u16_e32 v1, v1, v10
; GFX8-NEXT: v_add_u16_e32 v1, v1, v5
; GFX8-NEXT: v_add_u16_e32 v1, v1, v6
; GFX8-NEXT: v_add_u16_e32 v1, v1, v7
; GFX8-NEXT: v_add_u16_e32 v1, v1, v8
; GFX8-NEXT: v_add_u16_e32 v1, v1, v9
; GFX8-NEXT: v_add_u16_e32 v1, v1, v4
; GFX8-NEXT: v_add_u16_e32 v0, v1, v0
; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
; GFX8-NEXT: flat_store_byte v[2:3], v0
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_acc4_vecMul: ; GFX9-LABEL: udot8_acc4_vecMul:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s22, -1 ; GFX9-NEXT: s_mov_b32 s10, -1
; GFX9-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-NEXT: s_add_u32 s20, s20, s3 ; GFX9-NEXT: s_add_u32 s8, s8, s3
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_addc_u32 s9, s9, 0
; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_and_b32 s10, s0, 15 ; GFX9-NEXT: v_and_b32_e32 v0, 15, v2
; GFX9-NEXT: s_and_b32 s17, s1, 15
; GFX9-NEXT: s_bfe_u32 s16, s1, 0x40004
; GFX9-NEXT: v_mov_b32_e32 v3, s17
; GFX9-NEXT: s_bfe_u32 s12, s1, 0x40018
; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40014
; GFX9-NEXT: s_bfe_u32 s14, s1, 0x40010
; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40008
; GFX9-NEXT: s_lshr_b32 s11, s1, 28
; GFX9-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX9-NEXT: s_bfe_u32 s9, s0, 0x40004
; GFX9-NEXT: v_mov_b32_e32 v4, s16
; GFX9-NEXT: s_lshr_b32 s4, s0, 28
; GFX9-NEXT: s_bfe_u32 s5, s0, 0x40018
; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40014
; GFX9-NEXT: s_bfe_u32 s7, s0, 0x40010
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008
; GFX9-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: v_mov_b32_e32 v5, s15
; GFX9-NEXT: v_mul_u32_u24_e32 v2, s0, v2
; GFX9-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-NEXT: v_mov_b32_e32 v6, s14
; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_mov_b32_e32 v8, s12
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-NEXT: v_and_b32_e32 v10, 15, v3
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-NEXT: v_bfe_u32 v4, v2, 4, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-NEXT: v_bfe_u32 v11, v3, 4, 4
; GFX9-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-NEXT: v_bfe_u32 v5, v2, 8, 4
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-NEXT: v_bfe_u32 v12, v3, 8, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-NEXT: v_bfe_u32 v13, v3, 12, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-NEXT: v_bfe_u32 v7, v2, 16, 4
; GFX9-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-NEXT: v_bfe_u32 v14, v3, 16, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-NEXT: v_bfe_u32 v8, v2, 20, 4
; GFX9-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-NEXT: v_bfe_u32 v9, v2, 24, 4
; GFX9-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-NEXT: v_bfe_u32 v15, v3, 20, 4
; GFX9-NEXT: v_bfe_u32 v16, v3, 24, 4
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v3
; GFX9-NEXT: v_mul_u32_u24_e32 v2, v2, v3
; GFX9-NEXT: v_mul_u32_u24_e32 v3, v9, v16
; GFX9-NEXT: global_load_ubyte v9, v1, s[2:3]
; GFX9-NEXT: v_mul_u32_u24_e32 v0, v0, v10
; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11
; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12
; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13
; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u16_e32 v0, v0, v9
; GFX9-NEXT: v_add_u16_e32 v0, v0, v4
; GFX9-NEXT: v_add_u16_e32 v0, v0, v5
; GFX9-NEXT: v_add_u16_e32 v0, v0, v6
; GFX9-NEXT: v_add_u16_e32 v0, v0, v7
; GFX9-NEXT: v_add_u16_e32 v0, v0, v8
; GFX9-NEXT: v_add_u16_e32 v0, v0, v3
; GFX9-NEXT: v_add_u16_e32 v0, v0, v2
; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_acc4_vecMul: ; GFX9-DL-LABEL: udot8_acc4_vecMul:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_mov_b32 s20, SCRATCH_RSRC_DWORD0 ; GFX9-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX9-DL-NEXT: s_mov_b32 s21, SCRATCH_RSRC_DWORD1 ; GFX9-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s22, -1 ; GFX9-DL-NEXT: s_mov_b32 s10, -1
; GFX9-DL-NEXT: s_mov_b32 s23, 0xe00000 ; GFX9-DL-NEXT: s_mov_b32 s11, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s20, s20, s3 ; GFX9-DL-NEXT: s_add_u32 s8, s8, s3
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_addc_u32 s21, s21, 0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX9-DL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: s_and_b32 s10, s0, 15 ; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v2
; GFX9-DL-NEXT: s_and_b32 s17, s1, 15
; GFX9-DL-NEXT: s_bfe_u32 s16, s1, 0x40004
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s17
; GFX9-DL-NEXT: s_bfe_u32 s12, s1, 0x40018
; GFX9-DL-NEXT: s_bfe_u32 s13, s1, 0x40014
; GFX9-DL-NEXT: s_bfe_u32 s14, s1, 0x40010
; GFX9-DL-NEXT: s_bfe_u32 s15, s1, 0x40008
; GFX9-DL-NEXT: s_lshr_b32 s11, s1, 28
; GFX9-DL-NEXT: s_bfe_u32 s1, s1, 0x4000c
; GFX9-DL-NEXT: s_bfe_u32 s9, s0, 0x40004
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s16
; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 28
; GFX9-DL-NEXT: s_bfe_u32 s5, s0, 0x40018
; GFX9-DL-NEXT: s_bfe_u32 s6, s0, 0x40014
; GFX9-DL-NEXT: s_bfe_u32 s7, s0, 0x40010
; GFX9-DL-NEXT: s_bfe_u32 s8, s0, 0x40008
; GFX9-DL-NEXT: s_bfe_u32 s0, s0, 0x4000c
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s15
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, s0, v2
; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v6, s14
; GFX9-DL-NEXT: v_mov_b32_e32 v7, s13
; GFX9-DL-NEXT: v_mov_b32_e32 v8, s12
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s10, v3, v1 ; GFX9-DL-NEXT: v_and_b32_e32 v10, 15, v3
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s9, v4, v1 ; GFX9-DL-NEXT: v_bfe_u32 v4, v2, 4, 4
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s8, v5, v1 ; GFX9-DL-NEXT: v_bfe_u32 v11, v3, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-DL-NEXT: v_bfe_u32 v5, v2, 8, 4
; GFX9-DL-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-DL-NEXT: v_bfe_u32 v12, v3, 8, 4
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v6, v1 ; GFX9-DL-NEXT: v_bfe_u32 v6, v2, 12, 4
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v7, v1 ; GFX9-DL-NEXT: v_bfe_u32 v13, v3, 12, 4
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v8, v1 ; GFX9-DL-NEXT: v_bfe_u32 v7, v2, 16, 4
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s11 ; GFX9-DL-NEXT: v_bfe_u32 v14, v3, 16, 4
; GFX9-DL-NEXT: v_mad_u32_u24 v1, s4, v2, v1 ; GFX9-DL-NEXT: v_bfe_u32 v8, v2, 20, 4
; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX9-DL-NEXT: v_bfe_u32 v9, v2, 24, 4
; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3] ; GFX9-DL-NEXT: v_bfe_u32 v15, v3, 20, 4
; GFX9-DL-NEXT: v_bfe_u32 v16, v3, 24, 4
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, v2, v3
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v3, v9, v16
; GFX9-DL-NEXT: global_load_ubyte v9, v1, s[2:3]
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v0, v0, v10
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v11
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v12
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v13
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v14
; GFX9-DL-NEXT: v_mul_u32_u24_e32 v8, v8, v15
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v9
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v4
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v5
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v6
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v7
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v8
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v3
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v2
; GFX9-DL-NEXT: v_and_b32_e32 v0, 15, v0
; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_acc4_vecMul: ; GFX10-DL-LABEL: udot8_acc4_vecMul:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 ; GFX10-DL-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 ; GFX10-DL-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; GFX10-DL-NEXT: s_mov_b32 s10, -1 ; GFX10-DL-NEXT: s_mov_b32 s10, -1
; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000 ; GFX10-DL-NEXT: s_mov_b32 s11, 0x31c16000
; GFX10-DL-NEXT: s_add_u32 s8, s8, s3 ; GFX10-DL-NEXT: s_add_u32 s8, s8, s3
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0 ; GFX10-DL-NEXT: s_addc_u32 s9, s9, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: global_load_ubyte v1, v0, s[4:5] ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s7, s[2:3], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: s_and_b32 s0, s6, 15 ; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
; GFX10-DL-NEXT: s_and_b32 s1, s7, 15 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: s_bfe_u32 s2, s7, 0x40008 ; GFX10-DL-NEXT: v_and_b32_e32 v4, 15, v1
; GFX10-DL-NEXT: s_bfe_u32 s3, s7, 0x4000c ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v2
; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 4, 4
; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 4, 4
; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 24, 4
; GFX10-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v5
; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 8, 4
; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v7
; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 12, 4
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v4, v3
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40004 ; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 12, 4
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40004 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v8
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 16, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40008 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v6
; GFX10-DL-NEXT: s_bfe_u32 s1, s6, 0x4000c ; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 16, 4
; GFX10-DL-NEXT: v_mul_u32_u24_e64 v2, s1, s3 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v7
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s2, v1 ; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 20, 4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40010 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v5
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40010 ; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 20, 4
; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v8
; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 24, 4
; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 28, v1
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v4
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40014 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v7
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40014 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v4, v11, v8
; GFX10-DL-NEXT: s_bfe_u32 s0, s6, 0x40018 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v6
; GFX10-DL-NEXT: s_bfe_u32 s1, s7, 0x40018 ; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v1, v2
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v3, v3, v5
; GFX10-DL-NEXT: s_lshr_b32 s0, s6, 28 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v2, v3, v4
; GFX10-DL-NEXT: s_lshr_b32 s1, s7, 28 ; GFX10-DL-NEXT: v_add_nc_u16_e64 v1, v2, v1
; GFX10-DL-NEXT: v_mad_u32_u24 v1, s0, s1, v1
; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1 ; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v1
; GFX10-DL-NEXT: global_store_byte v0, v1, s[4:5] ; GFX10-DL-NEXT: global_store_byte v0, v1, s[2:3]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
<8 x i4> addrspace(1)* %src2, <8 x i4> addrspace(1)* %src2,
i4 addrspace(1)* nocapture %dst) { i4 addrspace(1)* nocapture %dst) {
entry: entry:
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2 %gep1 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src1, i32 %idx
%vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %gep1
%gep2 = getelementptr <8 x i4>, <8 x i4> addrspace(1)* %src2, i32 %idx
%vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %gep2
%mul = mul <8 x i4> %vec1, %vec2 %mul = mul <8 x i4> %vec1, %vec2
%mul0 = extractelement <8 x i4> %mul, i64 0 %mul0 = extractelement <8 x i4> %mul, i64 0
%mul1 = extractelement <8 x i4> %mul, i64 1 %mul1 = extractelement <8 x i4> %mul, i64 1
%mul2 = extractelement <8 x i4> %mul, i64 2 %mul2 = extractelement <8 x i4> %mul, i64 2
%mul3 = extractelement <8 x i4> %mul, i64 3 %mul3 = extractelement <8 x i4> %mul, i64 3
%mul4 = extractelement <8 x i4> %mul, i64 4 %mul4 = extractelement <8 x i4> %mul, i64 4
%mul5 = extractelement <8 x i4> %mul, i64 5 %mul5 = extractelement <8 x i4> %mul, i64 5
Show All 15 Lines
} }
define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr, define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
; GFX7-LABEL: udot8_variant1: ; GFX7-LABEL: udot8_variant1:
; GFX7: ; %bb.0: ; %entry ; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_mov_b32 s11, s3
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s20, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s6, s4, 15 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
; GFX7-NEXT: s_and_b32 s7, s5, 15 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: s_bfe_u32 s8, s4, 0x40004 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
; GFX7-NEXT: s_bfe_u32 s10, s4, 0x40008 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s12, s4, 0x4000c ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
; GFX7-NEXT: s_bfe_u32 s14, s4, 0x40010 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
; GFX7-NEXT: s_bfe_u32 s16, s4, 0x40014 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX7-NEXT: s_bfe_u32 s18, s4, 0x40018 ; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_lshr_b32 s4, s4, 28 ; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_mov_b32_e32 v0, s6 ; GFX7-NEXT: v_and_b32_e32 v1, 15, v2
; GFX7-NEXT: v_mov_b32_e32 v1, s20 ; GFX7-NEXT: v_bfe_u32 v3, v2, 4, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1 ; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_bfe_u32 s9, s5, 0x40004 ; GFX7-NEXT: v_and_b32_e32 v9, 15, v0
; GFX7-NEXT: s_bfe_u32 s11, s5, 0x40008 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 4
; GFX7-NEXT: s_bfe_u32 s13, s5, 0x4000c ; GFX7-NEXT: v_bfe_u32 v5, v2, 12, 4
; GFX7-NEXT: s_bfe_u32 s15, s5, 0x40010 ; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 4
; GFX7-NEXT: s_bfe_u32 s17, s5, 0x40014 ; GFX7-NEXT: v_bfe_u32 v7, v2, 20, 4
; GFX7-NEXT: s_bfe_u32 s19, s5, 0x40018 ; GFX7-NEXT: v_bfe_u32 v8, v2, 24, 4
; GFX7-NEXT: s_lshr_b32 s5, s5, 28 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX7-NEXT: v_mov_b32_e32 v1, s4 ; GFX7-NEXT: v_bfe_u32 v10, v0, 4, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: v_bfe_u32 v11, v0, 8, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s8 ; GFX7-NEXT: v_bfe_u32 v12, v0, 12, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0 ; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s10 ; GFX7-NEXT: v_bfe_u32 v14, v0, 20, 4
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0 ; GFX7-NEXT: v_bfe_u32 v15, v0, 24, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s12 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 28, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s13, v1, v0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v1, s14 ; GFX7-NEXT: v_mad_u32_u24 v1, v9, v1, s4
; GFX7-NEXT: v_mad_u32_u24 v0, s15, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s16 ; GFX7-NEXT: v_mad_u32_u24 v0, v10, v3, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s17, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v11, v4, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s18 ; GFX7-NEXT: v_mad_u32_u24 v0, v12, v5, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s19, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, v13, v6, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v14, v7, v0
; GFX7-NEXT: v_mad_u32_u24 v0, v15, v8, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm ; GFX7-NEXT: s_endpgm
; ;
; GFX8-LABEL: udot8_variant1: ; GFX8-LABEL: udot8_variant1:
; GFX8: ; %bb.0: ; %entry ; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-NEXT: s_load_dword s18, s[0:1], 0x0 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_and_b32 s4, s2, 15 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_and_b32 s5, s3, 15 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40004 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40008 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_bfe_u32 s10, s2, 0x4000c ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX8-NEXT: s_bfe_u32 s12, s2, 0x40010 ; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: s_bfe_u32 s14, s2, 0x40014 ; GFX8-NEXT: v_and_b32_e32 v1, 15, v3
; GFX8-NEXT: s_bfe_u32 s16, s2, 0x40018 ; GFX8-NEXT: v_bfe_u32 v4, v3, 4, 4
; GFX8-NEXT: s_lshr_b32 s2, s2, 28 ; GFX8-NEXT: v_bfe_u32 v6, v3, 8, 4
; GFX8-NEXT: v_mov_b32_e32 v0, s4 ; GFX8-NEXT: v_bfe_u32 v8, v3, 12, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s18 ; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v0
; GFX8-NEXT: s_bfe_u32 s7, s3, 0x40004 ; GFX8-NEXT: v_bfe_u32 v5, v0, 4, 4
; GFX8-NEXT: s_bfe_u32 s9, s3, 0x40008 ; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 4
; GFX8-NEXT: s_bfe_u32 s11, s3, 0x4000c ; GFX8-NEXT: v_bfe_u32 v9, v0, 12, 4
; GFX8-NEXT: s_bfe_u32 s13, s3, 0x40010 ; GFX8-NEXT: v_bfe_u32 v10, v3, 16, 4
; GFX8-NEXT: s_bfe_u32 s15, s3, 0x40014 ; GFX8-NEXT: v_bfe_u32 v11, v0, 16, 4
; GFX8-NEXT: s_bfe_u32 s17, s3, 0x40018 ; GFX8-NEXT: v_bfe_u32 v12, v3, 20, 4
; GFX8-NEXT: s_lshr_b32 s3, s3, 28 ; GFX8-NEXT: v_bfe_u32 v14, v3, 24, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 4
; GFX8-NEXT: v_mad_u32_u24 v0, s3, v1, v0 ; GFX8-NEXT: v_bfe_u32 v15, v0, 24, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s6 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 28, v3
; GFX8-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 28, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s8 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s10 ; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v1
; GFX8-NEXT: v_mad_u32_u24 v0, s11, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v5, v4, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s12 ; GFX8-NEXT: v_mad_u32_u24 v0, v7, v6, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s13, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v9, v8, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s14 ; GFX8-NEXT: v_mad_u32_u24 v0, v11, v10, v0
; GFX8-NEXT: v_mad_u32_u24 v0, s15, v1, v0 ; GFX8-NEXT: v_mad_u32_u24 v0, v13, v12, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s16 ; GFX8-NEXT: v_mad_u32_u24 v2, v15, v14, v0
; GFX8-NEXT: v_mad_u32_u24 v2, s17, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm ; GFX8-NEXT: s_endpgm
; ;
; GFX9-LABEL: udot8_variant1: ; GFX9-LABEL: udot8_variant1:
; GFX9: ; %bb.0: ; %entry ; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NEXT: s_load_dword s18, s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s0, 15 ; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_and_b32 s5, s1, 15 ; GFX9-NEXT: global_load_dword v2, v0, s[6:7]
; GFX9-NEXT: s_bfe_u32 s6, s0, 0x40004 ; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-NEXT: s_bfe_u32 s8, s0, 0x40008 ; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_bfe_u32 s10, s0, 0x4000c ; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: s_bfe_u32 s12, s0, 0x40010 ; GFX9-NEXT: v_and_b32_e32 v3, 15, v1
; GFX9-NEXT: s_bfe_u32 s14, s0, 0x40014 ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_bfe_u32 s16, s0, 0x40018 ; GFX9-NEXT: v_and_b32_e32 v4, 15, v2
; GFX9-NEXT: s_lshr_b32 s0, s0, 28 ; GFX9-NEXT: v_bfe_u32 v5, v1, 4, 4
; GFX9-NEXT: v_mov_b32_e32 v1, s4 ; GFX9-NEXT: v_bfe_u32 v6, v2, 4, 4
; GFX9-NEXT: v_mov_b32_e32 v2, s18 ; GFX9-NEXT: v_bfe_u32 v7, v1, 8, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s5, v1, v2 ; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 4
; GFX9-NEXT: s_bfe_u32 s7, s1, 0x40004 ; GFX9-NEXT: v_bfe_u32 v9, v1, 12, 4
; GFX9-NEXT: s_bfe_u32 s9, s1, 0x40008 ; GFX9-NEXT: v_bfe_u32 v10, v2, 12, 4
; GFX9-NEXT: s_bfe_u32 s11, s1, 0x4000c ; GFX9-NEXT: v_bfe_u32 v11, v1, 16, 4
; GFX9-NEXT: s_bfe_u32 s13, s1, 0x40010 ; GFX9-NEXT: v_bfe_u32 v12, v2, 16, 4
; GFX9-NEXT: s_bfe_u32 s15, s1, 0x40014 ; GFX9-NEXT: v_bfe_u32 v13, v1, 20, 4
; GFX9-NEXT: s_bfe_u32 s17, s1, 0x40018 ; GFX9-NEXT: v_bfe_u32 v15, v1, 24, 4
; GFX9-NEXT: s_lshr_b32 s1, s1, 28 ; GFX9-NEXT: v_bfe_u32 v14, v2, 20, 4
; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: v_bfe_u32 v16, v2, 24, 4
; GFX9-NEXT: v_mad_u32_u24 v1, s1, v2, v1 ; GFX9-NEXT: v_lshrrev_b32_e32 v1, 28, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s6 ; GFX9-NEXT: v_lshrrev_b32_e32 v2, 28, v2
; GFX9-NEXT: v_mad_u32_u24 v1, s7, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v3, v4, v3
; GFX9-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-NEXT: v_mul_u32_u24_e32 v1, v2, v1
; GFX9-NEXT: v_mad_u32_u24 v1, s9, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v4, v6, v5
; GFX9-NEXT: v_mov_b32_e32 v2, s10 ; GFX9-NEXT: v_mul_u32_u24_e32 v5, v8, v7
; GFX9-NEXT: v_mad_u32_u24 v1, s11, v2, v1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s12 ; GFX9-NEXT: v_add3_u32 v1, v3, s0, v1
; GFX9-NEXT: v_mad_u32_u24 v1, s13, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v6, v10, v9
; GFX9-NEXT: v_mov_b32_e32 v2, s14 ; GFX9-NEXT: v_mul_u32_u24_e32 v7, v12, v11
; GFX9-NEXT: v_mad_u32_u24 v1, s15, v2, v1 ; GFX9-NEXT: v_add3_u32 v1, v1, v4, v5
; GFX9-NEXT: v_mov_b32_e32 v2, s16 ; GFX9-NEXT: v_mul_u32_u24_e32 v8, v14, v13
; GFX9-NEXT: v_mad_u32_u24 v1, s17, v2, v1 ; GFX9-NEXT: v_mul_u32_u24_e32 v9, v16, v15
; GFX9-NEXT: v_add3_u32 v1, v1, v6, v7
; GFX9-NEXT: v_add3_u32 v1, v1, v8, v9
; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX9-DL-LABEL: udot8_variant1: ; GFX9-DL-LABEL: udot8_variant1:
; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
; GFX9-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s0 ; GFX9-DL-NEXT: v_dot8_u32_u4 v0, v3, v2, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 ; GFX9-DL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-DL-NEXT: v_dot8_u32_u4 v1, s1, v1, v2
; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-DL-NEXT: s_endpgm ; GFX9-DL-NEXT: s_endpgm
; ;
; GFX10-DL-LABEL: udot8_variant1: ; GFX10-DL-LABEL: udot8_variant1:
; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s8, s[2:3], 0x0 ; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s8 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: v_dot8_u32_u4 v0, s1, s0, v0 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v2, v1, s2
; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DL-NEXT: s_endpgm ; GFX10-DL-NEXT: s_endpgm
i32 addrspace(1)* %v2addr, i32 addrspace(1)* %v2addr,
i32 addrspace(1)* %dst) { i32 addrspace(1)* %dst) {
entry: entry:
%v1 = load i32, i32 addrspace(1)* %v1addr, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%v2 = load i32, i32 addrspace(1)* %v2addr, align 4 %gep1 = getelementptr i32, i32 addrspace(1)* %v1addr, i32 %idx
%v1 = load i32, i32 addrspace(1)* %gep1, align 4
%gep2 = getelementptr i32, i32 addrspace(1)* %v2addr, i32 %idx
%v2 = load i32, i32 addrspace(1)* %gep2, align 4
%and = and i32 %v1, 15 %and = and i32 %v1, 15
%and1 = and i32 %v2, 15 %and1 = and i32 %v2, 15
%mul1 = mul nuw nsw i32 %and1, %and %mul1 = mul nuw nsw i32 %and1, %and
%shr = lshr i32 %v1, 4 %shr = lshr i32 %v1, 4
%and2 = and i32 %shr, 15 %and2 = and i32 %shr, 15
%shr3 = lshr i32 %v2, 4 %shr3 = lshr i32 %v2, 4
%and4 = and i32 %shr3, 15 %and4 = and i32 %shr3, 15
Show All 40 Linesentry:
%add4 = add i32 %add3, %mul3 %add4 = add i32 %add3, %mul3
%add5 = add i32 %add4, %mul4 %add5 = add i32 %add4, %mul4
%add6 = add i32 %add5, %mul5 %add6 = add i32 %add5, %mul5
%add7 = add i32 %add6, %mul6 %add7 = add i32 %add6, %mul6
%add8 = add i32 %add7, %mul7 %add8 = add i32 %add7, %mul7
store i32 %add8, i32 addrspace(1)* %dst, align 4 store i32 %add8, i32 addrspace(1)* %dst, align 4
ret void ret void
} }
declare i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll

Show First 20 Lines • Show All 360 Lines • ▼ Show 20 Linesmain_body:
%tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0) %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
%tmp2 = sext i16 %tmp to i32 %tmp2 = sext i16 %tmp to i32
%val = bitcast i32 %tmp2 to float %val = bitcast i32 %tmp2 to float
ret float %val ret float %val
} }
;CHECK-LABEL: {{^}}buffer_load_ubyte_mul_bitcast: ;CHECK-LABEL: {{^}}buffer_load_ubyte_mul_bitcast:
;CHECK-NEXT: %bb. ;CHECK-NEXT: %bb.
;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, off, s[0:3], 0 offset:8 ;CHECK-NEXT: buffer_load_ubyte v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0) ;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}} ;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog ;CHECK-NEXT: ; return to shader part epilog
define amdgpu_ps float @buffer_load_ubyte_mul_bitcast(<4 x i32> inreg %rsrc) { define amdgpu_ps float @buffer_load_ubyte_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body: main_body:
%tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0) %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = zext i8 %tmp to i32 %tmp2 = zext i8 %tmp to i32
%tmp3 = mul i32 %tmp2, 255 %tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float %val = bitcast i32 %tmp3 to float
ret float %val ret float %val
} }
;CHECK-LABEL: {{^}}buffer_load_ushort_mul_bitcast: ;CHECK-LABEL: {{^}}buffer_load_ushort_mul_bitcast:
;CHECK-NEXT: %bb. ;CHECK-NEXT: %bb.
;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, off, s[0:3], 0 offset:8 ;CHECK-NEXT: buffer_load_ushort v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0) ;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}} ;CHECK-NEXT: v_mul_u32_u24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog ;CHECK-NEXT: ; return to shader part epilog
define amdgpu_ps float @buffer_load_ushort_mul_bitcast(<4 x i32> inreg %rsrc) { define amdgpu_ps float @buffer_load_ushort_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body: main_body:
%tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0) %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = zext i16 %tmp to i32 %tmp2 = zext i16 %tmp to i32
%tmp3 = mul i32 %tmp2, 255 %tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float %val = bitcast i32 %tmp3 to float
ret float %val ret float %val
} }
;CHECK-LABEL: {{^}}buffer_load_sbyte_mul_bitcast: ;CHECK-LABEL: {{^}}buffer_load_sbyte_mul_bitcast:
;CHECK-NEXT: %bb. ;CHECK-NEXT: %bb.
;CHECK-NEXT: buffer_load_sbyte v{{[0-9]}}, off, s[0:3], 0 offset:8 ;CHECK-NEXT: buffer_load_sbyte v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0) ;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}} ;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog ;CHECK-NEXT: ; return to shader part epilog
define amdgpu_ps float @buffer_load_sbyte_mul_bitcast(<4 x i32> inreg %rsrc) { define amdgpu_ps float @buffer_load_sbyte_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body: main_body:
%tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0) %tmp = call i8 @llvm.amdgcn.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = sext i8 %tmp to i32 %tmp2 = sext i8 %tmp to i32
%tmp3 = mul i32 %tmp2, 255 %tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float %val = bitcast i32 %tmp3 to float
ret float %val ret float %val
} }
;CHECK-LABEL: {{^}}buffer_load_sshort_mul_bitcast: ;CHECK-LABEL: {{^}}buffer_load_sshort_mul_bitcast:
;CHECK-NEXT: %bb. ;CHECK-NEXT: %bb.
;CHECK-NEXT: buffer_load_sshort v{{[0-9]}}, off, s[0:3], 0 offset:8 ;CHECK-NEXT: buffer_load_sshort v{{[0-9]}}, v0, s[0:3], 0 idxen offset:8
;CHECK-NEXT: s_waitcnt vmcnt(0) ;CHECK-NEXT: s_waitcnt vmcnt(0)
;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}} ;CHECK-NEXT: v_mul_i32_i24_e32 v{{[0-9]}}, 0xff, v{{[0-9]}}
;CHECK-NEXT: ; return to shader part epilog ;CHECK-NEXT: ; return to shader part epilog
define amdgpu_ps float @buffer_load_sshort_mul_bitcast(<4 x i32> inreg %rsrc) { define amdgpu_ps float @buffer_load_sshort_mul_bitcast(<4 x i32> inreg %rsrc, i32 %idx) {
main_body: main_body:
%tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0) %tmp = call i16 @llvm.amdgcn.buffer.load.i16(<4 x i32> %rsrc, i32 %idx, i32 8, i1 0, i1 0)
%tmp2 = sext i16 %tmp to i32 %tmp2 = sext i16 %tmp to i32
%tmp3 = mul i32 %tmp2, 255 %tmp3 = mul i32 %tmp2, 255
%val = bitcast i32 %tmp3 to float %val = bitcast i32 %tmp3 to float
ret float %val ret float %val
} }
;CHECK-LABEL: {{^}}buffer_load_sbyte_type_check: ;CHECK-LABEL: {{^}}buffer_load_sbyte_type_check:
;CHECK-NEXT: %bb. ;CHECK-NEXT: %bb.
▲ Show 20 Lines • Show All 48 Lines • Show Last 20 Lines

llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll

Show All 39 Lines
; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v[[DST0]], v[[DST1]] ; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v[[DST0]], v[[DST1]]
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; SDWA: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; SDWA: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in1, i32 addrspace(1)* %in2) #0 { define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in1, i32 addrspace(1)* %in2) #0 {
%a = load i32, i32 addrspace(1)* %in1, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load i32, i32 addrspace(1)* %in2, align 4 %gep1 = getelementptr i32, i32 addrspace(1)* %in1, i32 %idx
%gep2 = getelementptr i32, i32 addrspace(1)* %in2, i32 %idx
%a = load i32, i32 addrspace(1)* %gep1, align 4
%b = load i32, i32 addrspace(1)* %gep2, align 4
%shra = lshr i32 %a, 16 %shra = lshr i32 %a, 16
%shrb = lshr i32 %b, 16 %shrb = lshr i32 %b, 16
%mul = mul i32 %shra, %shrb %mul = mul i32 %shra, %shrb
store i32 %mul, i32 addrspace(1)* %out, align 4 store i32 %mul, i32 addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_i16: ; GCN-LABEL: {{^}}mul_i16:
; NOSDWA: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; GFX89: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX89: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX10: v_mul_lo_u16_e64 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; SDWA-NOT: v_mul_u32_u24_sdwa ; SDWA-NOT: v_mul_u32_u24_sdwa
define amdgpu_kernel void @mul_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %ina, i16 addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %ina, i16 addrspace(1)* %inb) #0 {
entry: entry:
%a = load i16, i16 addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load i16, i16 addrspace(1)* %inb, align 4 %gepa = getelementptr i16, i16 addrspace(1)* %ina, i32 %idx
%gepb = getelementptr i16, i16 addrspace(1)* %inb, i32 %idx
%a = load i16, i16 addrspace(1)* %gepa, align 4
%b = load i16, i16 addrspace(1)* %gepb, align 4
%mul = mul i16 %a, %b %mul = mul i16 %a, %b
store i16 %mul, i16 addrspace(1)* %out, align 4 store i16 %mul, i16 addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_v2i16: ; GCN-LABEL: {{^}}mul_v2i16:
; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v[[DST_MUL:[0-9]+]], v[[DST0]], v[[DST1]] ; NOSDWA: v_mul_lo_u16_e32 v[[DST_MUL:[0-9]+]], v[[DST1]], v[[DST0]]
; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]] ; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]] ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL_LO:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL_HI:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL_LO]], v[[DST_MUL_HI]]
; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
define amdgpu_kernel void @mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 {
entry: entry:
%a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4 %gepa = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %ina, i32 %idx
%gepb = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %inb, i32 %idx
%a = load <2 x i16>, <2 x i16> addrspace(1)* %gepa, align 4
%b = load <2 x i16>, <2 x i16> addrspace(1)* %gepb, align 4
%mul = mul <2 x i16> %a, %b %mul = mul <2 x i16> %a, %b
store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4 store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_v4i16: ; GCN-LABEL: {{^}}mul_v4i16:
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]]
; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]]
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
define amdgpu_kernel void @mul_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %ina, <4 x i16> addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %ina, <4 x i16> addrspace(1)* %inb) #0 {
entry: entry:
%a = load <4 x i16>, <4 x i16> addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load <4 x i16>, <4 x i16> addrspace(1)* %inb, align 4 %gepa = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %ina, i32 %idx
%gepb = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %inb, i32 %idx
%a = load <4 x i16>, <4 x i16> addrspace(1)* %gepa, align 4
%b = load <4 x i16>, <4 x i16> addrspace(1)* %gepb, align 4
%mul = mul <4 x i16> %a, %b %mul = mul <4 x i16> %a, %b
store <4 x i16> %mul, <4 x i16> addrspace(1)* %out, align 4 store <4 x i16> %mul, <4 x i16> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_v8i16: ; GCN-LABEL: {{^}}mul_v8i16:
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL0:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL2:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL3:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL4:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL4:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL5:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL5:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL6:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 ; VI-DAG: v_mul_lo_u16_e32 v[[DST_MUL6:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v[[DST_MUL7:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI-DAG: v_mul_lo_u16_sdwa v[[DST_MUL7:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL6]], v[[DST_MUL7]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL6]], v[[DST_MUL7]]
; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL4]], v[[DST_MUL5]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL4]], v[[DST_MUL5]]
; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL2]], v[[DST_MUL3]]
; VI-DAG: v_or_b32_sdwa v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v[[DST_MUL0]], v[[DST_MUL1]]
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
define amdgpu_kernel void @mul_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(1)* %ina, <8 x i16> addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(1)* %ina, <8 x i16> addrspace(1)* %inb) #0 {
entry: entry:
%a = load <8 x i16>, <8 x i16> addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load <8 x i16>, <8 x i16> addrspace(1)* %inb, align 4 %gepa = getelementptr <8 x i16>, <8 x i16> addrspace(1)* %ina, i32 %idx
%gepb = getelementptr <8 x i16>, <8 x i16> addrspace(1)* %inb, i32 %idx
%a = load <8 x i16>, <8 x i16> addrspace(1)* %gepa, align 4
%b = load <8 x i16>, <8 x i16> addrspace(1)* %gepb, align 4
%mul = mul <8 x i16> %a, %b %mul = mul <8 x i16> %a, %b
store <8 x i16> %mul, <8 x i16> addrspace(1)* %out, align 4 store <8 x i16> %mul, <8 x i16> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_half: ; GCN-LABEL: {{^}}mul_half:
; NOSDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_f16_sdwa ; NOSDWA-NOT: v_mul_f16_sdwa
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Linesentry:
%a = load <8 x half>, <8 x half> addrspace(1)* %ina, align 4 %a = load <8 x half>, <8 x half> addrspace(1)* %ina, align 4
%b = load <8 x half>, <8 x half> addrspace(1)* %inb, align 4 %b = load <8 x half>, <8 x half> addrspace(1)* %inb, align 4
%mul = fmul <8 x half> %a, %b %mul = fmul <8 x half> %a, %b
store <8 x half> %mul, <8 x half> addrspace(1)* %out, align 4 store <8 x half> %mul, <8 x half> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_i8: ; GCN-LABEL: {{^}}mul_i8:
; NOSDWA: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; GFX89: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX89: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX10: v_mul_lo_u16_e64 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; SDWA-NOT: v_mul_u32_u24_sdwa ; SDWA-NOT: v_mul_u32_u24_sdwa
define amdgpu_kernel void @mul_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %ina, i8 addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %ina, i8 addrspace(1)* %inb) #0 {
entry: entry:
%a = load i8, i8 addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load i8, i8 addrspace(1)* %inb, align 4 %gepa = getelementptr i8, i8 addrspace(1)* %ina, i32 %idx
%gepb = getelementptr i8, i8 addrspace(1)* %inb, i32 %idx
%a = load i8, i8 addrspace(1)* %gepa, align 4
%b = load i8, i8 addrspace(1)* %gepb, align 4
%mul = mul i8 %a, %b %mul = mul i8 %a, %b
store i8 %mul, i8 addrspace(1)* %out, align 4 store i8 %mul, i8 addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_v2i8: ; GCN-LABEL: {{^}}mul_v2i8:
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; VI: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX9-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX9: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10: v_lshlrev_b16_e64 v{{[0-9]+}}, 8, v ; GFX10: v_lshlrev_b16_e64 v{{[0-9]+}}, 8, v
; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) #0 {
entry: entry:
%a = load <2 x i8>, <2 x i8> addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load <2 x i8>, <2 x i8> addrspace(1)* %inb, align 4 %gepa = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %ina, i32 %idx
%gepb = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %inb, i32 %idx
%a = load <2 x i8>, <2 x i8> addrspace(1)* %gepa, align 4
%b = load <2 x i8>, <2 x i8> addrspace(1)* %gepb, align 4
%mul = mul <2 x i8> %a, %b %mul = mul <2 x i8> %a, %b
store <2 x i8> %mul, <2 x i8> addrspace(1)* %out, align 4 store <2 x i8> %mul, <2 x i8> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_v4i8: ; GCN-LABEL: {{^}}mul_v4i8:
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
define amdgpu_kernel void @mul_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %ina, <4 x i8> addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %ina, <4 x i8> addrspace(1)* %inb) #0 {
entry: entry:
%a = load <4 x i8>, <4 x i8> addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load <4 x i8>, <4 x i8> addrspace(1)* %inb, align 4 %gepa = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %ina, i32 %idx
%gepb = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %inb, i32 %idx
%a = load <4 x i8>, <4 x i8> addrspace(1)* %gepa, align 4
%b = load <4 x i8>, <4 x i8> addrspace(1)* %gepb, align 4
%mul = mul <4 x i8> %a, %b %mul = mul <4 x i8> %a, %b
store <4 x i8> %mul, <4 x i8> addrspace(1)* %out, align 4 store <4 x i8> %mul, <4 x i8> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}mul_v8i8: ; GCN-LABEL: {{^}}mul_v8i8:
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; NOSDWA: v_lshlrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; VI-DAG: v_mul_u32_u24_sdwa ; VI-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX9-DAG: v_mul_lo_u16_sdwa ; GFX9-DAG: v_mul_lo_u16_sdwa
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
; GFX10-DAG: v_mul_lo_u16_e64 ; GFX10-DAG: v_mul_lo_u16_e64
define amdgpu_kernel void @mul_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(1)* %ina, <8 x i8> addrspace(1)* %inb) #0 { define amdgpu_kernel void @mul_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(1)* %ina, <8 x i8> addrspace(1)* %inb) #0 {
entry: entry:
%a = load <8 x i8>, <8 x i8> addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load <8 x i8>, <8 x i8> addrspace(1)* %inb, align 4 %gepa = getelementptr <8 x i8>, <8 x i8> addrspace(1)* %ina, i32 %idx
%gepb = getelementptr <8 x i8>, <8 x i8> addrspace(1)* %inb, i32 %idx
%a = load <8 x i8>, <8 x i8> addrspace(1)* %gepa, align 4
%b = load <8 x i8>, <8 x i8> addrspace(1)* %gepb, align 4
%mul = mul <8 x i8> %a, %b %mul = mul <8 x i8> %a, %b
store <8 x i8> %mul, <8 x i8> addrspace(1)* %out, align 4 store <8 x i8> %mul, <8 x i8> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}sitofp_v2i16_to_v2f16: ; GCN-LABEL: {{^}}sitofp_v2i16_to_v2f16:
; NOSDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA-DAG: v_cvt_f16_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
Show All 37 Linesentry:
%mac = fadd <2 x half> %mul, %b %mac = fadd <2 x half> %mul, %b
store <2 x half> %mac, <2 x half> addrspace(1)* %out, align 4 store <2 x half> %mac, <2 x half> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}immediate_mul_v2i16: ; GCN-LABEL: {{^}}immediate_mul_v2i16:
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI-DAG: v_mov_b32_e32 v[[M321:[0-9]+]], 0x141 ; VI-DAG: v_mov_b32_e32 v[[M321:[0-9]+]], 0x141
; VI-DAG: v_mov_b32_e32 v[[M123:[0-9]+]], 0x7b ; VI-DAG: v_mul_lo_u16_e32 v{{[0-9]+}}, 0x7b, v{{[0-9]+}}
; VI-DAG: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M123]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-DAG: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M321]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-DAG: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[M321]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9: s_mov_b32 s[[IMM:[0-9]+]], 0x141007b ; GFX9: s_mov_b32 s[[IMM:[0-9]+]], 0x141007b
; GFX9: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, s[[IMM]] ; GFX9: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, s[[IMM]]
; GFX10: v_pk_mul_lo_u16 v{{[0-9]+}}, 0x141007b, v{{[0-9]+}} ; GFX10: v_pk_mul_lo_u16 v{{[0-9]+}}, 0x141007b, v{{[0-9]+}}
define amdgpu_kernel void @immediate_mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 { define amdgpu_kernel void @immediate_mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
entry: entry:
%a = load <2 x i16>, <2 x i16> addrspace(1)* %in, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i32 %idx
%a = load <2 x i16>, <2 x i16> addrspace(1)* %gep, align 4
%mul = mul <2 x i16> %a, <i16 123, i16 321> %mul = mul <2 x i16> %a, <i16 123, i16 321>
store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4 store <2 x i16> %mul, <2 x i16> addrspace(1)* %out, align 4
ret void ret void
} }
; Double use of same src - should not convert it ; Double use of same src - should not convert it
; GCN-LABEL: {{^}}mulmul_v2i16: ; GCN-LABEL: {{^}}mulmul_v2i16:
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_mul_lo_u16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; NOSDWA: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; NOSDWA-NOT: v_mul_u32_u24_sdwa ; NOSDWA-NOT: v_mul_u32_u24_sdwa
; VI: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD ; VI: v_mul_lo_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9_10: v_pk_mul_lo_u16 v[[DST1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} ; GFX9_10: v_pk_mul_lo_u16 v[[DST1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v[[DST1]], v{{[0-9]+}} ; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v[[DST1]], v{{[0-9]+}}
define amdgpu_kernel void @mulmul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 { define amdgpu_kernel void @mulmul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 {
entry: entry:
%a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4 %idx = call i32 @llvm.amdgcn.workitem.id.x()
%b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4 %gepa = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %ina, i32 %idx
%gepb = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %inb, i32 %idx
%a = load <2 x i16>, <2 x i16> addrspace(1)* %gepa, align 4
%b = load <2 x i16>, <2 x i16> addrspace(1)* %gepb, align 4
%mul = mul <2 x i16> %a, %b %mul = mul <2 x i16> %a, %b
%mul2 = mul <2 x i16> %mul, %b %mul2 = mul <2 x i16> %mul, %b
store <2 x i16> %mul2, <2 x i16> addrspace(1)* %out, align 4 store <2 x i16> %mul2, <2 x i16> addrspace(1)* %out, align 4
ret void ret void
} }
; GCN-LABEL: {{^}}add_bb_v2i16: ; GCN-LABEL: {{^}}add_bb_v2i16:
; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa ; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa
▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Linesbb2: ; preds = %bb1
br label %bb11 br label %bb11
bb11: ; preds = %bb10, %bb2 bb11: ; preds = %bb10, %bb2
%tmp12 = phi <2 x i32> [ %tmp6, %bb2 ], [ %tmp, %bb1 ] %tmp12 = phi <2 x i32> [ %tmp6, %bb2 ], [ %tmp, %bb1 ]
store volatile <2 x i32> %tmp12, <2 x i32> addrspace(1)* undef store volatile <2 x i32> %tmp12, <2 x i32> addrspace(1)* undef
br label %bb1 br label %bb1
} }
declare i32 @llvm.amdgcn.workitem.id.x()
attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" } attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
llvm/test/CodeGen/AMDGPU/idot4u.ll