This is an archive of the discontinued LLVM Phabricator instance.

Differential D92767

[AMDGPU] Resolve issues when picking between ds_read/write and ds_read2/write2
ClosedPublic

Authored by mbrkusanin on Dec 7 2020, 7:50 AM.

Details

Reviewers
foad
arsenm
Commits
rG0c7cce54eba3: [AMDGPU] Resolve issues when picking between ds_read/write and ds_read2/write2
Summary

Both ds_read_b128 and ds_read2_b64 are valid for 128bit 16-byte aligned
loads but the one that will be selected is determined either by the order in
tablegen or by the AddedComplexity attribute. Currently ds_read_b128 has
priority.

While ds_read2_b64 has lower alignment requirements, we cannot always
restrict ds_read_b128 to 16-byte alignment because of unaligned-access-mode
option. This was causing ds_read_b128 to be selected for 8-byte aligned
loads regardles of chosen access mode.

To resolve this we use two patterns for selecting ds_read_b128. One
requires alignment of 16-byte and the other requires
unaligned-access-mode option.

Same goes for ds_write2_b64 and ds_write_b128.

Diff Detail

Unit TestsFailed

TimeTest
50 msx64 windows > LLVM.CodeGen/XCore::threads.ll

Event Timeline

mbrkusanin created this revision.Dec 7 2020, 7:50 AM
Herald added subscribers: kerbowa, hiraditya, t-tye and 6 others. · View Herald TranscriptDec 7 2020, 7:50 AM
mbrkusanin requested review of this revision.Dec 7 2020, 7:50 AM
Herald added a subscriber: wdng. · View Herald TranscriptDec 7 2020, 7:50 AM
foad added inline comments.Dec 7 2020, 8:00 AM
llvm/lib/Target/AMDGPU/DSInstructions.td
683

Can't we relax this predicate to "isGFX7Plus" instead of duplicating the patterns?

Harbormaster completed remote builds in B81299: Diff 309917.Dec 7 2020, 8:21 AM
mbrkusanin updated this revision to Diff 310111.Dec 8 2020, 1:55 AM
mbrkusanin marked an inline comment as done.
mbrkusanin added inline comments.
llvm/lib/Target/AMDGPU/DSInstructions.td
683

Right. That way we do not increase number of patterns.

foad accepted this revision.Dec 10 2020, 3:34 AM

Looks good, thanks!

This revision is now accepted and ready to land.Dec 10 2020, 3:34 AM
Closed by commit rG0c7cce54eba3: [AMDGPU] Resolve issues when picking between ds_read/write and ds_read2/write2 (authored by mbrkusanin). · Explain WhyDec 10 2020, 3:50 AM
This revision was automatically updated to reflect the committed changes.
mbrkusanin added a commit: rG0c7cce54eba3: [AMDGPU] Resolve issues when picking between ds_read/write and ds_read2/write2.

Revision Contents

PathSize
llvm/
lib/
Target/
AMDGPU/
3 lines
29 lines
test/
CodeGen/
AMDGPU/
GlobalISel/
22 lines
32 lines
2 lines
2 lines
32 lines
2 lines
4 lines

Diff 309917

llvm/lib/Target/AMDGPU/AMDGPU.td

Show First 20 Lines • Show All 1,293 Lines • ▼ Show 20 Lines
def EnableLateCFGStructurize : Predicate< def EnableLateCFGStructurize : Predicate<
"EnableLateStructurizeCFG">; "EnableLateStructurizeCFG">;
def EnableFlatScratch : Predicate<"Subtarget->enableFlatScratch()">; def EnableFlatScratch : Predicate<"Subtarget->enableFlatScratch()">;
def DisableFlatScratch : Predicate<"!Subtarget->enableFlatScratch()">; def DisableFlatScratch : Predicate<"!Subtarget->enableFlatScratch()">;
def HasUnalignedAccessMode : Predicate<"Subtarget->hasUnalignedAccessMode()">,
AssemblerPredicate<(all_of FeatureUnalignedAccessMode)>;
// Include AMDGPU TD files // Include AMDGPU TD files
include "SISchedule.td" include "SISchedule.td"
include "GCNProcessors.td" include "GCNProcessors.td"
include "AMDGPUInstrInfo.td" include "AMDGPUInstrInfo.td"
include "SIRegisterInfo.td" include "SIRegisterInfo.td"
include "AMDGPURegisterBanks.td" include "AMDGPURegisterBanks.td"
include "AMDGPUInstructions.td" include "AMDGPUInstructions.td"
include "SIInstrInfo.td" include "SIInstrInfo.td"
include "AMDGPUCallingConv.td" include "AMDGPUCallingConv.td"
include "AMDGPUSearchableTables.td" include "AMDGPUSearchableTables.td"

llvm/lib/Target/AMDGPU/DSInstructions.td

Show First 20 Lines • Show All 674 Lines • ▼ Show 20 Lines
defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">; defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
let AddedComplexity = 100 in { let AddedComplexity = 100 in {
foreach vt = VReg_64.RegTypes in { foreach vt = VReg_64.RegTypes in {
defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">; defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
} }
let SubtargetPredicate = isGFX7GFX8 in { let SubtargetPredicate = isGFX7GFX8 in {
foadUnsubmitted
Done
ReplyInline Actions

Can't we relax this predicate to "isGFX7Plus" instead of duplicating the patterns?

foad: Can't we relax this predicate to "isGFX7Plus" instead of duplicating the patterns?
mbrkusaninAuthorUnsubmitted
Done
ReplyInline Actions

Right. That way we do not increase number of patterns.

mbrkusanin: Right. That way we do not increase number of patterns.
foreach vt = VReg_96.RegTypes in { foreach vt = VReg_96.RegTypes in {
defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">; defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
} }
foreach vt = VReg_128.RegTypes in { foreach vt = VReg_128.RegTypes in {
defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">; defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
} }
} }
let SubtargetPredicate = isGFX9Plus in { let SubtargetPredicate = isGFX9Plus in {
foreach vt = VReg_96.RegTypes in { foreach vt = VReg_96.RegTypes in {
defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
}
foreach vt = VReg_128.RegTypes in {
defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
}
let SubtargetPredicate = HasUnalignedAccessMode in {
foreach vt = VReg_96.RegTypes in {
defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">; defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
} }
foreach vt = VReg_128.RegTypes in { foreach vt = VReg_128.RegTypes in {
defm : DSReadPat_mc <DS_READ_B128, vt, "load_local">; defm : DSReadPat_mc <DS_READ_B128, vt, "load_local">;
} }
} } // End SubtargetPredicate = HasUnalignedAccessMode
} // End SubtargetPredicate = isGFX9Plus
} // End AddedComplexity = 100 } // End AddedComplexity = 100
let OtherPredicates = [D16PreservesUnusedBits] in { let OtherPredicates = [D16PreservesUnusedBits] in {
def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>; def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;
def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>; def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;
def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>; def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;
def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>; def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines
defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">; defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
} }
} }
let SubtargetPredicate = isGFX9Plus in { let SubtargetPredicate = isGFX9Plus in {
foreach vt = VReg_96.RegTypes in { foreach vt = VReg_96.RegTypes in {
defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
}
foreach vt = VReg_128.RegTypes in {
defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
}
let SubtargetPredicate = HasUnalignedAccessMode in {
foreach vt = VReg_96.RegTypes in {
defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">; defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
} }
foreach vt = VReg_128.RegTypes in { foreach vt = VReg_128.RegTypes in {
defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_local">; defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_local">;
} }
} } // End SubtargetPredicate = HasUnalignedAccessMode
} // End SubtargetPredicate = isGFX9Plus
} // End AddedComplexity = 100 } // End AddedComplexity = 100
class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat < class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat <
(frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value), (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
(inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds)) (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
>; >;
multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> { multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
let OtherPredicates = [LDSRequiresM0Init] in { let OtherPredicates = [LDSRequiresM0Init] in {
def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>; def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
▲ Show 20 Lines • Show All 475 Lines • Show Last 20 Lines

llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir

Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines bb.0:
; GFX7: liveins: $vgpr0 ; GFX7: liveins: $vgpr0
; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX7: $m0 = S_MOV_B32 -1 ; GFX7: $m0 = S_MOV_B32 -1
; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]] ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
; GFX9-LABEL: name: load_local_v4s32_align_8 ; GFX9-LABEL: name: load_local_v4s32_align_8
; GFX9: liveins: $vgpr0 ; GFX9: liveins: $vgpr0
; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0 %0:vgpr(p3) = COPY $vgpr0
%1:vgpr(<4 x s32>) = G_LOAD %0 :: (load 16, align 8, addrspace 3) %1:vgpr(<4 x s32>) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
$vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1 $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
... ...
--- ---
Show All 10 Lines bb.0:
; GFX7: liveins: $vgpr0 ; GFX7: liveins: $vgpr0
; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX7: $m0 = S_MOV_B32 -1 ; GFX7: $m0 = S_MOV_B32 -1
; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 50, 51, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 50, 51, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]] ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
; GFX9-LABEL: name: load_local_v4s32_align_8_offset_160 ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_160
; GFX9: liveins: $vgpr0 ; GFX9: liveins: $vgpr0
; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 400, 0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 50, 51, 0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0 %0:vgpr(p3) = COPY $vgpr0
%1:vgpr(s32) = G_CONSTANT i32 400 %1:vgpr(s32) = G_CONSTANT i32 400
%2:vgpr(p3) = G_PTR_ADD %0, %1 %2:vgpr(p3) = G_PTR_ADD %0, %1
%3:vgpr(<4 x s32>) = G_LOAD %2 :: (load 16, align 8, addrspace 3) %3:vgpr(<4 x s32>) = G_LOAD %2 :: (load 16, align 8, addrspace 3)
$vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3 $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
... ...
Show All 14 Lines bb.0:
; GFX7: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4000, implicit $exec ; GFX7: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4000, implicit $exec
; GFX7: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec ; GFX7: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
; GFX7: $m0 = S_MOV_B32 -1 ; GFX7: $m0 = S_MOV_B32 -1
; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 %2, 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 %2, 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]] ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
; GFX9-LABEL: name: load_local_v4s32_align_8_offset_320 ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_320
; GFX9: liveins: $vgpr0 ; GFX9: liveins: $vgpr0
; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 4000, 0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4000, implicit $exec
; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[V_ADD_U32_e64_]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0 %0:vgpr(p3) = COPY $vgpr0
%1:vgpr(s32) = G_CONSTANT i32 4000 %1:vgpr(s32) = G_CONSTANT i32 4000
%2:vgpr(p3) = G_PTR_ADD %0, %1 %2:vgpr(p3) = G_PTR_ADD %0, %1
%3:vgpr(<4 x s32>) = G_LOAD %2 :: (load 16, align 8, addrspace 3) %3:vgpr(<4 x s32>) = G_LOAD %2 :: (load 16, align 8, addrspace 3)
$vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3 $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
... ...
Show All 12 Lines bb.0:
; GFX7: liveins: $vgpr0 ; GFX7: liveins: $vgpr0
; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX7: $m0 = S_MOV_B32 -1 ; GFX7: $m0 = S_MOV_B32 -1
; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]] ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
; GFX9-LABEL: name: load_local_v2s64 ; GFX9-LABEL: name: load_local_v2s64
; GFX9: liveins: $vgpr0 ; GFX9: liveins: $vgpr0
; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0 %0:vgpr(p3) = COPY $vgpr0
%1:vgpr(<2 x s64>) = G_LOAD %0 :: (load 16, align 8, addrspace 3) %1:vgpr(<2 x s64>) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
$vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1 $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
... ...
--- ---
Show All 38 Lines bb.0:
; GFX7: liveins: $vgpr0 ; GFX7: liveins: $vgpr0
; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX7: $m0 = S_MOV_B32 -1 ; GFX7: $m0 = S_MOV_B32 -1
; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]] ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
; GFX9-LABEL: name: load_local_s128 ; GFX9-LABEL: name: load_local_s128
; GFX9: liveins: $vgpr0 ; GFX9: liveins: $vgpr0
; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3) ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
%0:vgpr(p3) = COPY $vgpr0 %0:vgpr(p3) = COPY $vgpr0
%1:vgpr(s128) = G_LOAD %0 :: (load 16, align 8, addrspace 3) %1:vgpr(s128) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
$vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1 $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
... ...
--- ---
Show All 25 Lines

llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll

; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s
; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s
; GCN-LABEL: test_local_misaligned_v2: ; GCN-LABEL: test_local_misaligned_v2:
; GCN-DAG: ds_read2_b32 ; GCN-DAG: ds_read2_b32
; GCN-DAG: ds_write2_b32 ; GCN-DAG: ds_write2_b32
define amdgpu_kernel void @test_local_misaligned_v2(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_misaligned_v2(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <2 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <2 x i32> addrspace(3)*
%load = load <2 x i32>, <2 x i32> addrspace(3)* %ptr, align 4 %load = load <2 x i32>, <2 x i32> addrspace(3)* %ptr, align 4
%v1 = extractelement <2 x i32> %load, i32 0 %v1 = extractelement <2 x i32> %load, i32 0
%v2 = extractelement <2 x i32> %load, i32 1 %v2 = extractelement <2 x i32> %load, i32 1
%v3 = insertelement <2 x i32> undef, i32 %v2, i32 0 %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
%v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1 %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
store <2 x i32> %v4, <2 x i32> addrspace(3)* %ptr, align 4 store <2 x i32> %v4, <2 x i32> addrspace(3)* %ptr, align 4
ret void ret void
} }
; GCN-LABEL: test_local_misaligned_v4: ; GCN-LABEL: test_local_misaligned_v4:
; GCN-DAG: ds_read2_b32 ; ALIGNED-DAG: ds_read2_b32
; GCN-DAG: ds_read2_b32 ; ALIGNED-DAG: ds_read2_b32
; GCN-DAG: ds_write2_b32 ; ALIGNED-DAG: ds_write2_b32
; GCN-DAG: ds_write2_b32 ; ALIGNED-DAG: ds_write2_b32
; UNALIGNED-DAG: ds_read_b128 ; UNALIGNED-DAG: ds_read_b128
; UNALIGNED-DAG: ds_write_b128 ; UNALIGNED-DAG: ds_write_b128
define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
%load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4 %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
%v1 = extractelement <4 x i32> %load, i32 0 %v1 = extractelement <4 x i32> %load, i32 0
%v2 = extractelement <4 x i32> %load, i32 1 %v2 = extractelement <4 x i32> %load, i32 1
%v3 = extractelement <4 x i32> %load, i32 2 %v3 = extractelement <4 x i32> %load, i32 2
%v4 = extractelement <4 x i32> %load, i32 3 %v4 = extractelement <4 x i32> %load, i32 3
%v5 = insertelement <4 x i32> undef, i32 %v4, i32 0 %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
%v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1 %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
%v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2 %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
%v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3 %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
store <4 x i32> %v8, <4 x i32> addrspace(3)* %ptr, align 4 store <4 x i32> %v8, <4 x i32> addrspace(3)* %ptr, align 4
ret void ret void
} }
; GCN-LABEL: test_local_misaligned_v3: ; GCN-LABEL: test_local_misaligned_v3:
; GCN-DAG: ds_read2_b32 ; ALIGNED-DAG: ds_read2_b32
; GCN-DAG: ds_read_b32 ; ALIGNED-DAG: ds_read_b32
; GCN-DAG: ds_write2_b32 ; ALIGNED-DAG: ds_write2_b32
; GCN-DAG: ds_write_b32 ; ALIGNED-DAG: ds_write_b32
; UNALIGNED-DAG: ds_read_b96 ; UNALIGNED-DAG: ds_read_b96
; UNALIGNED-DAG: ds_write_b96 ; UNALIGNED-DAG: ds_write_b96
define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <3 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <3 x i32> addrspace(3)*
%load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 4 %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 4
Show All 39 Linesbb:
%v5 = insertelement <3 x i32> undef, i32 %v3, i32 0 %v5 = insertelement <3 x i32> undef, i32 %v3, i32 0
%v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1 %v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1
%v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2 %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
store <3 x i32> %v7, <3 x i32> addrspace(3)* %ptr, align 16 store <3 x i32> %v7, <3 x i32> addrspace(3)* %ptr, align 16
ret void ret void
} }
; GCN-LABEL: test_local_v4_aligned8: ; GCN-LABEL: test_local_v4_aligned8:
; GCN-DAG: ds_read_b128 ; ALIGNED-DAG: ds_read2_b64
; GCN-DAG: ds_write_b128 ; ALIGNED-DAG: ds_write2_b64
; UNALIGNED-DAG: ds_read_b128
; UNALIGNED-DAG: ds_write_b128
define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
%load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8 %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8
%v1 = extractelement <4 x i32> %load, i32 0 %v1 = extractelement <4 x i32> %load, i32 0
%v2 = extractelement <4 x i32> %load, i32 1 %v2 = extractelement <4 x i32> %load, i32 1
Show All 11 Lines

llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll

Show First 20 Lines • Show All 261 Lines • ▼ Show 20 Lines; GFX7-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4 %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
ret <4 x i32> %load ret <4 x i32> %load
} }
define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) { define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) {
; GFX9-LABEL: load_lds_v4i32_align8: ; GFX9-LABEL: load_lds_v4i32_align8:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_b128 v[0:3], v0 ; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX9-NEXT: s_setpc_b64 s[30:31]
; ;
; GFX7-LABEL: load_lds_v4i32_align8: ; GFX7-LABEL: load_lds_v4i32_align8:
; GFX7: ; %bb.0: ; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1 ; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 ; GFX7-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
Show All 24 Lines

llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll

Show First 20 Lines • Show All 244 Lines • ▼ Show 20 Lines
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x24 ; GFX9-NEXT: s_load_dword s4, s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x34
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, s4 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3 ; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: ds_write_b128 v4, v[0:3] ; GFX9-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX7-LABEL: store_lds_v4i32_align8: ; GFX7-LABEL: store_lds_v4i32_align8:
; GFX7: ; %bb.0: ; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x9 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 m0, -1 ; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
▲ Show 20 Lines • Show All 41 Lines • Show Last 20 Lines

llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll

; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s ; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s ; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s
; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED,VECT %s ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s
; GCN-LABEL: test_local_misaligned_v2: ; GCN-LABEL: test_local_misaligned_v2:
; GCN-DAG: ds_read2_b32 ; GCN-DAG: ds_read2_b32
; GCN-DAG: ds_write2_b32 ; GCN-DAG: ds_write2_b32
define amdgpu_kernel void @test_local_misaligned_v2(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_misaligned_v2(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <2 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <2 x i32> addrspace(3)*
%load = load <2 x i32>, <2 x i32> addrspace(3)* %ptr, align 4 %load = load <2 x i32>, <2 x i32> addrspace(3)* %ptr, align 4
%v1 = extractelement <2 x i32> %load, i32 0 %v1 = extractelement <2 x i32> %load, i32 0
%v2 = extractelement <2 x i32> %load, i32 1 %v2 = extractelement <2 x i32> %load, i32 1
%v3 = insertelement <2 x i32> undef, i32 %v2, i32 0 %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
%v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1 %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
store <2 x i32> %v4, <2 x i32> addrspace(3)* %ptr, align 4 store <2 x i32> %v4, <2 x i32> addrspace(3)* %ptr, align 4
ret void ret void
} }
; GCN-LABEL: test_local_misaligned_v4: ; GCN-LABEL: test_local_misaligned_v4:
; GCN-DAG: ds_read2_b32 ; ALIGNED-DAG: ds_read2_b32
; GCN-DAG: ds_read2_b32 ; ALIGNED-DAG: ds_read2_b32
; GCN-DAG: ds_write2_b32 ; ALIGNED-DAG: ds_write2_b32
; GCN-DAG: ds_write2_b32 ; ALIGNED-DAG: ds_write2_b32
; UNALIGNED-DAG: ds_read_b128 ; UNALIGNED-DAG: ds_read_b128
; UNALIGNED-DAG: ds_write_b128 ; UNALIGNED-DAG: ds_write_b128
define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
%load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4 %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
%v1 = extractelement <4 x i32> %load, i32 0 %v1 = extractelement <4 x i32> %load, i32 0
%v2 = extractelement <4 x i32> %load, i32 1 %v2 = extractelement <4 x i32> %load, i32 1
%v3 = extractelement <4 x i32> %load, i32 2 %v3 = extractelement <4 x i32> %load, i32 2
%v4 = extractelement <4 x i32> %load, i32 3 %v4 = extractelement <4 x i32> %load, i32 3
%v5 = insertelement <4 x i32> undef, i32 %v4, i32 0 %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
%v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1 %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
%v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2 %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
%v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3 %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
store <4 x i32> %v8, <4 x i32> addrspace(3)* %ptr, align 4 store <4 x i32> %v8, <4 x i32> addrspace(3)* %ptr, align 4
ret void ret void
} }
; GCN-LABEL: test_local_misaligned_v3: ; GCN-LABEL: test_local_misaligned_v3:
; GCN-DAG: ds_read2_b32 ; ALIGNED-DAG: ds_read2_b32
; GCN-DAG: ds_read_b32 ; ALIGNED-DAG: ds_read_b32
; GCN-DAG: ds_write2_b32 ; ALIGNED-DAG: ds_write2_b32
; GCN-DAG: ds_write_b32 ; ALIGNED-DAG: ds_write_b32
; UNALIGNED-DAG: ds_read_b96 ; UNALIGNED-DAG: ds_read_b96
; UNALIGNED-DAG: ds_write_b96 ; UNALIGNED-DAG: ds_write_b96
define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <3 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <3 x i32> addrspace(3)*
%load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 4 %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 4
▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Linesbb:
%v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1 %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
%v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2 %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
%v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3 %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
store <4 x i32> %v8, <4 x i32>* %ptr, align 16 store <4 x i32> %v8, <4 x i32>* %ptr, align 16
ret void ret void
} }
; GCN-LABEL: test_local_v4_aligned8: ; GCN-LABEL: test_local_v4_aligned8:
; GCN-DAG: ds_read_b128 ; ALIGNED-DAG: ds_read2_b64
; GCN-DAG: ds_write_b128 ; ALIGNED-DAG: ds_write2_b64
; UNALIGNED-DAG: ds_read_b128
; UNALIGNED-DAG: ds_write_b128
define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) { define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
bb: bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x() %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
%ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)* %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
%load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8 %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8
%v1 = extractelement <4 x i32> %load, i32 0 %v1 = extractelement <4 x i32> %load, i32 0
%v2 = extractelement <4 x i32> %load, i32 1 %v2 = extractelement <4 x i32> %load, i32 1
Show All 36 Lines

llvm/test/CodeGen/AMDGPU/load-local.128.ll

Show First 20 Lines • Show All 319 Lines • ▼ Show 20 Lines; GFX6-NEXT: s_setpc_b64 s[30:31]
%load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4 %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
ret <4 x i32> %load ret <4 x i32> %load
} }
define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) { define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) {
; GFX9-LABEL: load_lds_v4i32_align8: ; GFX9-LABEL: load_lds_v4i32_align8:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_b128 v[0:3], v0 ; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX9-NEXT: s_setpc_b64 s[30:31]
; ;
; GFX7-LABEL: load_lds_v4i32_align8: ; GFX7-LABEL: load_lds_v4i32_align8:
; GFX7: ; %bb.0: ; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1 ; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 ; GFX7-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
▲ Show 20 Lines • Show All 44 Lines • Show Last 20 Lines

llvm/test/CodeGen/AMDGPU/store-local.128.ll

Show First 20 Lines • Show All 334 Lines • ▼ Show 20 Lines
define amdgpu_kernel void @store_lds_v4i32_align8(<4 x i32> addrspace(3)* %out, <4 x i32> %x) { define amdgpu_kernel void @store_lds_v4i32_align8(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
; GFX9-LABEL: store_lds_v4i32_align8: ; GFX9-LABEL: store_lds_v4i32_align8:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x24 ; GFX9-NEXT: s_load_dword s4, s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x34
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, s4 ; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v3, s3 ; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: ds_write_b128 v4, v[0:3] ; GFX9-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX7-LABEL: store_lds_v4i32_align8: ; GFX7-LABEL: store_lds_v4i32_align8:
; GFX7: ; %bb.0: ; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s4, s[0:1], 0x9 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x9
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 m0, -1 ; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
▲ Show 20 Lines • Show All 69 Lines • Show Last 20 Lines