diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp @@ -172,6 +172,18 @@ return false; } + // Go through each LDS, and if it is under-aligned, then find its natural + // alignment, and update it accordingly. This is will make sure that the + // members of sorted structure are properly aligned, and hence it will + // further reduce the probability of unaligned LDS access. + for (auto *GV : FoundLocalVars) { + Align Alignment = AMDGPU::getAlign(DL, GV); + Align FixedAlignment = AMDGPU::fixAlignment(GV, Alignment, DL); + if (Alignment != FixedAlignment) { + GV->setAlignment(FixedAlignment); + } + } + // Sort by alignment, descending, to minimise padding. // On ties, sort by size, descending, then by name, lexicographical. llvm::stable_sort( diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h @@ -42,6 +42,10 @@ /// \returns true if all uses of \p U end up in a function \p F. bool isUsedOnlyFromFunction(const User *U, const Function *F); +// Fix natural alignment for LDS global `GV` based on its size. +Align fixAlignment(const GlobalVariable *GV, Align Alignment, + const DataLayout &DL); + } // end namespace AMDGPU } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp @@ -149,6 +149,28 @@ return UsedList; } +// Fix natural alignment for LDS global `GV` based on its size. +Align fixAlignment(const GlobalVariable *GV, Align Alignment, + const DataLayout &DL) { + TypeSize GVSize = DL.getTypeAllocSize(GV->getValueType()); + + if (GVSize > 8) { + // We might want to use a b96 or b128 load/store + Alignment = std::max(Alignment, Align(16)); + } else if (GVSize > 4) { + // We might want to use a b64 load/store + Alignment = std::max(Alignment, Align(8)); + } else if (GVSize > 2) { + // We might want to use a b32 load/store + Alignment = std::max(Alignment, Align(4)); + } else if (GVSize > 1) { + // We might want to use a b16 load/store + Alignment = std::max(Alignment, Align(2)); + } + + return Alignment; +} + } // end namespace AMDGPU } // end namespace llvm diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-value.ll @@ -9,7 +9,7 @@ ; CHECK-LABEL: use_lds_globals: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 -; CHECK-NEXT: v_mov_b32_e32 v0, 8 +; CHECK-NEXT: v_mov_b32_e32 v0, 4 ; CHECK-NEXT: s_mov_b32 m0, -1 ; CHECK-NEXT: ds_read_b32 v3, v0 ; CHECK-NEXT: v_mov_b32_e32 v2, 9 @@ -19,7 +19,7 @@ ; CHECK-NEXT: v_mov_b32_e32 v0, s0 ; CHECK-NEXT: v_mov_b32_e32 v1, s1 ; CHECK-NEXT: flat_store_dword v[0:1], v3 -; CHECK-NEXT: v_mov_b32_e32 v0, 0 +; CHECK-NEXT: v_mov_b32_e32 v0, 0x200 ; CHECK-NEXT: ds_write_b32 v0, v2 ; CHECK-NEXT: s_endpgm entry: diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll --- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll +++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll @@ -1007,10 +1007,9 @@ define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) { ; CI-LABEL: load_misaligned64_constant_offsets: ; CI: ; %bb.0: -; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: v_mov_b32_e32 v0, 0 ; CI-NEXT: s_mov_b32 m0, -1 -; CI-NEXT: ds_read2_b32 v[0:1], v2 offset1:1 -; CI-NEXT: ds_read2_b32 v[2:3], v2 offset0:2 offset1:3 +; CI-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 ; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; CI-NEXT: s_mov_b32 s3, 0xf000 ; CI-NEXT: s_mov_b32 s2, -1 @@ -1023,8 +1022,7 @@ ; GFX9-ALIGNED-LABEL: load_misaligned64_constant_offsets: ; GFX9-ALIGNED: ; %bb.0: ; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v4, 0 -; GFX9-ALIGNED-NEXT: ds_read2_b32 v[0:1], v4 offset1:1 -; GFX9-ALIGNED-NEXT: ds_read2_b32 v[2:3], v4 offset0:2 offset1:3 +; GFX9-ALIGNED-NEXT: ds_read2_b64 v[0:3], v4 offset1:1 ; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-ALIGNED-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 @@ -1035,7 +1033,7 @@ ; GFX9-UNALIGNED-LABEL: load_misaligned64_constant_offsets: ; GFX9-UNALIGNED: ; %bb.0: ; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v4, 0 -; GFX9-UNALIGNED-NEXT: ds_read2_b64 v[0:3], v4 offset1:1 +; GFX9-UNALIGNED-NEXT: ds_read_b128 v[0:3], v4 ; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-UNALIGNED-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 @@ -1054,11 +1052,10 @@ define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspace(1)* %out) { ; CI-LABEL: load_misaligned64_constant_large_offsets: ; CI: ; %bb.0: -; CI-NEXT: v_mov_b32_e32 v0, 0x4000 -; CI-NEXT: v_mov_b32_e32 v2, 0x7ff8 +; CI-NEXT: v_mov_b32_e32 v2, 0 ; CI-NEXT: s_mov_b32 m0, -1 -; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 -; CI-NEXT: ds_read2_b32 v[2:3], v2 offset1:1 +; CI-NEXT: ds_read_b64 v[0:1], v2 offset:16384 +; CI-NEXT: ds_read_b64 v[2:3], v2 offset:32760 ; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; CI-NEXT: s_mov_b32 s3, 0xf000 ; CI-NEXT: s_mov_b32 s2, -1 @@ -1070,12 +1067,10 @@ ; ; GFX9-LABEL: load_misaligned64_constant_large_offsets: ; GFX9: ; %bb.0: -; GFX9-NEXT: v_mov_b32_e32 v0, 0x4000 -; GFX9-NEXT: v_mov_b32_e32 v2, 0x7ff8 -; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1 -; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset1:1 -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: ds_read_b64 v[0:1], v4 offset:16384 +; GFX9-NEXT: ds_read_b64 v[2:3], v4 offset:32760 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2_offset_order.ll b/llvm/test/CodeGen/AMDGPU/ds_read2_offset_order.ll --- a/llvm/test/CodeGen/AMDGPU/ds_read2_offset_order.ll +++ b/llvm/test/CodeGen/AMDGPU/ds_read2_offset_order.ll @@ -6,10 +6,10 @@ ; offset0 is larger than offset1 ; SI-LABEL: {{^}}offset_order: -; SI-DAG: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset1:14{{$}} -; SI-DAG: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:2 offset1:3 -; SI-DAG: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1024 +; SI-DAG: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} +; SI-DAG: ds_read_b64 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset:8 ; SI-DAG: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:11 offset1:12 +; SI-DAG: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:6 offset1:248 define amdgpu_kernel void @offset_order(float addrspace(1)* %out) { entry: %ptr0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 0 diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll --- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll +++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll @@ -820,33 +820,32 @@ define amdgpu_kernel void @store_misaligned64_constant_offsets() { ; CI-LABEL: store_misaligned64_constant_offsets: ; CI: ; %bb.0: -; CI-NEXT: v_mov_b32_e32 v0, 0 -; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: s_movk_i32 s0, 0x7b +; CI-NEXT: s_mov_b32 s1, 0 +; CI-NEXT: v_mov_b32_e32 v0, s0 +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: s_mov_b32 m0, -1 -; CI-NEXT: ds_write2_b32 v0, v1, v0 offset1:1 -; CI-NEXT: ds_write2_b32 v0, v1, v0 offset0:2 offset1:3 +; CI-NEXT: ds_write2_b64 v2, v[0:1], v[0:1] offset1:1 ; CI-NEXT: s_endpgm ; ; GFX9-ALIGNED-LABEL: store_misaligned64_constant_offsets: ; GFX9-ALIGNED: ; %bb.0: -; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 0 -; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, 0x7b -; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v0 offset1:1 -; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v0 offset0:2 offset1:3 +; GFX9-ALIGNED-NEXT: s_movk_i32 s0, 0x7b +; GFX9-ALIGNED-NEXT: s_mov_b32 s1, 0 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-ALIGNED-NEXT: ds_write2_b64 v2, v[0:1], v[0:1] offset1:1 ; GFX9-ALIGNED-NEXT: s_endpgm ; ; GFX9-UNALIGNED-LABEL: store_misaligned64_constant_offsets: ; GFX9-UNALIGNED: ; %bb.0: -; GFX9-UNALIGNED-NEXT: s_movk_i32 s0, 0x7b -; GFX9-UNALIGNED-NEXT: s_mov_b32 s1, 0 -; GFX9-UNALIGNED-NEXT: s_mov_b32 s2, s0 -; GFX9-UNALIGNED-NEXT: s_mov_b32 s3, s1 -; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, s0 -; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, s2 -; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v4, 0 -; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, s1 -; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, s3 -; GFX9-UNALIGNED-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:1 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0x7b +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-UNALIGNED-NEXT: ds_write_b128 v1, v[0:3] ; GFX9-UNALIGNED-NEXT: s_endpgm store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4 @@ -858,23 +857,25 @@ define amdgpu_kernel void @store_misaligned64_constant_large_offsets() { ; CI-LABEL: store_misaligned64_constant_large_offsets: ; CI: ; %bb.0: -; CI-NEXT: v_mov_b32_e32 v0, 0x4000 -; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: s_movk_i32 s0, 0x7b +; CI-NEXT: s_mov_b32 s1, 0 +; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: s_mov_b32 m0, -1 -; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 -; CI-NEXT: v_mov_b32_e32 v0, 0x7ff8 -; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; CI-NEXT: ds_write_b64 v2, v[0:1] offset:16384 +; CI-NEXT: ds_write_b64 v2, v[0:1] offset:32760 ; CI-NEXT: s_endpgm ; ; GFX9-LABEL: store_misaligned64_constant_large_offsets: ; GFX9: ; %bb.0: -; GFX9-NEXT: v_mov_b32_e32 v0, 0x4000 -; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b +; GFX9-NEXT: s_movk_i32 s0, 0x7b +; GFX9-NEXT: s_mov_b32 s1, 0 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 -; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 -; GFX9-NEXT: v_mov_b32_e32 v0, 0x7ff8 -; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1 +; GFX9-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:16384 +; GFX9-NEXT: ds_write_b64 v2, v[0:1] offset:32760 ; GFX9-NEXT: s_endpgm store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4 store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4 diff --git a/llvm/test/CodeGen/AMDGPU/fix-lds-alignment1.ll b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment1.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment1.ll @@ -0,0 +1,36 @@ +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s | FileCheck %s + +; Properly aligned, same size as alignment. +; CHECK: %llvm.amdgcn.kernel.k0.lds.t = type { [16 x i8], [8 x i8], [4 x i8], [2 x i8], [1 x i8] } +; CHECK: @llvm.amdgcn.kernel.k0.lds = internal addrspace(3) global %llvm.amdgcn.kernel.k0.lds.t undef, align 16 + +; CHECK-NOT: @lds.size.1.align.1 +; CHECK-NOT: @lds.size.2.align.2 +; CHECK-NOT: @lds.size.4.align.4 +; CHECK-NOT: @lds.size.8.align.8 +; CHECK-NOT: @lds.size.16.align.16 +@lds.size.1.align.1 = internal unnamed_addr addrspace(3) global [1 x i8] undef, align 1 +@lds.size.2.align.2 = internal unnamed_addr addrspace(3) global [2 x i8] undef, align 2 +@lds.size.4.align.4 = internal unnamed_addr addrspace(3) global [4 x i8] undef, align 4 +@lds.size.8.align.8 = internal unnamed_addr addrspace(3) global [8 x i8] undef, align 8 +@lds.size.16.align.16 = internal unnamed_addr addrspace(3) global [16 x i8] undef, align 16 + +define amdgpu_kernel void @k0() { + %lds.size.1.align.1.bc = bitcast [1 x i8] addrspace(3)* @lds.size.1.align.1 to i8 addrspace(3)* + store i8 1, i8 addrspace(3)* %lds.size.1.align.1.bc, align 1 + + %lds.size.2.align.2.bc = bitcast [2 x i8] addrspace(3)* @lds.size.2.align.2 to i8 addrspace(3)* + store i8 2, i8 addrspace(3)* %lds.size.2.align.2.bc, align 2 + + %lds.size.4.align.4.bc = bitcast [4 x i8] addrspace(3)* @lds.size.4.align.4 to i8 addrspace(3)* + store i8 3, i8 addrspace(3)* %lds.size.4.align.4.bc, align 4 + + %lds.size.8.align.8.bc = bitcast [8 x i8] addrspace(3)* @lds.size.8.align.8 to i8 addrspace(3)* + store i8 4, i8 addrspace(3)* %lds.size.8.align.8.bc, align 8 + + %lds.size.16.align.16.bc = bitcast [16 x i8] addrspace(3)* @lds.size.16.align.16 to i8 addrspace(3)* + store i8 5, i8 addrspace(3)* %lds.size.16.align.16.bc, align 16 + + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/fix-lds-alignment2.ll b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment2.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment2.ll @@ -0,0 +1,37 @@ +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s | FileCheck %s + +; Different properly aligned values, but same size of 1. +; CHECK: %llvm.amdgcn.kernel.k0.lds.t = type { [1 x i8], [7 x i8], [1 x i8], [3 x i8], [1 x i8], [1 x i8], [1 x i8], [1 x i8] } +; CHECK: @llvm.amdgcn.kernel.k0.lds = internal addrspace(3) global %llvm.amdgcn.kernel.k0.lds.t undef, align 16 + +; CHECK-NOT: @size.1.align.1 +; CHECK-NOT: @size.1.align.2 +; CHECK-NOT: @size.1.align.4 +; CHECK-NOT: @size.1.align.8 +; CHECK-NOT: @size.1.align.16 +@size.1.align.1 = internal unnamed_addr addrspace(3) global [1 x i8] undef, align 1 +@size.1.align.2 = internal unnamed_addr addrspace(3) global [1 x i8] undef, align 2 +@size.1.align.4 = internal unnamed_addr addrspace(3) global [1 x i8] undef, align 4 +@size.1.align.8 = internal unnamed_addr addrspace(3) global [1 x i8] undef, align 8 +@size.1.align.16 = internal unnamed_addr addrspace(3) global [1 x i8] undef, align 16 + +define amdgpu_kernel void @k0() { + %lds.size.1.align.1.bc = bitcast [1 x i8] addrspace(3)* @size.1.align.1 to i8 addrspace(3)* + store i8 1, i8 addrspace(3)* %lds.size.1.align.1.bc, align 1 + + %lds.size.1.align.2.bc = bitcast [1 x i8] addrspace(3)* @size.1.align.2 to i8 addrspace(3)* + store i8 2, i8 addrspace(3)* %lds.size.1.align.2.bc, align 2 + + %lds.size.1.align.4.bc = bitcast [1 x i8] addrspace(3)* @size.1.align.4 to i8 addrspace(3)* + store i8 3, i8 addrspace(3)* %lds.size.1.align.4.bc, align 4 + + %lds.size.1.align.8.bc = bitcast [1 x i8] addrspace(3)* @size.1.align.8 to i8 addrspace(3)* + store i8 4, i8 addrspace(3)* %lds.size.1.align.8.bc, align 8 + + %lds.size.1.align.16.bc = bitcast [1 x i8] addrspace(3)* @size.1.align.16 to i8 addrspace(3)* + store i8 5, i8 addrspace(3)* %lds.size.1.align.16.bc, align 16 + + ret void +} + diff --git a/llvm/test/CodeGen/AMDGPU/fix-lds-alignment3.ll b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment3.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment3.ll @@ -0,0 +1,32 @@ +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s | FileCheck %s + +; All are under-aligned, requires to fix each on different alignment boundary. +; CHECK: %llvm.amdgcn.kernel.k0.lds.t = type { [9 x i8], [7 x i8], [5 x i8], [3 x i8], [3 x i8], [1 x i8], [2 x i8] } +; CHECK: @llvm.amdgcn.kernel.k0.lds = internal addrspace(3) global %llvm.amdgcn.kernel.k0.lds.t undef, align 16 + +; CHECK-NOT: @lds.size.2.align.1 +; CHECK-NOT: @lds.size.3.align.2 +; CHECK-NOT: @lds.size.5.align.4 +; CHECK-NOT: @lds.size.9.align.8 +@lds.size.2.align.1 = internal unnamed_addr addrspace(3) global [2 x i8] undef, align 1 +@lds.size.3.align.2 = internal unnamed_addr addrspace(3) global [3 x i8] undef, align 2 +@lds.size.5.align.4 = internal unnamed_addr addrspace(3) global [5 x i8] undef, align 4 +@lds.size.9.align.8 = internal unnamed_addr addrspace(3) global [9 x i8] undef, align 8 + +define amdgpu_kernel void @k0() { + %lds.size.2.align.1.bc = bitcast [2 x i8] addrspace(3)* @lds.size.2.align.1 to i8 addrspace(3)* + store i8 1, i8 addrspace(3)* %lds.size.2.align.1.bc, align 1 + + %lds.size.3.align.2.bc = bitcast [3 x i8] addrspace(3)* @lds.size.3.align.2 to i8 addrspace(3)* + store i8 2, i8 addrspace(3)* %lds.size.3.align.2.bc, align 2 + + %lds.size.5.align.4.bc = bitcast [5 x i8] addrspace(3)* @lds.size.5.align.4 to i8 addrspace(3)* + store i8 3, i8 addrspace(3)* %lds.size.5.align.4.bc, align 4 + + %lds.size.9.align.8.bc = bitcast [9 x i8] addrspace(3)* @lds.size.9.align.8 to i8 addrspace(3)* + store i8 4, i8 addrspace(3)* %lds.size.9.align.8.bc, align 8 + + ret void +} + diff --git a/llvm/test/CodeGen/AMDGPU/fix-lds-alignment4.ll b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment4.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment4.ll @@ -0,0 +1,31 @@ +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s | FileCheck %s + +; All LDS are underaligned, requires to allocate on 16 byte boundary +; CHECK: %llvm.amdgcn.kernel.k0.lds.t = type { [12 x i8], [4 x i8], [11 x i8], [5 x i8], [10 x i8], [6 x i8], [9 x i8] } +; CHECK: @llvm.amdgcn.kernel.k0.lds = internal addrspace(3) global %llvm.amdgcn.kernel.k0.lds.t undef, align 16 + +; CHECK-NOT: @lds.size.9.align.1 +; CHECK-NOT: @lds.size.10.align.2 +; CHECK-NOT: @lds.size.11.align.4 +; CHECK-NOT: @lds.size.12.align.8 +@lds.size.9.align.1 = internal unnamed_addr addrspace(3) global [9 x i8] undef, align 1 +@lds.size.10.align.2 = internal unnamed_addr addrspace(3) global [10 x i8] undef, align 2 +@lds.size.11.align.4 = internal unnamed_addr addrspace(3) global [11 x i8] undef, align 4 +@lds.size.12.align.8 = internal unnamed_addr addrspace(3) global [12 x i8] undef, align 8 + +define amdgpu_kernel void @k0() { + %lds.size.9.align.1.bc = bitcast [9 x i8] addrspace(3)* @lds.size.9.align.1 to i8 addrspace(3)* + store i8 1, i8 addrspace(3)* %lds.size.9.align.1.bc, align 1 + + %lds.size.10.align.2.bc = bitcast [10 x i8] addrspace(3)* @lds.size.10.align.2 to i8 addrspace(3)* + store i8 2, i8 addrspace(3)* %lds.size.10.align.2.bc, align 2 + + %lds.size.11.align.4.bc = bitcast [11 x i8] addrspace(3)* @lds.size.11.align.4 to i8 addrspace(3)* + store i8 3, i8 addrspace(3)* %lds.size.11.align.4.bc, align 4 + + %lds.size.12.align.8.bc = bitcast [12 x i8] addrspace(3)* @lds.size.12.align.8 to i8 addrspace(3)* + store i8 4, i8 addrspace(3)* %lds.size.12.align.8.bc, align 8 + + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/fix-lds-alignment5.ll b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment5.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment5.ll @@ -0,0 +1,31 @@ +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s | FileCheck %s + +; All LDS are underaligned, requires to allocate on 8 byte boundary +; CHECK: %llvm.amdgcn.kernel.k0.lds.t = type { [7 x i8], [1 x i8], [7 x i8], [1 x i8], [6 x i8], [2 x i8], [5 x i8] } +; CHECK: @llvm.amdgcn.kernel.k0.lds = internal addrspace(3) global %llvm.amdgcn.kernel.k0.lds.t undef, align 8 + +; CHECK-NOT: @lds.size.5.align.2 +; CHECK-NOT: @lds.size.6.align.2 +; CHECK-NOT: @lds.size.7.align.2 +; CHECK-NOT: @lds.size.7.align.4 +@lds.size.5.align.2 = internal unnamed_addr addrspace(3) global [5 x i8] undef, align 2 +@lds.size.6.align.2 = internal unnamed_addr addrspace(3) global [6 x i8] undef, align 2 +@lds.size.7.align.2 = internal unnamed_addr addrspace(3) global [7 x i8] undef, align 2 +@lds.size.7.align.4 = internal unnamed_addr addrspace(3) global [7 x i8] undef, align 4 + +define amdgpu_kernel void @k0() { + %lds.size.5.align.2.bc = bitcast [5 x i8] addrspace(3)* @lds.size.5.align.2 to i8 addrspace(3)* + store i8 1, i8 addrspace(3)* %lds.size.5.align.2.bc, align 2 + + %lds.size.6.align.2.bc = bitcast [6 x i8] addrspace(3)* @lds.size.6.align.2 to i8 addrspace(3)* + store i8 2, i8 addrspace(3)* %lds.size.6.align.2.bc, align 2 + + %lds.size.7.align.2.bc = bitcast [7 x i8] addrspace(3)* @lds.size.7.align.2 to i8 addrspace(3)* + store i8 3, i8 addrspace(3)* %lds.size.7.align.2.bc, align 2 + + %lds.size.7.align.4.bc = bitcast [7 x i8] addrspace(3)* @lds.size.7.align.4 to i8 addrspace(3)* + store i8 4, i8 addrspace(3)* %lds.size.7.align.4.bc, align 4 + + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/fix-lds-alignment6.ll b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment6.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fix-lds-alignment6.ll @@ -0,0 +1,32 @@ +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s | FileCheck %s + +; All LDS are properly aligned on 16 byte boundary, but they are of different size. +; CHECK: %llvm.amdgcn.kernel.k0.lds.t = type { [20 x i8], [12 x i8], [19 x i8], [13 x i8], [18 x i8], [14 x i8], [17 x i8] } +; CHECK: @llvm.amdgcn.kernel.k0.lds = internal addrspace(3) global %llvm.amdgcn.kernel.k0.lds.t undef, align 16 + +; CHECK-NOT: @lds.size.17.align.16 +; CHECK-NOT: @lds.size.18.align.16 +; CHECK-NOT: @lds.size.19.align.16 +; CHECK-NOT: @lds.size.20.align.16 +@lds.size.17.align.16 = internal unnamed_addr addrspace(3) global [17 x i8] undef, align 16 +@lds.size.18.align.16 = internal unnamed_addr addrspace(3) global [18 x i8] undef, align 16 +@lds.size.19.align.16 = internal unnamed_addr addrspace(3) global [19 x i8] undef, align 16 +@lds.size.20.align.16 = internal unnamed_addr addrspace(3) global [20 x i8] undef, align 16 + +define amdgpu_kernel void @k0() { + %lds.size.17.align.16.bc = bitcast [17 x i8] addrspace(3)* @lds.size.17.align.16 to i8 addrspace(3)* + store i8 1, i8 addrspace(3)* %lds.size.17.align.16.bc, align 16 + + %lds.size.18.align.16.bc = bitcast [18 x i8] addrspace(3)* @lds.size.18.align.16 to i8 addrspace(3)* + store i8 2, i8 addrspace(3)* %lds.size.18.align.16.bc, align 16 + + %lds.size.19.align.16.bc = bitcast [19 x i8] addrspace(3)* @lds.size.19.align.16 to i8 addrspace(3)* + store i8 3, i8 addrspace(3)* %lds.size.19.align.16.bc, align 16 + + %lds.size.20.align.16.bc = bitcast [20 x i8] addrspace(3)* @lds.size.20.align.16 to i8 addrspace(3)* + store i8 4, i8 addrspace(3)* %lds.size.20.align.16.bc, align 16 + + ret void +} + diff --git a/llvm/test/CodeGen/AMDGPU/lds-alignment.ll b/llvm/test/CodeGen/AMDGPU/lds-alignment.ll --- a/llvm/test/CodeGen/AMDGPU/lds-alignment.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-alignment.ll @@ -48,7 +48,7 @@ ; 38 + (2 pad) + 38 ; HSA-LABEL: {{^}}test_round_size_2_align_8: -; HSA: workgroup_group_segment_byte_size = 78 +; HSA: workgroup_group_segment_byte_size = 86 ; HSA: group_segment_alignment = 4 define amdgpu_kernel void @test_round_size_2_align_8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 { %lds.align16.0.bc = bitcast [38 x i8] addrspace(3)* @lds.align16.0 to i8 addrspace(3)* @@ -130,7 +130,7 @@ ; align 32, 16, 8 ; 38 + (10 pad) + 38 + (2 pad) + 38 = 126 ; HSA-LABEL: {{^}}test_round_size_3_order0: -; HSA: workgroup_group_segment_byte_size = 126 +; HSA: workgroup_group_segment_byte_size = 134 ; HSA: group_segment_alignment = 4 define amdgpu_kernel void @test_round_size_3_order0(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 { %lds.align32.0.bc = bitcast [38 x i8] addrspace(3)* @lds.align32.0 to i8 addrspace(3)* @@ -151,7 +151,7 @@ ; align 32, 8, 16 ; 38 (+ 10 pad) + 38 + (2 pad) + 38 = 126 ; HSA-LABEL: {{^}}test_round_size_3_order1: -; HSA: workgroup_group_segment_byte_size = 126 +; HSA: workgroup_group_segment_byte_size = 134 ; HSA: group_segment_alignment = 4 define amdgpu_kernel void @test_round_size_3_order1(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 { %lds.align32.0.bc = bitcast [38 x i8] addrspace(3)* @lds.align32.0 to i8 addrspace(3)* @@ -172,7 +172,7 @@ ; align 16, 32, 8 ; 38 + (10 pad) + 38 + (2 pad) + 38 = 126 ; HSA-LABEL: {{^}}test_round_size_3_order2: -; HSA: workgroup_group_segment_byte_size = 126 +; HSA: workgroup_group_segment_byte_size = 134 ; HSA: group_segment_alignment = 4 define amdgpu_kernel void @test_round_size_3_order2(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 { %lds.align16.0.bc = bitcast [38 x i8] addrspace(3)* @lds.align16.0 to i8 addrspace(3)* @@ -194,7 +194,7 @@ ; align 16, 8, 32 ; 38 + (10 pad) + 38 + (2 pad) + 38 ; HSA-LABEL: {{^}}test_round_size_3_order3: -; HSA: workgroup_group_segment_byte_size = 126 +; HSA: workgroup_group_segment_byte_size = 134 ; HSA: group_segment_alignment = 4 define amdgpu_kernel void @test_round_size_3_order3(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 { %lds.align16.0.bc = bitcast [38 x i8] addrspace(3)* @lds.align16.0 to i8 addrspace(3)* @@ -215,7 +215,7 @@ ; align 8, 32, 16 ; 38 + (10 pad) + 38 + (2 pad) + 38 = 126 ; HSA-LABEL: {{^}}test_round_size_3_order4: -; HSA: workgroup_group_segment_byte_size = 126 +; HSA: workgroup_group_segment_byte_size = 134 ; HSA: group_segment_alignment = 4 define amdgpu_kernel void @test_round_size_3_order4(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 { %lds.align8.0.bc = bitcast [38 x i8] addrspace(3)* @lds.align8.0 to i8 addrspace(3)* @@ -236,7 +236,7 @@ ; align 8, 16, 32 ; 38 + (10 pad) + 38 + (2 pad) + 38 = 126 ; HSA-LABEL: {{^}}test_round_size_3_order5: -; HSA: workgroup_group_segment_byte_size = 126 +; HSA: workgroup_group_segment_byte_size = 134 ; HSA: group_segment_alignment = 4 define amdgpu_kernel void @test_round_size_3_order5(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 { %lds.align8.0.bc = bitcast [38 x i8] addrspace(3)* @lds.align8.0 to i8 addrspace(3)* diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll --- a/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll +++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-globals.ll @@ -8,7 +8,7 @@ ; IR-LABEL: define amdgpu_kernel void @promote_alloca_size_256(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) { ; IR: alloca [10 x i32] ; ASM-LABEL: {{^}}promote_alloca_size_256: -; ASM: .amdgpu_lds llvm.amdgcn.kernel.promote_alloca_size_256.lds, 60000, 4 +; ASM: .amdgpu_lds llvm.amdgcn.kernel.promote_alloca_size_256.lds, 60000, 16 ; ASM-NOT: .amdgpu_lds define amdgpu_kernel void @promote_alloca_size_256(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) {