diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1470,12 +1470,21 @@ } } + if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) { + bool AlignedBy4 = Alignment >= Align(4); + if (IsFast) + *IsFast = AlignedBy4; + + return AlignedBy4 || + Subtarget->enableFlatScratch() || + Subtarget->hasUnalignedScratchAccess(); + } + // FIXME: We have to be conservative here and assume that flat operations // will access scratch. If we had access to the IR function, then we // could determine if any private memory was used in the function. - if (!Subtarget->hasUnalignedScratchAccess() && - (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS || - AddrSpace == AMDGPUAS::FLAT_ADDRESS)) { + if (AddrSpace == AMDGPUAS::FLAT_ADDRESS && + !Subtarget->hasUnalignedScratchAccess()) { bool AlignedBy4 = Alignment >= Align(4); if (IsFast) *IsFast = AlignedBy4; diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll --- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll +++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll @@ -271,16 +271,9 @@ ; FLATSCR-NEXT: s_waitcnt vmcnt(0) ; FLATSCR-NEXT: scratch_store_short off, v0, vcc_hi offset:8 ; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0 -; FLATSCR-NEXT: scratch_load_ushort v0, off, vcc_hi offset:4 +; FLATSCR-NEXT: scratch_load_dword v0, off, vcc_hi offset:4 ; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0 -; FLATSCR-NEXT: scratch_load_ushort v3, off, vcc_hi offset:6 -; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0 -; FLATSCR-NEXT: s_waitcnt vmcnt(1) -; FLATSCR-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; FLATSCR-NEXT: s_waitcnt vmcnt(0) -; FLATSCR-NEXT: v_mov_b32_e32 v1, v3 -; FLATSCR-NEXT: scratch_load_short_d16_hi v1, off, vcc_hi offset:8 -; FLATSCR-NEXT: v_lshl_or_b32 v0, v3, 16, v0 +; FLATSCR-NEXT: scratch_load_dword v1, off, vcc_hi offset:6 ; FLATSCR-NEXT: s_waitcnt vmcnt(0) ; FLATSCR-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; FLATSCR-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll --- a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll +++ b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll @@ -1,6 +1,7 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s -; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=UNALIGNED %s -; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s +; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s +; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,UNALIGNED %s +; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,FLATSCR,ALIGNED %s ; SI-LABEL: {{^}}local_unaligned_load_store_i16: ; SI: ds_read_u8 @@ -602,64 +603,70 @@ } ; SI-LABEL: {{^}}private_load_align1_f64: -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; MUBUF: buffer_load_ubyte +; FLATSCR: scratch_load_dwordx2 define double @private_load_align1_f64(double addrspace(5)* %in) { %x = load double, double addrspace(5)* %in, align 1 ret double %x } ; SI-LABEL: {{^}}private_store_align1_f64: -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte +; MUBUF: buffer_store_byte +; MUBUF: buffer_store_byte +; MUBUF: buffer_store_byte +; MUBUF: buffer_store_byte +; MUBUF: buffer_store_byte +; MUBUF: buffer_store_byte +; MUBUF: buffer_store_byte +; MUBUF: buffer_store_byte +; FLATSCR: scratch_store_dwordx2 define void @private_store_align1_f64(double addrspace(5)* %out, double %x) #0 { store double %x, double addrspace(5)* %out, align 1 ret void } ; SI-LABEL: {{^}}private_load_align4_f64: -; SI: buffer_load_dword -; SI: buffer_load_dword +; MUBUF: buffer_load_dword +; MUBUF: buffer_load_dword +; FLATSCR: scratch_load_dwordx2 define double @private_load_align4_f64(double addrspace(5)* %in) { %x = load double, double addrspace(5)* %in, align 4 ret double %x } ; SI-LABEL: {{^}}private_store_align4_f64: -; SI: buffer_store_dword -; SI: buffer_store_dword +; MUBUF: buffer_store_dword +; MUBUF: buffer_store_dword +; FLATSCR: scratch_store_dwordx2 define void @private_store_align4_f64(double addrspace(5)* %out, double %x) #0 { store double %x, double addrspace(5)* %out, align 4 ret void } ; SI-LABEL: {{^}}private_load_align2_f64: -; SI: buffer_load_ushort -; SI: buffer_load_ushort -; SI: buffer_load_ushort -; SI: buffer_load_ushort +; MUBUF: buffer_load_ushort +; MUBUF: buffer_load_ushort +; MUBUF: buffer_load_ushort +; MUBUF: buffer_load_ushort +; FLATSCR: scratch_load_dwordx2 define double @private_load_align2_f64(double addrspace(5)* %in) { %x = load double, double addrspace(5)* %in, align 2 ret double %x } ; SI-LABEL: {{^}}private_store_align2_f64: -; SI: buffer_store_short -; SI: buffer_store_short -; SI: buffer_store_short -; SI: buffer_store_short +; MUBUF: buffer_store_short +; MUBUF: buffer_store_short +; MUBUF: buffer_store_short +; MUBUF: buffer_store_short +; FLATSCR: scratch_store_dwordx2 define void @private_store_align2_f64(double addrspace(5)* %out, double %x) #0 { store double %x, double addrspace(5)* %out, align 2 ret void diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll --- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll +++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll @@ -9,9 +9,6 @@ ; ALL-LABEL: @load_unknown_offset_align1_i8( ; ALL: alloca [128 x i8], align 1 ; UNALIGNED: load <2 x i8>, <2 x i8> addrspace(5)* %{{[0-9]+}}, align 1{{$}} - -; ALIGNED: load i8, i8 addrspace(5)* %ptr0, align 1{{$}} -; ALIGNED: load i8, i8 addrspace(5)* %ptr1, align 1{{$}} define amdgpu_kernel void @load_unknown_offset_align1_i8(i8 addrspace(1)* noalias %out, i32 %offset) #0 { %alloca = alloca [128 x i8], align 1, addrspace(5) %ptr0 = getelementptr inbounds [128 x i8], [128 x i8] addrspace(5)* %alloca, i32 0, i32 %offset @@ -60,13 +57,11 @@ ret void } -; FIXME: Should always increase alignment of the load ; Make sure alloca alignment isn't decreased ; ALL-LABEL: @load_alloca16_unknown_offset_align1_i32( ; ALL: alloca [128 x i32], align 16 -; UNALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}} -; ALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}} +; ALL: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}} define amdgpu_kernel void @load_alloca16_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 { %alloca = alloca [128 x i32], align 16, addrspace(5) %ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset @@ -128,11 +123,8 @@ } ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32( -; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5) -; ALIGNED: store <4 x i32> , <4 x i32> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5) -; UNALIGNED: store <4 x i32> , <4 x i32> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5) +; ALL: store <4 x i32> , <4 x i32> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() { %alloca = alloca [8 x i32], align 1, addrspace(5) %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)* @@ -148,11 +140,8 @@ } ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8( -; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5) -; ALIGNED: store <4 x i8> , <4 x i8> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5) -; UNALIGNED: store <4 x i8> , <4 x i8> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5) +; ALL: store <4 x i8> , <4 x i8> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() { %alloca = alloca [8 x i8], align 1, addrspace(5) %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)* @@ -168,11 +157,8 @@ } ; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i32( -; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5) -; ALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5) -; UNALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5) +; ALL: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() { %alloca = alloca [8 x i32], align 1, addrspace(5) %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)* @@ -188,11 +174,8 @@ } ; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i8( -; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5) -; ALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5) -; UNALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5) +; ALL: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() { %alloca = alloca [8 x i8], align 1, addrspace(5) %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*