Index: llvm/lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1470,12 +1470,21 @@ } } + if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) { + bool AlignedBy4 = Alignment >= Align(4); + if (IsFast) + *IsFast = AlignedBy4; + + return AlignedBy4 || + Subtarget->enableFlatScratch() || + Subtarget->hasUnalignedScratchAccess(); + } + // FIXME: We have to be conservative here and assume that flat operations // will access scratch. If we had access to the IR function, then we // could determine if any private memory was used in the function. - if (!Subtarget->hasUnalignedScratchAccess() && - (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS || - AddrSpace == AMDGPUAS::FLAT_ADDRESS)) { + if (AddrSpace == AMDGPUAS::FLAT_ADDRESS && + !Subtarget->hasUnalignedScratchAccess()) { bool AlignedBy4 = Alignment >= Align(4); if (IsFast) *IsFast = AlignedBy4; Index: llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll +++ llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll @@ -271,16 +271,9 @@ ; FLATSCR-NEXT: s_waitcnt vmcnt(0) ; FLATSCR-NEXT: scratch_store_short off, v0, vcc_hi offset:8 ; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0 -; FLATSCR-NEXT: scratch_load_ushort v0, off, vcc_hi offset:4 +; FLATSCR-NEXT: scratch_load_dword v0, off, vcc_hi offset:4 ; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0 -; FLATSCR-NEXT: scratch_load_ushort v3, off, vcc_hi offset:6 -; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0 -; FLATSCR-NEXT: s_waitcnt vmcnt(1) -; FLATSCR-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; FLATSCR-NEXT: s_waitcnt vmcnt(0) -; FLATSCR-NEXT: v_mov_b32_e32 v1, v3 -; FLATSCR-NEXT: scratch_load_short_d16_hi v1, off, vcc_hi offset:8 -; FLATSCR-NEXT: v_lshl_or_b32 v0, v3, 16, v0 +; FLATSCR-NEXT: scratch_load_dword v1, off, vcc_hi offset:6 ; FLATSCR-NEXT: s_waitcnt vmcnt(0) ; FLATSCR-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; FLATSCR-NEXT: s_endpgm Index: llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll =================================================================== --- llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll +++ llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll @@ -9,9 +9,6 @@ ; ALL-LABEL: @load_unknown_offset_align1_i8( ; ALL: alloca [128 x i8], align 1 ; UNALIGNED: load <2 x i8>, <2 x i8> addrspace(5)* %{{[0-9]+}}, align 1{{$}} - -; ALIGNED: load i8, i8 addrspace(5)* %ptr0, align 1{{$}} -; ALIGNED: load i8, i8 addrspace(5)* %ptr1, align 1{{$}} define amdgpu_kernel void @load_unknown_offset_align1_i8(i8 addrspace(1)* noalias %out, i32 %offset) #0 { %alloca = alloca [128 x i8], align 1, addrspace(5) %ptr0 = getelementptr inbounds [128 x i8], [128 x i8] addrspace(5)* %alloca, i32 0, i32 %offset @@ -60,13 +57,11 @@ ret void } -; FIXME: Should always increase alignment of the load ; Make sure alloca alignment isn't decreased ; ALL-LABEL: @load_alloca16_unknown_offset_align1_i32( ; ALL: alloca [128 x i32], align 16 -; UNALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}} -; ALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}} +; ALL: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}} define amdgpu_kernel void @load_alloca16_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 { %alloca = alloca [128 x i32], align 16, addrspace(5) %ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset @@ -128,11 +123,8 @@ } ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32( -; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5) -; ALIGNED: store <4 x i32> , <4 x i32> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5) -; UNALIGNED: store <4 x i32> , <4 x i32> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5) +; ALL: store <4 x i32> , <4 x i32> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() { %alloca = alloca [8 x i32], align 1, addrspace(5) %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)* @@ -148,11 +140,8 @@ } ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8( -; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5) -; ALIGNED: store <4 x i8> , <4 x i8> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5) -; UNALIGNED: store <4 x i8> , <4 x i8> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5) +; ALL: store <4 x i8> , <4 x i8> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() { %alloca = alloca [8 x i8], align 1, addrspace(5) %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)* @@ -168,11 +157,8 @@ } ; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i32( -; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5) -; ALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5) -; UNALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5) +; ALL: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() { %alloca = alloca [8 x i32], align 1, addrspace(5) %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)* @@ -188,11 +174,8 @@ } ; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i8( -; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5) -; ALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4 - -; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5) -; UNALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 1 +; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5) +; ALL: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4 define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() { %alloca = alloca [8 x i8], align 1, addrspace(5) %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*