Index: llvm/lib/Target/AMDGPU/AMDGPU.td =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPU.td +++ llvm/lib/Target/AMDGPU/AMDGPU.td @@ -1299,6 +1299,9 @@ def DisableFlatScratch : Predicate<"!Subtarget->enableFlatScratch()">; +def HasUnalignedAccessMode : Predicate<"Subtarget->hasUnalignedAccessMode()">, + AssemblerPredicate<(all_of FeatureUnalignedAccessMode)>; + // Include AMDGPU TD files include "SISchedule.td" include "GCNProcessors.td" Index: llvm/lib/Target/AMDGPU/DSInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/DSInstructions.td +++ llvm/lib/Target/AMDGPU/DSInstructions.td @@ -694,6 +694,16 @@ let SubtargetPredicate = isGFX9Plus in { +foreach vt = VReg_96.RegTypes in { +defm : DSReadPat_mc ; +} + +foreach vt = VReg_128.RegTypes in { +defm : DSReadPat_mc ; +} + +let SubtargetPredicate = HasUnalignedAccessMode in { + foreach vt = VReg_96.RegTypes in { defm : DSReadPat_mc ; } @@ -702,7 +712,9 @@ defm : DSReadPat_mc ; } -} +} // End SubtargetPredicate = HasUnalignedAccessMode + +} // End SubtargetPredicate = isGFX9Plus } // End AddedComplexity = 100 @@ -849,6 +861,16 @@ let SubtargetPredicate = isGFX9Plus in { +foreach vt = VReg_96.RegTypes in { +defm : DSWritePat_mc ; +} + +foreach vt = VReg_128.RegTypes in { +defm : DSWritePat_mc ; +} + +let SubtargetPredicate = HasUnalignedAccessMode in { + foreach vt = VReg_96.RegTypes in { defm : DSWritePat_mc ; } @@ -857,9 +879,12 @@ defm : DSWritePat_mc ; } -} +} // End SubtargetPredicate = HasUnalignedAccessMode + +} // End SubtargetPredicate = isGFX9Plus } // End AddedComplexity = 100 + class DSAtomicRetPat : GCNPat < (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value), (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 gds)) Index: llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir +++ llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir @@ -50,8 +50,8 @@ ; GFX9-LABEL: name: load_local_v4s32_align_8 ; GFX9: liveins: $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3) - ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] + ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3) + ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]] %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(<4 x s32>) = G_LOAD %0 :: (load 16, align 8, addrspace 3) $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1 @@ -78,8 +78,8 @@ ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_160 ; GFX9: liveins: $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 400, 0, implicit $exec :: (load 16, align 8, addrspace 3) - ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] + ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 50, 51, 0, implicit $exec :: (load 16, align 8, addrspace 3) + ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]] %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(s32) = G_CONSTANT i32 400 %2:vgpr(p3) = G_PTR_ADD %0, %1 @@ -110,8 +110,10 @@ ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_320 ; GFX9: liveins: $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 4000, 0, implicit $exec :: (load 16, align 8, addrspace 3) - ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] + ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4000, implicit $exec + ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec + ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[V_ADD_U32_e64_]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3) + ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]] %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(s32) = G_CONSTANT i32 4000 %2:vgpr(p3) = G_PTR_ADD %0, %1 @@ -140,8 +142,8 @@ ; GFX9-LABEL: name: load_local_v2s64 ; GFX9: liveins: $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3) - ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] + ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3) + ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]] %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(<2 x s64>) = G_LOAD %0 :: (load 16, align 8, addrspace 3) $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1 @@ -196,8 +198,8 @@ ; GFX9-LABEL: name: load_local_s128 ; GFX9: liveins: $vgpr0 ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3) - ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]] + ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3) + ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]] %0:vgpr(p3) = COPY $vgpr0 %1:vgpr(s128) = G_LOAD %0 :: (load 16, align 8, addrspace 3) $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1 Index: llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll +++ llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll @@ -1,8 +1,8 @@ -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED %s +; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s +; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s +; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s +; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s +; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s ; GCN-LABEL: test_local_misaligned_v2: ; GCN-DAG: ds_read2_b32 @@ -22,10 +22,10 @@ } ; GCN-LABEL: test_local_misaligned_v4: -; GCN-DAG: ds_read2_b32 -; GCN-DAG: ds_read2_b32 -; GCN-DAG: ds_write2_b32 -; GCN-DAG: ds_write2_b32 +; ALIGNED-DAG: ds_read2_b32 +; ALIGNED-DAG: ds_read2_b32 +; ALIGNED-DAG: ds_write2_b32 +; ALIGNED-DAG: ds_write2_b32 ; UNALIGNED-DAG: ds_read_b128 ; UNALIGNED-DAG: ds_write_b128 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) { @@ -47,10 +47,10 @@ } ; GCN-LABEL: test_local_misaligned_v3: -; GCN-DAG: ds_read2_b32 -; GCN-DAG: ds_read_b32 -; GCN-DAG: ds_write2_b32 -; GCN-DAG: ds_write_b32 +; ALIGNED-DAG: ds_read2_b32 +; ALIGNED-DAG: ds_read_b32 +; ALIGNED-DAG: ds_write2_b32 +; ALIGNED-DAG: ds_write_b32 ; UNALIGNED-DAG: ds_read_b96 ; UNALIGNED-DAG: ds_write_b96 define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) { @@ -106,8 +106,10 @@ } ; GCN-LABEL: test_local_v4_aligned8: -; GCN-DAG: ds_read_b128 -; GCN-DAG: ds_write_b128 +; ALIGNED-DAG: ds_read2_b64 +; ALIGNED-DAG: ds_write2_b64 +; UNALIGNED-DAG: ds_read_b128 +; UNALIGNED-DAG: ds_write_b128 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) { bb: %lid = tail call i32 @llvm.amdgcn.workitem.id.x() Index: llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll +++ llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll @@ -267,7 +267,7 @@ ; GFX9-LABEL: load_lds_v4i32_align8: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_read_b128 v[0:3], v0 +; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; Index: llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll +++ llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll @@ -250,7 +250,7 @@ ; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: v_mov_b32_e32 v3, s3 -; GFX9-NEXT: ds_write_b128 v4, v[0:3] +; GFX9-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:1 ; GFX9-NEXT: s_endpgm ; ; GFX7-LABEL: store_lds_v4i32_align8: Index: llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll +++ llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll @@ -1,8 +1,8 @@ -; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s -; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s -; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s -; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s -; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED,VECT %s +; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s +; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s +; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s +; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s +; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s ; GCN-LABEL: test_local_misaligned_v2: ; GCN-DAG: ds_read2_b32 @@ -22,10 +22,10 @@ } ; GCN-LABEL: test_local_misaligned_v4: -; GCN-DAG: ds_read2_b32 -; GCN-DAG: ds_read2_b32 -; GCN-DAG: ds_write2_b32 -; GCN-DAG: ds_write2_b32 +; ALIGNED-DAG: ds_read2_b32 +; ALIGNED-DAG: ds_read2_b32 +; ALIGNED-DAG: ds_write2_b32 +; ALIGNED-DAG: ds_write2_b32 ; UNALIGNED-DAG: ds_read_b128 ; UNALIGNED-DAG: ds_write_b128 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) { @@ -47,10 +47,10 @@ } ; GCN-LABEL: test_local_misaligned_v3: -; GCN-DAG: ds_read2_b32 -; GCN-DAG: ds_read_b32 -; GCN-DAG: ds_write2_b32 -; GCN-DAG: ds_write_b32 +; ALIGNED-DAG: ds_read2_b32 +; ALIGNED-DAG: ds_read_b32 +; ALIGNED-DAG: ds_write2_b32 +; ALIGNED-DAG: ds_write_b32 ; UNALIGNED-DAG: ds_read_b96 ; UNALIGNED-DAG: ds_write_b96 define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) { @@ -221,8 +221,10 @@ } ; GCN-LABEL: test_local_v4_aligned8: -; GCN-DAG: ds_read_b128 -; GCN-DAG: ds_write_b128 +; ALIGNED-DAG: ds_read2_b64 +; ALIGNED-DAG: ds_write2_b64 +; UNALIGNED-DAG: ds_read_b128 +; UNALIGNED-DAG: ds_write_b128 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) { bb: %lid = tail call i32 @llvm.amdgcn.workitem.id.x() Index: llvm/test/CodeGen/AMDGPU/load-local.128.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/load-local.128.ll +++ llvm/test/CodeGen/AMDGPU/load-local.128.ll @@ -325,7 +325,7 @@ ; GFX9-LABEL: load_lds_v4i32_align8: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: ds_read_b128 v[0:3], v0 +; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; Index: llvm/test/CodeGen/AMDGPU/store-local.128.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/store-local.128.ll +++ llvm/test/CodeGen/AMDGPU/store-local.128.ll @@ -340,10 +340,10 @@ ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v4, s4 ; GFX9-NEXT: v_mov_b32_e32 v0, s0 -; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v3, s3 -; GFX9-NEXT: ds_write_b128 v4, v[0:3] +; GFX9-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:1 ; GFX9-NEXT: s_endpgm ; ; GFX7-LABEL: store_lds_v4i32_align8: