Index: llvm/lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1176,9 +1176,13 @@ case Intrinsic::amdgcn_atomic_dec: case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_swap: + case Intrinsic::amdgcn_ds_append: + case Intrinsic::amdgcn_ds_consume: case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fmin: - case Intrinsic::amdgcn_ds_fmax: { + case Intrinsic::amdgcn_ds_fmax: + case Intrinsic::amdgcn_global_atomic_fadd: + case Intrinsic::amdgcn_global_atomic_csub: { Value *Ptr = II->getArgOperand(0); AccessTy = II->getType(); Ops.push_back(Ptr); Index: llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll @@ -0,0 +1,75 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=OPT %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GCN %s + +; Make sure we match the addressing mode offset of csub intrinsics across blocks. + +define amdgpu_kernel void @test_sink_small_offset_global_atomic_csub_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) { +; OPT-LABEL: @test_sink_small_offset_global_atomic_csub_i32( +; OPT-NEXT: entry: +; OPT-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[OUT:%.*]], i32 999999 +; OPT-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #3 +; OPT-NEXT: [[CMP:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-NEXT: br i1 [[CMP]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT: if: +; OPT-NEXT: [[TMP0:%.*]] = bitcast i32 addrspace(1)* [[IN:%.*]] to i8 addrspace(1)* +; OPT-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 28 +; OPT-NEXT: [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SUNKADDR]] to i32 addrspace(1)* +; OPT-NEXT: [[VAL:%.*]] = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* [[TMP1]], i32 2) +; OPT-NEXT: br label [[ENDIF]] +; OPT: endif: +; OPT-NEXT: [[X:%.*]] = phi i32 [ [[VAL]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-NEXT: store i32 [[X]], i32 addrspace(1)* [[OUT_GEP]], align 4 +; OPT-NEXT: ret void +; +; GCN-LABEL: test_sink_small_offset_global_atomic_csub_i32: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, -1, 0 +; GCN-NEXT: ; implicit-def: $vcc_hi +; GCN-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0 +; GCN-NEXT: v_mov_b32_e32 v0, 0 +; GCN-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GCN-NEXT: s_cbranch_execz BB0_2 +; GCN-NEXT: ; %bb.1: ; %if +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: v_mov_b32_e32 v2, 2 +; GCN-NEXT: global_atomic_csub v0, v[0:1], v2, off offset:28 glc +; GCN-NEXT: BB0_2: ; %endif +; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_co_u32_e64 v1, s0, 0x3d0800, s0 +; GCN-NEXT: v_add_co_ci_u32_e64 v2, s0, 0, s1, s0 +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: global_store_dword v[1:2], v0, off offset:252 +; GCN-NEXT: s_endpgm +entry: + %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 999999 + %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 7 + %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %cmp = icmp eq i32 %tid, 0 + br i1 %cmp, label %endif, label %if + +if: + %val = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %in.gep, i32 2) + br label %endif + +endif: + %x = phi i32 [ %val, %if ], [ 0, %entry ] + store i32 %x, i32 addrspace(1)* %out.gep + br label %done + +done: + ret void +} + +declare i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* nocapture, i32) #0 +declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1 +declare void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* nocapture, float) #2 + +attributes #0 = { argmemonly nounwind } +attributes #1 = { nounwind readnone willreturn } +attributes #2 = { argmemonly nounwind willreturn } Index: llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll @@ -0,0 +1,78 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=OPT %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=GCN %s + +; Make sure we match the addressing mode offset of globla.atomic.fadd intrinsics across blocks. + +define amdgpu_kernel void @test_sink_small_offset_global_atomic_fadd_f32(float addrspace(1)* %out, float addrspace(1)* %in) { +; OPT-LABEL: @test_sink_small_offset_global_atomic_fadd_f32( +; OPT-NEXT: entry: +; OPT-NEXT: [[OUT_GEP:%.*]] = getelementptr float, float addrspace(1)* [[OUT:%.*]], i32 999999 +; OPT-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #3 +; OPT-NEXT: [[CMP:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-NEXT: br i1 [[CMP]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT: if: +; OPT-NEXT: [[TMP0:%.*]] = bitcast float addrspace(1)* [[IN:%.*]] to i8 addrspace(1)* +; OPT-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 28 +; OPT-NEXT: [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SUNKADDR]] to float addrspace(1)* +; OPT-NEXT: call void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* [[TMP1]], float 2.000000e+00) +; OPT-NEXT: [[VAL:%.*]] = load volatile float, float addrspace(1)* undef, align 4 +; OPT-NEXT: br label [[ENDIF]] +; OPT: endif: +; OPT-NEXT: [[X:%.*]] = phi float [ [[VAL]], [[IF]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; OPT-NEXT: store float [[X]], float addrspace(1)* [[OUT_GEP]], align 4 +; OPT-NEXT: ret void +; +; GCN-LABEL: test_sink_small_offset_global_atomic_fadd_f32: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_mov_b32_e32 v0, 0 +; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GCN-NEXT: s_cbranch_execz BB0_2 +; GCN-NEXT: ; %bb.1: ; %if +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, s3 +; GCN-NEXT: v_mov_b32_e32 v2, 2.0 +; GCN-NEXT: global_atomic_add_f32 v[0:1], v2, off offset:28 +; GCN-NEXT: global_load_dword v0, v[0:1], off +; GCN-NEXT: BB0_2: ; %endif +; GCN-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v1, s0 +; GCN-NEXT: v_add_co_u32_e32 v1, vcc, 0x3d0000, v1 +; GCN-NEXT: v_mov_b32_e32 v2, s1 +; GCN-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: global_store_dword v[1:2], v0, off offset:2300 +; GCN-NEXT: s_endpgm +entry: + %out.gep = getelementptr float, float addrspace(1)* %out, i32 999999 + %in.gep = getelementptr float, float addrspace(1)* %in, i32 7 + %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %cmp = icmp eq i32 %tid, 0 + br i1 %cmp, label %endif, label %if + +if: + call void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* %in.gep, float 2.0) + %val = load volatile float, float addrspace(1)* undef + br label %endif + +endif: + %x = phi float [ %val, %if ], [ 0.0, %entry ] + store float %x, float addrspace(1)* %out.gep + br label %done + +done: + ret void +} + +declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1 +declare void @llvm.amdgcn.global.atomic.fadd.p1f32.f32(float addrspace(1)* nocapture, float) #2 + +attributes #0 = { argmemonly nounwind } +attributes #1 = { nounwind readnone willreturn } +attributes #2 = { argmemonly nounwind willreturn } Index: llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll +++ llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll @@ -748,10 +748,65 @@ ret void } +; OPT-LABEL: @test_sink_small_offset_ds_append( +; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)* +; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28 +; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)* +; OPT: %tmp1 = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %1, i1 false) +define amdgpu_kernel void @test_sink_small_offset_ds_append(i32 addrspace(3)* %out, i32 addrspace(3)* %in) { +entry: + %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999 + %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 + %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %tmp0 = icmp eq i32 %tid, 0 + br i1 %tmp0, label %endif, label %if + +if: + %tmp1 = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %in.gep, i1 false) + br label %endif + +endif: + %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] + store i32 %x, i32 addrspace(3)* %out.gep + br label %done + +done: + ret void +} + +; OPT-LABEL: @test_sink_small_offset_ds_consume( +; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)* +; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28 +; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)* +; OPT: %tmp1 = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %1, i1 false) +define amdgpu_kernel void @test_sink_small_offset_ds_consume(i32 addrspace(3)* %out, i32 addrspace(3)* %in) { +entry: + %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999 + %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 + %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %tmp0 = icmp eq i32 %tid, 0 + br i1 %tmp0, label %endif, label %if + +if: + %tmp1 = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %in.gep, i1 false) + br label %endif + +endif: + %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] + store i32 %x, i32 addrspace(3)* %out.gep + br label %done + +done: + ret void +} + declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0 declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2 declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2 +declare i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* nocapture, i1 immarg) #3 +declare i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* nocapture, i1 immarg) #3 attributes #0 = { nounwind readnone } attributes #1 = { nounwind } attributes #2 = { nounwind argmemonly } +attributes #3 = { argmemonly convergent nounwind willreturn }