Index: llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll +++ llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll @@ -1,36 +1,168 @@ -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck --check-prefixes=OPT,OPT-CI,OPT-CIVI %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck --check-prefixes=OPT,OPT-CIVI %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck --check-prefixes=OPT,OPT-GFX9 %s -; RUN: llc -march=amdgcn -amdgpu-scalarize-global-loads=false -mcpu=bonaire -mattr=-promote-alloca < %s | FileCheck --check-prefixes=GCN,CI,CIVI %s -; RUN: llc -march=amdgcn -amdgpu-scalarize-global-loads=false -mcpu=tonga -mattr=-flat-for-global -mattr=-promote-alloca < %s | FileCheck --check-prefixes=GCN,CIVI %s -; RUN: llc -march=amdgcn -amdgpu-scalarize-global-loads=false -mcpu=gfx900 -mattr=-flat-for-global -mattr=-promote-alloca < %s | FileCheck --check-prefixes=GCN,GFX9 %s - -; OPT-LABEL: @test_no_sink_flat_small_offset_i32( -; OPT-CIVI: getelementptr i32, i32* %in -; OPT-CIVI: br i1 -; OPT-CIVI-NOT: ptrtoint - -; OPT-GFX9: br -; OPT-GFX9: %sunkaddr = getelementptr i8, i8* %0, i64 28 -; OPT-GFX9: %1 = bitcast i8* %sunkaddr to i32* -; OPT-GFX9: load i32, i32* %1 - -; GCN-LABEL: {{^}}test_no_sink_flat_small_offset_i32: -; GCN: flat_load_dword -; GCN: {{^}}.LBB0_2: -define amdgpu_kernel void @test_no_sink_flat_small_offset_i32(i32* %out, i32* %in, i32 %cond) { +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck --check-prefixes=OPT,OPT-GFX7 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga < %s | FileCheck --check-prefixes=OPT,OPT-GFX8 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck --check-prefixes=OPT,OPT-GFX9 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx1030 < %s | FileCheck --check-prefixes=OPT,OPT-GFX10 %s + +; RUN: llc -march=amdgcn -mcpu=bonaire < %s | FileCheck --check-prefix=GFX7 %s +; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=GFX8 %s +; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s +; RUN: llc -march=amdgcn -mcpu=gfx1030 < %s | FileCheck --check-prefix=GFX10 %s + +define void @test_sinkable_flat_small_offset_i32(i32* %out, i32* %in, i32 %cond) { +; OPT-GFX7-LABEL: @test_sinkable_flat_small_offset_i32( +; OPT-GFX7-NEXT: entry: +; OPT-GFX7-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX7-NEXT: [[IN_GEP:%.*]] = getelementptr i32, i32* [[IN:%.*]], i64 7 +; OPT-GFX7-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX7-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX7: if: +; OPT-GFX7-NEXT: [[LOAD:%.*]] = load i32, i32* [[IN_GEP]], align 4 +; OPT-GFX7-NEXT: br label [[ENDIF]] +; OPT-GFX7: endif: +; OPT-GFX7-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX7-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX7-NEXT: br label [[DONE:%.*]] +; OPT-GFX7: done: +; OPT-GFX7-NEXT: ret void +; +; OPT-GFX8-LABEL: @test_sinkable_flat_small_offset_i32( +; OPT-GFX8-NEXT: entry: +; OPT-GFX8-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX8-NEXT: [[IN_GEP:%.*]] = getelementptr i32, i32* [[IN:%.*]], i64 7 +; OPT-GFX8-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX8-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX8: if: +; OPT-GFX8-NEXT: [[LOAD:%.*]] = load i32, i32* [[IN_GEP]], align 4 +; OPT-GFX8-NEXT: br label [[ENDIF]] +; OPT-GFX8: endif: +; OPT-GFX8-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX8-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX8-NEXT: br label [[DONE:%.*]] +; OPT-GFX8: done: +; OPT-GFX8-NEXT: ret void +; +; OPT-GFX9-LABEL: @test_sinkable_flat_small_offset_i32( +; OPT-GFX9-NEXT: entry: +; OPT-GFX9-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX9-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX9-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX9: if: +; OPT-GFX9-NEXT: [[TMP0:%.*]] = bitcast i32* [[IN:%.*]] to i8* +; OPT-GFX9-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8* [[TMP0]], i64 28 +; OPT-GFX9-NEXT: [[TMP1:%.*]] = bitcast i8* [[SUNKADDR]] to i32* +; OPT-GFX9-NEXT: [[LOAD:%.*]] = load i32, i32* [[TMP1]], align 4 +; OPT-GFX9-NEXT: br label [[ENDIF]] +; OPT-GFX9: endif: +; OPT-GFX9-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX9-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX9-NEXT: ret void +; +; OPT-GFX10-LABEL: @test_sinkable_flat_small_offset_i32( +; OPT-GFX10-NEXT: entry: +; OPT-GFX10-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX10-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX10-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX10: if: +; OPT-GFX10-NEXT: [[TMP0:%.*]] = bitcast i32* [[IN:%.*]] to i8* +; OPT-GFX10-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8* [[TMP0]], i64 28 +; OPT-GFX10-NEXT: [[TMP1:%.*]] = bitcast i8* [[SUNKADDR]] to i32* +; OPT-GFX10-NEXT: [[LOAD:%.*]] = load i32, i32* [[TMP1]], align 4 +; OPT-GFX10-NEXT: br label [[ENDIF]] +; OPT-GFX10: endif: +; OPT-GFX10-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX10-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX10-NEXT: ret void +; +; GFX7-LABEL: test_sinkable_flat_small_offset_i32: +; GFX7: ; %bb.0: ; %entry +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX7-NEXT: v_mov_b32_e32 v4, 0 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB0_2 +; GFX7-NEXT: ; %bb.1: ; %if +; GFX7-NEXT: v_add_i32_e32 v2, vcc, 28, v2 +; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX7-NEXT: flat_load_dword v4, v[2:3] +; GFX7-NEXT: .LBB0_2: ; %endif +; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x3d08fc, v0 +; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: flat_store_dword v[0:1], v4 +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: test_sinkable_flat_small_offset_i32: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX8-NEXT: v_mov_b32_e32 v4, 0 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB0_2 +; GFX8-NEXT: ; %bb.1: ; %if +; GFX8-NEXT: v_add_u32_e32 v2, vcc, 28, v2 +; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX8-NEXT: flat_load_dword v4, v[2:3] +; GFX8-NEXT: .LBB0_2: ; %endif +; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x3d08fc, v0 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: flat_store_dword v[0:1], v4 +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: test_sinkable_flat_small_offset_i32: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB0_2 +; GFX9-NEXT: ; %bb.1: ; %if +; GFX9-NEXT: flat_load_dword v4, v[2:3] offset:28 +; GFX9-NEXT: .LBB0_2: ; %endif +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x3d0000, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: flat_store_dword v[0:1], v4 offset:2300 +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: test_sinkable_flat_small_offset_i32: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4 +; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB0_2 +; GFX10-NEXT: ; %bb.1: ; %if +; GFX10-NEXT: flat_load_dword v4, v[2:3] offset:28 +; GFX10-NEXT: .LBB0_2: ; %endif +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x3d0800, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-NEXT: flat_store_dword v[0:1], v4 offset:252 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: s_setpc_b64 s[30:31] entry: %out.gep = getelementptr i32, i32* %out, i64 999999 %in.gep = getelementptr i32, i32* %in, i64 7 - %tmp0 = icmp eq i32 %cond, 0 - br i1 %tmp0, label %endif, label %if + %cmp0 = icmp eq i32 %cond, 0 + br i1 %cmp0, label %endif, label %if if: - %tmp1 = load i32, i32* %in.gep + %load = load i32, i32* %in.gep br label %endif endif: - %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] + %x = phi i32 [ %load, %if ], [ 0, %entry ] store i32 %x, i32* %out.gep br label %done @@ -38,32 +170,162 @@ ret void } -; OPT-LABEL: @test_sink_noop_addrspacecast_flat_to_global_i32( -; OPT: getelementptr i32, i32* %out, -; rOPT-CI-NOT: getelementptr -; OPT: br i1 - -; OPT-CI: addrspacecast -; OPT-CI: getelementptr -; OPT-CI: bitcast -; OPT: br label - -; GCN-LABEL: {{^}}test_sink_noop_addrspacecast_flat_to_global_i32: -; CI: buffer_load_dword {{v[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:28 -define amdgpu_kernel void @test_sink_noop_addrspacecast_flat_to_global_i32(i32* %out, i32* %in, i32 %cond) { +define void @test_sink_noop_addrspacecast_flat_to_global_i32(i32* %out, i32* %in, i32 %cond) { +; OPT-GFX7-LABEL: @test_sink_noop_addrspacecast_flat_to_global_i32( +; OPT-GFX7-NEXT: entry: +; OPT-GFX7-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX7-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX7-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX7: if: +; OPT-GFX7-NEXT: [[TMP0:%.*]] = addrspacecast i32* [[IN:%.*]] to i8 addrspace(1)* +; OPT-GFX7-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 28 +; OPT-GFX7-NEXT: [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SUNKADDR]] to i32 addrspace(1)* +; OPT-GFX7-NEXT: [[LOAD:%.*]] = load i32, i32 addrspace(1)* [[TMP1]], align 4 +; OPT-GFX7-NEXT: br label [[ENDIF]] +; OPT-GFX7: endif: +; OPT-GFX7-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX7-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX7-NEXT: ret void +; +; OPT-GFX8-LABEL: @test_sink_noop_addrspacecast_flat_to_global_i32( +; OPT-GFX8-NEXT: entry: +; OPT-GFX8-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX8-NEXT: [[IN_GEP:%.*]] = getelementptr i32, i32* [[IN:%.*]], i64 7 +; OPT-GFX8-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX8-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX8: if: +; OPT-GFX8-NEXT: [[TMP0:%.*]] = addrspacecast i32* [[IN_GEP]] to i32 addrspace(1)* +; OPT-GFX8-NEXT: [[LOAD:%.*]] = load i32, i32 addrspace(1)* [[TMP0]], align 4 +; OPT-GFX8-NEXT: br label [[ENDIF]] +; OPT-GFX8: endif: +; OPT-GFX8-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX8-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX8-NEXT: ret void +; +; OPT-GFX9-LABEL: @test_sink_noop_addrspacecast_flat_to_global_i32( +; OPT-GFX9-NEXT: entry: +; OPT-GFX9-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX9-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX9-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX9: if: +; OPT-GFX9-NEXT: [[TMP0:%.*]] = addrspacecast i32* [[IN:%.*]] to i8 addrspace(1)* +; OPT-GFX9-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 28 +; OPT-GFX9-NEXT: [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SUNKADDR]] to i32 addrspace(1)* +; OPT-GFX9-NEXT: [[LOAD:%.*]] = load i32, i32 addrspace(1)* [[TMP1]], align 4 +; OPT-GFX9-NEXT: br label [[ENDIF]] +; OPT-GFX9: endif: +; OPT-GFX9-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX9-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX9-NEXT: ret void +; +; OPT-GFX10-LABEL: @test_sink_noop_addrspacecast_flat_to_global_i32( +; OPT-GFX10-NEXT: entry: +; OPT-GFX10-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-GFX10-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-GFX10-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX10: if: +; OPT-GFX10-NEXT: [[TMP0:%.*]] = addrspacecast i32* [[IN:%.*]] to i8 addrspace(1)* +; OPT-GFX10-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 28 +; OPT-GFX10-NEXT: [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SUNKADDR]] to i32 addrspace(1)* +; OPT-GFX10-NEXT: [[LOAD:%.*]] = load i32, i32 addrspace(1)* [[TMP1]], align 4 +; OPT-GFX10-NEXT: br label [[ENDIF]] +; OPT-GFX10: endif: +; OPT-GFX10-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX10-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX10-NEXT: ret void +; +; GFX7-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32: +; GFX7: ; %bb.0: ; %entry +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_mov_b32 s6, 0 +; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX7-NEXT: v_mov_b32_e32 v4, 0 +; GFX7-NEXT: s_and_saveexec_b64 s[8:9], vcc +; GFX7-NEXT: s_cbranch_execz .LBB1_2 +; GFX7-NEXT: ; %bb.1: ; %if +; GFX7-NEXT: s_mov_b32 s7, 0xf000 +; GFX7-NEXT: s_mov_b32 s4, s6 +; GFX7-NEXT: s_mov_b32 s5, s6 +; GFX7-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64 offset:28 +; GFX7-NEXT: .LBB1_2: ; %endif +; GFX7-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x3d08fc, v0 +; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: flat_store_dword v[0:1], v4 +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX8-NEXT: v_mov_b32_e32 v4, 0 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB1_2 +; GFX8-NEXT: ; %bb.1: ; %if +; GFX8-NEXT: v_add_u32_e32 v2, vcc, 28, v2 +; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX8-NEXT: flat_load_dword v4, v[2:3] +; GFX8-NEXT: .LBB1_2: ; %endif +; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x3d08fc, v0 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: flat_store_dword v[0:1], v4 +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB1_2 +; GFX9-NEXT: ; %bb.1: ; %if +; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:28 +; GFX9-NEXT: .LBB1_2: ; %endif +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x3d0000, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: flat_store_dword v[0:1], v4 offset:2300 +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4 +; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB1_2 +; GFX10-NEXT: ; %bb.1: ; %if +; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:28 +; GFX10-NEXT: .LBB1_2: ; %endif +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x3d0800, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: flat_store_dword v[0:1], v4 offset:252 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: s_setpc_b64 s[30:31] entry: %out.gep = getelementptr i32, i32* %out, i64 999999 %in.gep = getelementptr i32, i32* %in, i64 7 %cast = addrspacecast i32* %in.gep to i32 addrspace(1)* - %tmp0 = icmp eq i32 %cond, 0 - br i1 %tmp0, label %endif, label %if + %cmp0 = icmp eq i32 %cond, 0 + br i1 %cmp0, label %endif, label %if if: - %tmp1 = load i32, i32 addrspace(1)* %cast + %load = load i32, i32 addrspace(1)* %cast br label %endif endif: - %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] + %x = phi i32 [ %load, %if ], [ 0, %entry ] store i32 %x, i32* %out.gep br label %done @@ -71,32 +333,115 @@ ret void } +define void @test_sink_noop_addrspacecast_flat_to_constant_i32(i32* %out, i32* %in, i32 %cond) { ; OPT-LABEL: @test_sink_noop_addrspacecast_flat_to_constant_i32( -; OPT: getelementptr i32, i32* %out, -; OPT-CI-NOT: getelementptr -; OPT: br i1 - -; OPT-CI: addrspacecast -; OPT-CI: getelementptr -; OPT-CI: bitcast -; OPT: br label - -; GCN-LABEL: {{^}}test_sink_noop_addrspacecast_flat_to_constant_i32: -; CI: s_load_dword {{s[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0xd -define amdgpu_kernel void @test_sink_noop_addrspacecast_flat_to_constant_i32(i32* %out, i32* %in, i32 %cond) { +; OPT-NEXT: entry: +; OPT-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 999999 +; OPT-NEXT: [[CMP0:%.*]] = icmp eq i32 [[COND:%.*]], 0 +; OPT-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT: if: +; OPT-NEXT: [[TMP0:%.*]] = addrspacecast i32* [[IN:%.*]] to i8 addrspace(4)* +; OPT-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8 addrspace(4)* [[TMP0]], i64 28 +; OPT-NEXT: [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[SUNKADDR]] to i32 addrspace(4)* +; OPT-NEXT: [[LOAD:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4 +; OPT-NEXT: br label [[ENDIF]] +; OPT: endif: +; OPT-NEXT: [[X:%.*]] = phi i32 [ [[LOAD]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-NEXT: ret void +; +; GFX7-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32: +; GFX7: ; %bb.0: ; %entry +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_mov_b32 s6, 0 +; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX7-NEXT: v_mov_b32_e32 v4, 0 +; GFX7-NEXT: s_and_saveexec_b64 s[8:9], vcc +; GFX7-NEXT: s_cbranch_execz .LBB2_2 +; GFX7-NEXT: ; %bb.1: ; %if +; GFX7-NEXT: s_mov_b32 s7, 0xf000 +; GFX7-NEXT: s_mov_b32 s4, s6 +; GFX7-NEXT: s_mov_b32 s5, s6 +; GFX7-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64 offset:28 +; GFX7-NEXT: .LBB2_2: ; %endif +; GFX7-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x3d08fc, v0 +; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: flat_store_dword v[0:1], v4 +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX8-NEXT: v_mov_b32_e32 v4, 0 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB2_2 +; GFX8-NEXT: ; %bb.1: ; %if +; GFX8-NEXT: v_add_u32_e32 v2, vcc, 28, v2 +; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX8-NEXT: flat_load_dword v4, v[2:3] +; GFX8-NEXT: .LBB2_2: ; %endif +; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x3d08fc, v0 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: flat_store_dword v[0:1], v4 +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB2_2 +; GFX9-NEXT: ; %bb.1: ; %if +; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:28 +; GFX9-NEXT: .LBB2_2: ; %endif +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x3d0000, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: flat_store_dword v[0:1], v4 offset:2300 +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4 +; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB2_2 +; GFX10-NEXT: ; %bb.1: ; %if +; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:28 +; GFX10-NEXT: .LBB2_2: ; %endif +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x3d0800, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: flat_store_dword v[0:1], v4 offset:252 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: s_setpc_b64 s[30:31] entry: %out.gep = getelementptr i32, i32* %out, i64 999999 %in.gep = getelementptr i32, i32* %in, i64 7 %cast = addrspacecast i32* %in.gep to i32 addrspace(4)* - %tmp0 = icmp eq i32 %cond, 0 - br i1 %tmp0, label %endif, label %if + %cmp0 = icmp eq i32 %cond, 0 + br i1 %cmp0, label %endif, label %if if: - %tmp1 = load i32, i32 addrspace(4)* %cast + %load = load i32, i32 addrspace(4)* %cast br label %endif endif: - %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] + %x = phi i32 [ %load, %if ], [ 0, %entry ] store i32 %x, i32* %out.gep br label %done @@ -104,34 +449,174 @@ ret void } -; OPT-LABEL: @test_sink_flat_small_max_flat_offset( -; OPT-CIVI: %in.gep = getelementptr i8, i8* %in, i64 4095 -; OPT-CIVI: br -; OPT-CIVI-NOT: getelementptr -; OPT-CIVI: load i8, i8* %in.gep - -; OPT-GFX9: br -; OPT-GFX9: %sunkaddr = getelementptr i8, i8* %in, i64 4095 -; OPT-GFX9: load i8, i8* %sunkaddr - -; GCN-LABEL: {{^}}test_sink_flat_small_max_flat_offset: -; GFX9: flat_load_sbyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}} offset:4095{{$}} -; CIVI: flat_load_sbyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]$}} -define amdgpu_kernel void @test_sink_flat_small_max_flat_offset(i32* %out, i8* %in) #1 { +define void @test_sink_flat_small_max_flat_offset(i32* %out, i8* %in) #1 { +; OPT-GFX7-LABEL: @test_sink_flat_small_max_flat_offset( +; OPT-GFX7-NEXT: entry: +; OPT-GFX7-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i32 1024 +; OPT-GFX7-NEXT: [[IN_GEP:%.*]] = getelementptr i8, i8* [[IN:%.*]], i64 4095 +; OPT-GFX7-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #[[ATTR3:[0-9]+]] +; OPT-GFX7-NEXT: [[CMP0:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-GFX7-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX7: if: +; OPT-GFX7-NEXT: [[LOAD:%.*]] = load i8, i8* [[IN_GEP]], align 1 +; OPT-GFX7-NEXT: [[CAST:%.*]] = sext i8 [[LOAD]] to i32 +; OPT-GFX7-NEXT: br label [[ENDIF]] +; OPT-GFX7: endif: +; OPT-GFX7-NEXT: [[X:%.*]] = phi i32 [ [[CAST]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX7-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX7-NEXT: br label [[DONE:%.*]] +; OPT-GFX7: done: +; OPT-GFX7-NEXT: ret void +; +; OPT-GFX8-LABEL: @test_sink_flat_small_max_flat_offset( +; OPT-GFX8-NEXT: entry: +; OPT-GFX8-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i32 1024 +; OPT-GFX8-NEXT: [[IN_GEP:%.*]] = getelementptr i8, i8* [[IN:%.*]], i64 4095 +; OPT-GFX8-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #[[ATTR3:[0-9]+]] +; OPT-GFX8-NEXT: [[CMP0:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-GFX8-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX8: if: +; OPT-GFX8-NEXT: [[LOAD:%.*]] = load i8, i8* [[IN_GEP]], align 1 +; OPT-GFX8-NEXT: [[CAST:%.*]] = sext i8 [[LOAD]] to i32 +; OPT-GFX8-NEXT: br label [[ENDIF]] +; OPT-GFX8: endif: +; OPT-GFX8-NEXT: [[X:%.*]] = phi i32 [ [[CAST]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX8-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX8-NEXT: br label [[DONE:%.*]] +; OPT-GFX8: done: +; OPT-GFX8-NEXT: ret void +; +; OPT-GFX9-LABEL: @test_sink_flat_small_max_flat_offset( +; OPT-GFX9-NEXT: entry: +; OPT-GFX9-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i32 1024 +; OPT-GFX9-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #[[ATTR3:[0-9]+]] +; OPT-GFX9-NEXT: [[CMP0:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-GFX9-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX9: if: +; OPT-GFX9-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, i8* [[IN:%.*]], i64 4095 +; OPT-GFX9-NEXT: [[LOAD:%.*]] = load i8, i8* [[SUNKADDR]], align 1 +; OPT-GFX9-NEXT: [[CAST:%.*]] = sext i8 [[LOAD]] to i32 +; OPT-GFX9-NEXT: br label [[ENDIF]] +; OPT-GFX9: endif: +; OPT-GFX9-NEXT: [[X:%.*]] = phi i32 [ [[CAST]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX9-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX9-NEXT: ret void +; +; OPT-GFX10-LABEL: @test_sink_flat_small_max_flat_offset( +; OPT-GFX10-NEXT: entry: +; OPT-GFX10-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i32 1024 +; OPT-GFX10-NEXT: [[IN_GEP:%.*]] = getelementptr i8, i8* [[IN:%.*]], i64 4095 +; OPT-GFX10-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #[[ATTR3:[0-9]+]] +; OPT-GFX10-NEXT: [[CMP0:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-GFX10-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT-GFX10: if: +; OPT-GFX10-NEXT: [[LOAD:%.*]] = load i8, i8* [[IN_GEP]], align 1 +; OPT-GFX10-NEXT: [[CAST:%.*]] = sext i8 [[LOAD]] to i32 +; OPT-GFX10-NEXT: br label [[ENDIF]] +; OPT-GFX10: endif: +; OPT-GFX10-NEXT: [[X:%.*]] = phi i32 [ [[CAST]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-GFX10-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-GFX10-NEXT: br label [[DONE:%.*]] +; OPT-GFX10: done: +; OPT-GFX10-NEXT: ret void +; +; GFX7-LABEL: test_sink_flat_small_max_flat_offset: +; GFX7: ; %bb.0: ; %entry +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v5, -1, 0 +; GFX7-NEXT: v_mov_b32_e32 v4, 0 +; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB3_2 +; GFX7-NEXT: ; %bb.1: ; %if +; GFX7-NEXT: v_add_i32_e32 v2, vcc, 0xfff, v2 +; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX7-NEXT: flat_load_sbyte v4, v[2:3] +; GFX7-NEXT: .LBB3_2: ; %endif +; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x1000, v0 +; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: flat_store_dword v[0:1], v4 +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: test_sink_flat_small_max_flat_offset: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0 +; GFX8-NEXT: v_mov_b32_e32 v4, 0 +; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB3_2 +; GFX8-NEXT: ; %bb.1: ; %if +; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0xfff, v2 +; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX8-NEXT: flat_load_sbyte v4, v[2:3] +; GFX8-NEXT: .LBB3_2: ; %endif +; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x1000, v0 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: flat_store_dword v[0:1], v4 +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: test_sink_flat_small_max_flat_offset: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB3_2 +; GFX9-NEXT: ; %bb.1: ; %if +; GFX9-NEXT: flat_load_sbyte v4, v[2:3] offset:4095 +; GFX9-NEXT: .LBB3_2: ; %endif +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: flat_store_dword v[0:1], v4 +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: test_sink_flat_small_max_flat_offset: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4 +; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB3_2 +; GFX10-NEXT: ; %bb.1: ; %if +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0x800, v2 +; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo +; GFX10-NEXT: flat_load_sbyte v4, v[2:3] offset:2047 +; GFX10-NEXT: .LBB3_2: ; %endif +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x1000, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-NEXT: flat_store_dword v[0:1], v4 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: s_setpc_b64 s[30:31] entry: %out.gep = getelementptr i32, i32* %out, i32 1024 %in.gep = getelementptr i8, i8* %in, i64 4095 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 - %tmp0 = icmp eq i32 %tid, 0 - br i1 %tmp0, label %endif, label %if + %cmp0 = icmp eq i32 %tid, 0 + br i1 %cmp0, label %endif, label %if if: - %tmp1 = load i8, i8* %in.gep - %tmp2 = sext i8 %tmp1 to i32 + %load = load i8, i8* %in.gep + %cast = sext i8 %load to i32 br label %endif endif: - %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] + %x = phi i32 [ %cast, %if ], [ 0, %entry ] store i32 %x, i32* %out.gep br label %done @@ -139,29 +624,124 @@ ret void } +define void @test_sink_flat_small_max_plus_1_flat_offset(i32* %out, i8* %in) #1 { ; OPT-LABEL: @test_sink_flat_small_max_plus_1_flat_offset( -; OPT: %in.gep = getelementptr i8, i8* %in, i64 4096 -; OPT: br -; OPT-NOT: getelementptr -; OPT: load i8, i8* %in.gep - -; GCN-LABEL: {{^}}test_sink_flat_small_max_plus_1_flat_offset: -; GCN: flat_load_sbyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]$}} -define amdgpu_kernel void @test_sink_flat_small_max_plus_1_flat_offset(i32* %out, i8* %in) #1 { +; OPT-NEXT: entry: +; OPT-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 99999 +; OPT-NEXT: [[IN_GEP:%.*]] = getelementptr i8, i8* [[IN:%.*]], i64 4096 +; OPT-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #[[ATTR3:[0-9]+]] +; OPT-NEXT: [[CMP0:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT: if: +; OPT-NEXT: [[LOAD:%.*]] = load i8, i8* [[IN_GEP]], align 1 +; OPT-NEXT: [[CAST:%.*]] = sext i8 [[LOAD]] to i32 +; OPT-NEXT: br label [[ENDIF]] +; OPT: endif: +; OPT-NEXT: [[X:%.*]] = phi i32 [ [[CAST]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-NEXT: br label [[DONE:%.*]] +; OPT: done: +; OPT-NEXT: ret void +; +; GFX7-LABEL: test_sink_flat_small_max_plus_1_flat_offset: +; GFX7: ; %bb.0: ; %entry +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v5, -1, 0 +; GFX7-NEXT: v_mov_b32_e32 v4, 0 +; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB4_2 +; GFX7-NEXT: ; %bb.1: ; %if +; GFX7-NEXT: v_add_i32_e32 v2, vcc, 0x1000, v2 +; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX7-NEXT: flat_load_sbyte v4, v[2:3] +; GFX7-NEXT: .LBB4_2: ; %endif +; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x61a7c, v0 +; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: flat_store_dword v[0:1], v4 +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: test_sink_flat_small_max_plus_1_flat_offset: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0 +; GFX8-NEXT: v_mov_b32_e32 v4, 0 +; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB4_2 +; GFX8-NEXT: ; %bb.1: ; %if +; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x1000, v2 +; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc +; GFX8-NEXT: flat_load_sbyte v4, v[2:3] +; GFX8-NEXT: .LBB4_2: ; %endif +; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x61a7c, v0 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: flat_store_dword v[0:1], v4 +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: test_sink_flat_small_max_plus_1_flat_offset: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB4_2 +; GFX9-NEXT: ; %bb.1: ; %if +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, 0x1000, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc +; GFX9-NEXT: flat_load_sbyte v4, v[2:3] +; GFX9-NEXT: .LBB4_2: ; %endif +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x61000, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: flat_store_dword v[0:1], v4 offset:2684 +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: test_sink_flat_small_max_plus_1_flat_offset: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4 +; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB4_2 +; GFX10-NEXT: ; %bb.1: ; %if +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0x1000, v2 +; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo +; GFX10-NEXT: flat_load_sbyte v4, v[2:3] +; GFX10-NEXT: .LBB4_2: ; %endif +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x61800, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-NEXT: flat_store_dword v[0:1], v4 offset:636 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: s_setpc_b64 s[30:31] entry: %out.gep = getelementptr i32, i32* %out, i64 99999 %in.gep = getelementptr i8, i8* %in, i64 4096 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 - %tmp0 = icmp eq i32 %tid, 0 - br i1 %tmp0, label %endif, label %if + %cmp0 = icmp eq i32 %tid, 0 + br i1 %cmp0, label %endif, label %if if: - %tmp1 = load i8, i8* %in.gep - %tmp2 = sext i8 %tmp1 to i32 + %load = load i8, i8* %in.gep + %cast = sext i8 %load to i32 br label %endif endif: - %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] + %x = phi i32 [ %cast, %if ], [ 0, %entry ] store i32 %x, i32* %out.gep br label %done @@ -169,30 +749,124 @@ ret void } -; OPT-LABEL: @test_no_sink_flat_reg_offset( -; OPT: %in.gep = getelementptr i8, i8* %in, i64 %reg -; OPT: br - -; OPT-NOT: getelementptr -; OPT: load i8, i8* %in.gep - -; GCN-LABEL: {{^}}test_no_sink_flat_reg_offset: -; GCN: flat_load_sbyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]$}} -define amdgpu_kernel void @test_no_sink_flat_reg_offset(i32* %out, i8* %in, i64 %reg) #1 { +define void @test_sinkable_flat_reg_offset(i32* %out, i8* %in, i64 %reg) #1 { +; OPT-LABEL: @test_sinkable_flat_reg_offset( +; OPT-NEXT: entry: +; OPT-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i32 1024 +; OPT-NEXT: [[IN_GEP:%.*]] = getelementptr i8, i8* [[IN:%.*]], i64 [[REG:%.*]] +; OPT-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #[[ATTR3]] +; OPT-NEXT: [[CMP0:%.*]] = icmp eq i32 [[TID]], 0 +; OPT-NEXT: br i1 [[CMP0]], label [[ENDIF:%.*]], label [[IF:%.*]] +; OPT: if: +; OPT-NEXT: [[LOAD:%.*]] = load i8, i8* [[IN_GEP]], align 1 +; OPT-NEXT: [[CAST:%.*]] = sext i8 [[LOAD]] to i32 +; OPT-NEXT: br label [[ENDIF]] +; OPT: endif: +; OPT-NEXT: [[X:%.*]] = phi i32 [ [[CAST]], [[IF]] ], [ 0, [[ENTRY:%.*]] ] +; OPT-NEXT: store i32 [[X]], i32* [[OUT_GEP]], align 4 +; OPT-NEXT: br label [[DONE:%.*]] +; OPT: done: +; OPT-NEXT: ret void +; +; GFX7-LABEL: test_sinkable_flat_reg_offset: +; GFX7: ; %bb.0: ; %entry +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v7, -1, 0 +; GFX7-NEXT: v_mov_b32_e32 v6, 0 +; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB5_2 +; GFX7-NEXT: ; %bb.1: ; %if +; GFX7-NEXT: v_add_i32_e32 v2, vcc, v2, v4 +; GFX7-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc +; GFX7-NEXT: flat_load_sbyte v6, v[2:3] +; GFX7-NEXT: .LBB5_2: ; %endif +; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x1000, v0 +; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: flat_store_dword v[0:1], v6 +; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX7-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: test_sinkable_flat_reg_offset: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0 +; GFX8-NEXT: v_mov_b32_e32 v6, 0 +; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB5_2 +; GFX8-NEXT: ; %bb.1: ; %if +; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc +; GFX8-NEXT: flat_load_sbyte v6, v[2:3] +; GFX8-NEXT: .LBB5_2: ; %endif +; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x1000, v0 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: flat_store_dword v[0:1], v6 +; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: test_sinkable_flat_reg_offset: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB5_2 +; GFX9-NEXT: ; %bb.1: ; %if +; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc +; GFX9-NEXT: flat_load_sbyte v6, v[2:3] +; GFX9-NEXT: .LBB5_2: ; %endif +; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: flat_store_dword v[0:1], v6 +; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: test_sinkable_flat_reg_offset: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v6, -1, 0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6 +; GFX10-NEXT: v_mov_b32_e32 v6, 0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB5_2 +; GFX10-NEXT: ; %bb.1: ; %if +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 +; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo +; GFX10-NEXT: flat_load_sbyte v6, v[2:3] +; GFX10-NEXT: .LBB5_2: ; %endif +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x1000, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-NEXT: flat_store_dword v[0:1], v6 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: s_setpc_b64 s[30:31] entry: %out.gep = getelementptr i32, i32* %out, i32 1024 %in.gep = getelementptr i8, i8* %in, i64 %reg %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 - %tmp0 = icmp eq i32 %tid, 0 - br i1 %tmp0, label %endif, label %if + %cmp0 = icmp eq i32 %tid, 0 + br i1 %cmp0, label %endif, label %if if: - %tmp1 = load i8, i8* %in.gep - %tmp2 = sext i8 %tmp1 to i32 + %load = load i8, i8* %in.gep + %cast = sext i8 %load to i32 br label %endif endif: - %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] + %x = phi i32 [ %cast, %if ], [ 0, %entry ] store i32 %x, i32* %out.gep br label %done