diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -1002,6 +1002,13 @@ if (TM.getOptLevel() > CodeGenOpt::None) addPass(createInferAddressSpacesPass()); + // Run atomic optimizer before Atomic Expand + if ((TM.getTargetTriple().getArch() == Triple::amdgcn) && + (TM.getOptLevel() >= CodeGenOpt::Less) && + (AMDGPUAtomicOptimizerStrategy != ScanOptions::None)) { + addPass(createAMDGPUAtomicOptimizerPass(AMDGPUAtomicOptimizerStrategy)); + } + addPass(createAtomicExpandPass()); if (TM.getOptLevel() > CodeGenOpt::None) { @@ -1128,11 +1135,6 @@ if (TM->getOptLevel() > CodeGenOpt::None) addPass(createAMDGPULateCodeGenPreparePass()); - if ((TM->getOptLevel() >= CodeGenOpt::Less) && - (AMDGPUAtomicOptimizerStrategy != ScanOptions::None)) { - addPass(createAMDGPUAtomicOptimizerPass(AMDGPUAtomicOptimizerStrategy)); - } - if (TM->getOptLevel() > CodeGenOpt::None) addPass(createSinkingPass()); diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll @@ -15,6 +15,7 @@ ; GFX908_GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2 ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32 [[REG_SEQUENCE]], [[COPY2]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_no_rtn_intrinsic ; GFX90A_GFX940: bb.1 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -41,6 +42,7 @@ ; GFX908_GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY2]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_saddr_no_rtn_intrinsic ; GFX90A_GFX940: bb.1 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -67,6 +69,7 @@ ; GFX908_GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2 ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32 [[REG_SEQUENCE]], [[COPY2]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_no_rtn_flat_intrinsic ; GFX90A_GFX940: bb.1 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -93,6 +96,7 @@ ; GFX908_GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY2]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_saddr_no_rtn_flat_intrinsic ; GFX90A_GFX940: bb.1 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -119,6 +123,7 @@ ; GFX908_GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2 ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32 [[REG_SEQUENCE]], [[COPY2]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_no_rtn_atomicrmw ; GFX90A_GFX940: bb.1 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -136,7 +141,7 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspace(1) inreg %ptr, float %data) #0 { ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_saddr_no_rtn_atomicrmw ; GFX90A_GFX940: bb.1 (%ir-block.0): - ; GFX90A_GFX940-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000) + ; GFX90A_GFX940-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000) ; GFX90A_GFX940-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 @@ -144,11 +149,11 @@ ; GFX90A_GFX940-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1 ; GFX90A_GFX940-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX90A_GFX940-NEXT: [[SI_PS_LIVE:%[0-9]+]]:sreg_64_xexec = SI_PS_LIVE - ; GFX90A_GFX940-NEXT: [[SI_IF:%[0-9]+]]:sreg_64_xexec = SI_IF [[SI_PS_LIVE]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec + ; GFX90A_GFX940-NEXT: [[SI_IF:%[0-9]+]]:sreg_64_xexec = SI_IF [[SI_PS_LIVE]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX90A_GFX940-NEXT: S_BRANCH %bb.2 ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: bb.2 (%ir-block.5): - ; GFX90A_GFX940-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000) + ; GFX90A_GFX940-NEXT: successors: %bb.3(0x40000000), %bb.4(0x40000000) ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: [[COPY3:%[0-9]+]]:sreg_64 = COPY $exec ; GFX90A_GFX940-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0 @@ -185,29 +190,22 @@ ; GFX90A_GFX940-NEXT: [[STRICT_WWM:%[0-9]+]]:vgpr_32 = STRICT_WWM [[COPY16]], implicit $exec ; GFX90A_GFX940-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]] ; GFX90A_GFX940-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY17]], implicit $exec - ; GFX90A_GFX940-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec + ; GFX90A_GFX940-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX90A_GFX940-NEXT: S_BRANCH %bb.3 ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: bb.3 (%ir-block.54): - ; GFX90A_GFX940-NEXT: successors: %bb.5(0x80000000) + ; GFX90A_GFX940-NEXT: bb.3 (%ir-block.46): + ; GFX90A_GFX940-NEXT: successors: %bb.4(0x80000000) ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec ; GFX90A_GFX940-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[STRICT_WWM]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) - ; GFX90A_GFX940-NEXT: S_BRANCH %bb.5 ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: bb.4.Flow: - ; GFX90A_GFX940-NEXT: successors: %bb.6(0x80000000) - ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec - ; GFX90A_GFX940-NEXT: S_BRANCH %bb.6 - ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: bb.5 (%ir-block.56): - ; GFX90A_GFX940-NEXT: successors: %bb.4(0x80000000) + ; GFX90A_GFX940-NEXT: successors: %bb.5(0x80000000) ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec - ; GFX90A_GFX940-NEXT: S_BRANCH %bb.4 ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: bb.6 (%ir-block.57): + ; GFX90A_GFX940-NEXT: bb.5 (%ir-block.48): + ; GFX90A_GFX940-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec ; GFX90A_GFX940-NEXT: S_ENDPGM 0 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic ret void diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll @@ -15,6 +15,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_RTN [[REG_SEQUENCE]], [[COPY2]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_rtn_intrinsic ; GFX11: bb.1 (%ir-block.0): ; GFX11-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -43,6 +44,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN [[V_MOV_B32_e32_]], [[COPY2]], [[REG_SEQUENCE]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_saddr_rtn_intrinsic ; GFX11: bb.1 (%ir-block.0): ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -71,6 +73,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_RTN [[REG_SEQUENCE]], [[COPY2]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_rtn_flat_intrinsic ; GFX11: bb.1 (%ir-block.0): ; GFX11-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -99,6 +102,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN [[V_MOV_B32_e32_]], [[COPY2]], [[REG_SEQUENCE]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_saddr_rtn_flat_intrinsic ; GFX11: bb.1 (%ir-block.0): ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -127,6 +131,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_RTN [[REG_SEQUENCE]], [[COPY2]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_rtn_atomicrmw ; GFX11: bb.1 (%ir-block.0): ; GFX11-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -199,7 +204,7 @@ ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.3 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.3 (%ir-block.52): + ; GFX11-NEXT: bb.3 (%ir-block.42): ; GFX11-NEXT: successors: %bb.5(0x80000000) ; GFX11-NEXT: {{ $}} ; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec @@ -213,7 +218,7 @@ ; GFX11-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.6 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.5 (%ir-block.55): + ; GFX11-NEXT: bb.5 (%ir-block.45): ; GFX11-NEXT: successors: %bb.4(0x80000000) ; GFX11-NEXT: {{ $}} ; GFX11-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]], %bb.3, [[DEF]], %bb.2 @@ -224,7 +229,7 @@ ; GFX11-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY15]], 0, [[STRICT_WWM1]], 0, 0, implicit $mode, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.4 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.6 (%ir-block.63): + ; GFX11-NEXT: bb.6 (%ir-block.53): ; GFX11-NEXT: $vgpr0 = COPY [[PHI]] ; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll --- a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll @@ -14,8 +14,7 @@ ; GFX10-NEXT: s_mov_b32 s4, 0 ; GFX10-NEXT: s_mov_b32 s2, 0 ; GFX10-NEXT: s_branch .LBB0_2 -; GFX10-NEXT: .LBB0_1: ; %Flow -; GFX10-NEXT: ; in Loop: Header=BB0_2 Depth=1 +; GFX10-NEXT: .LBB0_1: ; in Loop: Header=BB0_2 Depth=1 ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3 ; GFX10-NEXT: s_and_b32 s0, exec_lo, vcc_lo diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll @@ -41,6 +41,7 @@ ; GFX7-NEXT: .LBB0_4: ; %Flow ; GFX7-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX7-NEXT: s_wqm_b64 s[4:5], -1 +; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX7-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GFX7-NEXT: s_cbranch_vccnz .LBB0_6 ; GFX7-NEXT: ; %bb.5: ; %if @@ -75,6 +76,7 @@ ; GFX89-NEXT: .LBB0_4: ; %Flow ; GFX89-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX89-NEXT: s_wqm_b64 s[4:5], -1 +; GFX89-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX89-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GFX89-NEXT: s_cbranch_vccnz .LBB0_6 ; GFX89-NEXT: ; %bb.5: ; %if @@ -110,6 +112,7 @@ ; GFX1064-NEXT: .LBB0_4: ; %Flow ; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1064-NEXT: s_wqm_b64 s[4:5], -1 +; GFX1064-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX1064-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GFX1064-NEXT: s_cbranch_vccnz .LBB0_6 ; GFX1064-NEXT: ; %bb.5: ; %if @@ -144,6 +147,7 @@ ; GFX1032-NEXT: .LBB0_4: ; %Flow ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1032-NEXT: s_wqm_b32 s4, -1 +; GFX1032-NEXT: s_and_b32 s4, s4, s4 ; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4 ; GFX1032-NEXT: s_cbranch_vccnz .LBB0_6 ; GFX1032-NEXT: ; %bb.5: ; %if @@ -182,7 +186,8 @@ ; GFX1164-NEXT: .LBB0_4: ; %Flow ; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1164-NEXT: s_wqm_b64 s[4:5], -1 -; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX1164-NEXT: s_and_not1_b64 vcc, exec, s[4:5] ; GFX1164-NEXT: s_cbranch_vccnz .LBB0_6 ; GFX1164-NEXT: ; %bb.5: ; %if @@ -222,7 +227,8 @@ ; GFX1132-NEXT: .LBB0_4: ; %Flow ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1132-NEXT: s_wqm_b32 s4, -1 -; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_b32 s4, s4, s4 ; GFX1132-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4 ; GFX1132-NEXT: s_cbranch_vccnz .LBB0_6 ; GFX1132-NEXT: ; %bb.5: ; %if @@ -307,6 +313,7 @@ ; GFX8-NEXT: .LBB1_4: ; %Flow ; GFX8-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX8-NEXT: s_wqm_b64 s[4:5], -1 +; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX8-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GFX8-NEXT: s_cbranch_vccnz .LBB1_6 ; GFX8-NEXT: ; %bb.5: ; %if @@ -363,6 +370,7 @@ ; GFX9-NEXT: .LBB1_4: ; %Flow ; GFX9-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX9-NEXT: s_wqm_b64 s[4:5], -1 +; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GFX9-NEXT: s_cbranch_vccnz .LBB1_6 ; GFX9-NEXT: ; %bb.5: ; %if @@ -426,6 +434,7 @@ ; GFX1064-NEXT: .LBB1_4: ; %Flow ; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1064-NEXT: s_wqm_b64 s[4:5], -1 +; GFX1064-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX1064-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GFX1064-NEXT: s_cbranch_vccnz .LBB1_6 ; GFX1064-NEXT: ; %bb.5: ; %if @@ -480,6 +489,7 @@ ; GFX1032-NEXT: .LBB1_4: ; %Flow ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1032-NEXT: s_wqm_b32 s4, -1 +; GFX1032-NEXT: s_and_b32 s4, s4, s4 ; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4 ; GFX1032-NEXT: s_cbranch_vccnz .LBB1_6 ; GFX1032-NEXT: ; %bb.5: ; %if @@ -552,7 +562,8 @@ ; GFX1164-NEXT: .LBB1_4: ; %Flow ; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1164-NEXT: s_wqm_b64 s[4:5], -1 -; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5] ; GFX1164-NEXT: s_and_not1_b64 vcc, exec, s[4:5] ; GFX1164-NEXT: s_cbranch_vccnz .LBB1_6 ; GFX1164-NEXT: ; %bb.5: ; %if @@ -616,7 +627,8 @@ ; GFX1132-NEXT: .LBB1_4: ; %Flow ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1132-NEXT: s_wqm_b32 s4, -1 -; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_b32 s4, s4, s4 ; GFX1132-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4 ; GFX1132-NEXT: s_cbranch_vccnz .LBB1_6 ; GFX1132-NEXT: ; %bb.5: ; %if diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll @@ -40,11 +40,11 @@ ; GCN-NEXT: s_cbranch_execz .LBB0_3 ; GCN-NEXT: ; %bb.2: ; GCN-NEXT: s_bcnt1_i32_b64 s8, s[8:9] -; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s8 -; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: v_add_f32_e32 v1, v1, v1 +; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, s8 +; GCN-NEXT: v_add_f32_e32 v0, v0, v0 +; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: global_atomic_add_f32 v0, v1, s[2:3] offset:28 +; GCN-NEXT: global_atomic_add_f32 v1, v0, s[2:3] offset:28 ; GCN-NEXT: .LBB0_3: ; GCN-NEXT: s_or_b64 exec, exec, s[6:7] ; GCN-NEXT: global_load_dword v0, v[0:1], off glc diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll --- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll @@ -16,6 +16,7 @@ ; GFX908_GFX11-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]] ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32 killed [[COPY3]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_no_rtn_intrinsic ; GFX90A_GFX940: bb.0 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -43,6 +44,7 @@ ; GFX908_GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_saddr_no_rtn_intrinsic ; GFX90A_GFX940: bb.0 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -70,6 +72,7 @@ ; GFX908_GFX11-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]] ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32 killed [[COPY3]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_no_rtn_flat_intrinsic ; GFX90A_GFX940: bb.0 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -97,6 +100,7 @@ ; GFX908_GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_saddr_no_rtn_flat_intrinsic ; GFX90A_GFX940: bb.0 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -124,6 +128,7 @@ ; GFX908_GFX11-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]] ; GFX908_GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32 killed [[COPY3]], [[COPY]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) ; GFX908_GFX11-NEXT: S_ENDPGM 0 + ; ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_no_rtn_atomicrmw ; GFX90A_GFX940: bb.0 (%ir-block.0): ; GFX90A_GFX940-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -142,7 +147,7 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspace(1) inreg %ptr, float %data) #0 { ; GFX90A_GFX940-LABEL: name: global_atomic_fadd_f32_saddr_no_rtn_atomicrmw ; GFX90A_GFX940: bb.0 (%ir-block.0): - ; GFX90A_GFX940-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000) + ; GFX90A_GFX940-NEXT: successors: %bb.1(0x40000000), %bb.4(0x40000000) ; GFX90A_GFX940-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 @@ -151,11 +156,11 @@ ; GFX90A_GFX940-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1 ; GFX90A_GFX940-NEXT: [[COPY3:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE]] ; GFX90A_GFX940-NEXT: [[SI_PS_LIVE:%[0-9]+]]:sreg_64 = SI_PS_LIVE - ; GFX90A_GFX940-NEXT: [[SI_IF:%[0-9]+]]:sreg_64 = SI_IF killed [[SI_PS_LIVE]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec + ; GFX90A_GFX940-NEXT: [[SI_IF:%[0-9]+]]:sreg_64 = SI_IF killed [[SI_PS_LIVE]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX90A_GFX940-NEXT: S_BRANCH %bb.1 ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: bb.1 (%ir-block.5): - ; GFX90A_GFX940-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000) + ; GFX90A_GFX940-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000) ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec ; GFX90A_GFX940-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY4]].sub0 @@ -180,30 +185,23 @@ ; GFX90A_GFX940-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32 = V_READLANE_B32 killed [[V_ADD_F32_e64_5]], killed [[S_MOV_B32_]] ; GFX90A_GFX940-NEXT: early-clobber %1:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec ; GFX90A_GFX940-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[V_MOV_B32_e32_]], implicit $exec - ; GFX90A_GFX940-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec + ; GFX90A_GFX940-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX90A_GFX940-NEXT: S_BRANCH %bb.2 ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: bb.2 (%ir-block.54): - ; GFX90A_GFX940-NEXT: successors: %bb.4(0x80000000) + ; GFX90A_GFX940-NEXT: bb.2 (%ir-block.46): + ; GFX90A_GFX940-NEXT: successors: %bb.3(0x80000000) ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec ; GFX90A_GFX940-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY %1 ; GFX90A_GFX940-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_1]], [[COPY7]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) - ; GFX90A_GFX940-NEXT: S_BRANCH %bb.4 ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: bb.3.Flow: - ; GFX90A_GFX940-NEXT: successors: %bb.5(0x80000000) - ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec - ; GFX90A_GFX940-NEXT: S_BRANCH %bb.5 - ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: bb.4 (%ir-block.56): - ; GFX90A_GFX940-NEXT: successors: %bb.3(0x80000000) + ; GFX90A_GFX940-NEXT: successors: %bb.4(0x80000000) ; GFX90A_GFX940-NEXT: {{ $}} ; GFX90A_GFX940-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec - ; GFX90A_GFX940-NEXT: S_BRANCH %bb.3 ; GFX90A_GFX940-NEXT: {{ $}} - ; GFX90A_GFX940-NEXT: bb.5 (%ir-block.57): + ; GFX90A_GFX940-NEXT: bb.4 (%ir-block.48): + ; GFX90A_GFX940-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX90A_GFX940-NEXT: S_ENDPGM 0 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic ret void diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll --- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll @@ -16,6 +16,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_RTN killed [[COPY3]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_rtn_intrinsic ; GFX11: bb.0 (%ir-block.0): ; GFX11-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -45,6 +46,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_]], [[COPY]], killed [[REG_SEQUENCE]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_saddr_rtn_intrinsic ; GFX11: bb.0 (%ir-block.0): ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -74,6 +76,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_RTN killed [[COPY3]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_rtn_flat_intrinsic ; GFX11: bb.0 (%ir-block.0): ; GFX11-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -103,6 +106,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_]], [[COPY]], killed [[REG_SEQUENCE]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_saddr_rtn_flat_intrinsic ; GFX11: bb.0 (%ir-block.0): ; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0 @@ -132,6 +136,7 @@ ; GFX90A_GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_RTN killed [[COPY3]], [[COPY]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) ; GFX90A_GFX940-NEXT: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_F32_RTN]] ; GFX90A_GFX940-NEXT: SI_RETURN_TO_EPILOG $vgpr0 + ; ; GFX11-LABEL: name: global_atomic_fadd_f32_rtn_atomicrmw ; GFX11: bb.0 (%ir-block.0): ; GFX11-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2 @@ -196,7 +201,7 @@ ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.2 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.2 (%ir-block.52): + ; GFX11-NEXT: bb.2 (%ir-block.42): ; GFX11-NEXT: successors: %bb.4(0x80000000) ; GFX11-NEXT: {{ $}} ; GFX11-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec @@ -211,7 +216,7 @@ ; GFX11-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.5 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.4 (%ir-block.55): + ; GFX11-NEXT: bb.4 (%ir-block.45): ; GFX11-NEXT: successors: %bb.3(0x80000000) ; GFX11-NEXT: {{ $}} ; GFX11-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF1]], %bb.1, [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]], %bb.2 @@ -221,7 +226,7 @@ ; GFX11-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_READFIRSTLANE_B32_]], 0, killed %42, 0, 0, implicit $mode, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.3 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.5 (%ir-block.63): + ; GFX11-NEXT: bb.5 (%ir-block.53): ; GFX11-NEXT: $vgpr0 = COPY [[PHI]] ; GFX11-NEXT: SI_RETURN_TO_EPILOG $vgpr0 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic diff --git a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll --- a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll @@ -8,138 +8,216 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32(ptr addrspace(1) %ptr) #0 { ; GFX900-LABEL: global_atomic_fadd_ret_f32: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[4:5], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: ; implicit-def: $vgpr1 +; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX900-NEXT: s_cbranch_execz .LBB0_4 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v0, 0 +; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX900-NEXT: s_mov_b64 s[4:5], 0 +; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB0_1: ; %atomicrmw.start +; GFX900-NEXT: v_mov_b32_e32 v1, s6 +; GFX900-NEXT: .LBB0_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_mov_b32_e32 v2, v1 -; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX900-NEXT: v_mov_b32_e32 v5, v1 +; GFX900-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: buffer_wbinvl1_vol -; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB0_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX900-NEXT: s_cbranch_execnz .LBB0_2 +; GFX900-NEXT: ; %bb.3: ; %Flow +; GFX900-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX900-NEXT: .LBB0_4: ; %Flow1 ; GFX900-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX900-NEXT: global_store_dword v[0:1], v1, off +; GFX900-NEXT: v_readfirstlane_b32 s0, v1 +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX900-NEXT: global_store_dword v[0:1], v0, off ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: global_atomic_fadd_ret_f32: ; GFX908: ; %bb.0: +; GFX908-NEXT: s_mov_b64 s[4:5], exec +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX908-NEXT: ; implicit-def: $vgpr1 +; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX908-NEXT: s_cbranch_execz .LBB0_4 +; GFX908-NEXT: ; %bb.1: ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX908-NEXT: s_mov_b64 s[2:3], 0 -; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX908-NEXT: s_mov_b64 s[4:5], 0 +; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX908-NEXT: v_mov_b32_e32 v3, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start +; GFX908-NEXT: v_mov_b32_e32 v1, s6 +; GFX908-NEXT: .LBB0_2: ; %atomicrmw.start ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: v_mov_b32_e32 v2, v1 -; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX908-NEXT: v_mov_b32_e32 v5, v1 +; GFX908-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: buffer_wbinvl1_vol -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX908-NEXT: s_cbranch_execnz .LBB0_1 -; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX908-NEXT: s_cbranch_execnz .LBB0_2 +; GFX908-NEXT: ; %bb.3: ; %Flow +; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX908-NEXT: .LBB0_4: ; %Flow1 ; GFX908-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX908-NEXT: global_store_dword v[0:1], v1, off +; GFX908-NEXT: v_readfirstlane_b32 s0, v1 +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX908-NEXT: global_store_dword v[0:1], v0, off ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: global_atomic_fadd_ret_f32: ; GFX90A: ; %bb.0: +; GFX90A-NEXT: s_mov_b64 s[4:5], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: ; implicit-def: $vgpr1 +; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB0_4 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX90A-NEXT: s_mov_b64 s[4:5], 0 +; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v3, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v1, s4 -; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start +; GFX90A-NEXT: v_mov_b32_e32 v1, s6 +; GFX90A-NEXT: .LBB0_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_mov_b32_e32 v3, v1 -; GFX90A-NEXT: v_add_f32_e32 v2, 4.0, v3 +; GFX90A-NEXT: v_mov_b32_e32 v5, v1 +; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX90A-NEXT: buffer_wbl2 ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol -; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB0_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX90A-NEXT: s_cbranch_execnz .LBB0_2 +; GFX90A-NEXT: ; %bb.3: ; %Flow +; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX90A-NEXT: .LBB0_4: ; %Flow1 ; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX90A-NEXT: global_store_dword v[0:1], v1, off +; GFX90A-NEXT: v_readfirstlane_b32 s0, v1 +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX90A-NEXT: global_store_dword v[0:1], v0, off ; GFX90A-NEXT: s_endpgm ; ; GFX10-LABEL: global_atomic_fadd_ret_f32: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s4, exec_lo +; GFX10-NEXT: s_mov_b32 s3, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX10-NEXT: ; implicit-def: $vgpr1 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB0_4 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4 +; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB0_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-NEXT: .LBB0_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_mov_b32_e32 v2, v1 -; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX10-NEXT: v_mov_b32_e32 v5, v1 +; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 -; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB0_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 +; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 +; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: s_cbranch_execnz .LBB0_2 +; GFX10-NEXT: ; %bb.3: ; %Flow +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: .LBB0_4: ; %Flow1 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: global_store_dword v[0:1], v1, off +; GFX10-NEXT: v_readfirstlane_b32 s0, v1 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX10-NEXT: global_store_dword v[0:1], v0, off ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_ret_f32: ; GFX11: ; %bb.0: +; GFX11-NEXT: s_mov_b32 s4, exec_lo +; GFX11-NEXT: s_mov_b32 s3, 0 +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX11-NEXT: s_mov_b32 s2, exec_lo +; GFX11-NEXT: ; implicit-def: $vgpr1 +; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX11-NEXT: s_cbranch_execz .LBB0_4 +; GFX11-NEXT: ; %bb.1: ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4 +; GFX11-NEXT: v_mov_b32_e32 v3, 0 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x0 +; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_mov_b32_e32 v1, s2 -; GFX11-NEXT: s_mov_b32 s2, 0 -; GFX11-NEXT: .LBB0_1: ; %atomicrmw.start +; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5 +; GFX11-NEXT: .LBB0_2: ; %atomicrmw.start ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-NEXT: v_mov_b32_e32 v2, v1 -; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v1 +; GFX11-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc +; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-NEXT: s_or_b32 s2, vcc_lo, s2 -; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 -; GFX11-NEXT: s_cbranch_execnz .LBB0_1 -; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 +; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3 +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3 +; GFX11-NEXT: s_cbranch_execnz .LBB0_2 +; GFX11-NEXT: ; %bb.3: ; %Flow +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX11-NEXT: .LBB0_4: ; %Flow1 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 -; GFX11-NEXT: global_store_b32 v[0:1], v1, off +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX11-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0 +; GFX11-NEXT: v_add_f32_e32 v0, s0, v0 +; GFX11-NEXT: global_store_b32 v[0:1], v0, off ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_endpgm @@ -151,54 +229,88 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(ptr addrspace(1) %ptr) #2 { ; GFX900-LABEL: global_atomic_fadd_ret_f32_ieee: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[4:5], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: ; implicit-def: $vgpr1 +; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX900-NEXT: s_cbranch_execz .LBB1_4 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v0, 0 +; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX900-NEXT: s_mov_b64 s[4:5], 0 +; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB1_1: ; %atomicrmw.start +; GFX900-NEXT: v_mov_b32_e32 v1, s6 +; GFX900-NEXT: .LBB1_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_mov_b32_e32 v2, v1 -; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX900-NEXT: v_mov_b32_e32 v5, v1 +; GFX900-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: buffer_wbinvl1_vol -; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB1_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX900-NEXT: s_cbranch_execnz .LBB1_2 +; GFX900-NEXT: ; %bb.3: ; %Flow +; GFX900-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX900-NEXT: .LBB1_4: ; %Flow1 ; GFX900-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX900-NEXT: global_store_dword v[0:1], v1, off +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX900-NEXT: v_readfirstlane_b32 s0, v1 +; GFX900-NEXT: v_mul_f32_e32 v0, 4.0, v0 +; GFX900-NEXT: v_add_f32_e32 v0, s0, v0 +; GFX900-NEXT: global_store_dword v[0:1], v0, off ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: global_atomic_fadd_ret_f32_ieee: ; GFX908: ; %bb.0: +; GFX908-NEXT: s_mov_b64 s[4:5], exec +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX908-NEXT: ; implicit-def: $vgpr1 +; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX908-NEXT: s_cbranch_execz .LBB1_4 +; GFX908-NEXT: ; %bb.1: ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX908-NEXT: s_mov_b64 s[2:3], 0 -; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX908-NEXT: s_mov_b64 s[4:5], 0 +; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX908-NEXT: v_mov_b32_e32 v3, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start +; GFX908-NEXT: v_mov_b32_e32 v1, s6 +; GFX908-NEXT: .LBB1_2: ; %atomicrmw.start ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: v_mov_b32_e32 v2, v1 -; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX908-NEXT: v_mov_b32_e32 v5, v1 +; GFX908-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: buffer_wbinvl1_vol -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX908-NEXT: s_cbranch_execnz .LBB1_1 -; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX908-NEXT: s_cbranch_execnz .LBB1_2 +; GFX908-NEXT: ; %bb.3: ; %Flow +; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX908-NEXT: .LBB1_4: ; %Flow1 ; GFX908-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX908-NEXT: global_store_dword v[0:1], v1, off +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX908-NEXT: v_readfirstlane_b32 s0, v1 +; GFX908-NEXT: v_mul_f32_e32 v0, 4.0, v0 +; GFX908-NEXT: v_add_f32_e32 v0, s0, v0 +; GFX908-NEXT: global_store_dword v[0:1], v0, off ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: global_atomic_fadd_ret_f32_ieee: @@ -231,30 +343,46 @@ ; ; GFX10-LABEL: global_atomic_fadd_ret_f32_ieee: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s4, exec_lo +; GFX10-NEXT: s_mov_b32 s3, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX10-NEXT: ; implicit-def: $vgpr1 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB1_4 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4 +; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB1_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-NEXT: .LBB1_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_mov_b32_e32 v2, v1 -; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX10-NEXT: v_mov_b32_e32 v5, v1 +; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 -; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB1_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 +; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 +; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: s_cbranch_execnz .LBB1_2 +; GFX10-NEXT: ; %bb.3: ; %Flow +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: .LBB1_4: ; %Flow1 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: global_store_dword v[0:1], v1, off +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX10-NEXT: v_readfirstlane_b32 s0, v1 +; GFX10-NEXT: v_mul_f32_e32 v0, 4.0, v0 +; GFX10-NEXT: v_add_f32_e32 v0, s0, v0 +; GFX10-NEXT: global_store_dword v[0:1], v0, off ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_ret_f32_ieee: @@ -294,26 +422,36 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32(ptr addrspace(1) %ptr) #0 { ; GFX900-LABEL: global_atomic_fadd_noret_f32: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[2:3], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX900-NEXT: s_cbranch_execz .LBB2_3 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3] +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5 ; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v2, 0 +; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB2_1: ; %atomicrmw.start +; GFX900-NEXT: .LBB2_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX900-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX900-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: buffer_wbinvl1_vol ; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX900-NEXT: v_mov_b32_e32 v1, v0 ; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB2_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: s_cbranch_execnz .LBB2_2 +; GFX900-NEXT: .LBB2_3: ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: global_atomic_fadd_noret_f32: @@ -360,19 +498,28 @@ ; ; GFX10-LABEL: global_atomic_fadd_noret_f32: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s3, exec_lo +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB2_3 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v2, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3 +; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB2_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: .LBB2_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX10-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv @@ -380,8 +527,8 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB2_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: s_cbranch_execnz .LBB2_2 +; GFX10-NEXT: .LBB2_3: ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_noret_f32: @@ -411,26 +558,36 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(ptr addrspace(1) %ptr) #2 { ; GFX900-LABEL: global_atomic_fadd_noret_f32_ieee: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[2:3], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX900-NEXT: s_cbranch_execz .LBB3_3 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3] +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5 ; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v2, 0 +; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB3_1: ; %atomicrmw.start +; GFX900-NEXT: .LBB3_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX900-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX900-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: buffer_wbinvl1_vol ; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX900-NEXT: v_mov_b32_e32 v1, v0 ; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB3_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: s_cbranch_execnz .LBB3_2 +; GFX900-NEXT: .LBB3_3: ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: global_atomic_fadd_noret_f32_ieee: @@ -477,19 +634,28 @@ ; ; GFX10-LABEL: global_atomic_fadd_noret_f32_ieee: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s3, exec_lo +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB3_3 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v2, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3 +; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: .LBB3_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX10-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv @@ -497,8 +663,8 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB3_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: s_cbranch_execnz .LBB3_2 +; GFX10-NEXT: .LBB3_3: ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_noret_f32_ieee: @@ -528,54 +694,86 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(ptr addrspace(1) %ptr) #0 { ; GFX900-LABEL: global_atomic_fadd_ret_f32_agent: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[4:5], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: ; implicit-def: $vgpr1 +; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX900-NEXT: s_cbranch_execz .LBB4_4 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v0, 0 +; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX900-NEXT: s_mov_b64 s[4:5], 0 +; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB4_1: ; %atomicrmw.start +; GFX900-NEXT: v_mov_b32_e32 v1, s6 +; GFX900-NEXT: .LBB4_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_mov_b32_e32 v2, v1 -; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX900-NEXT: v_mov_b32_e32 v5, v1 +; GFX900-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: buffer_wbinvl1_vol -; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB4_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX900-NEXT: s_cbranch_execnz .LBB4_2 +; GFX900-NEXT: ; %bb.3: ; %Flow +; GFX900-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX900-NEXT: .LBB4_4: ; %Flow1 ; GFX900-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX900-NEXT: global_store_dword v[0:1], v1, off +; GFX900-NEXT: v_readfirstlane_b32 s0, v1 +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX900-NEXT: global_store_dword v[0:1], v0, off ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: global_atomic_fadd_ret_f32_agent: ; GFX908: ; %bb.0: +; GFX908-NEXT: s_mov_b64 s[4:5], exec +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX908-NEXT: ; implicit-def: $vgpr1 +; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX908-NEXT: s_cbranch_execz .LBB4_4 +; GFX908-NEXT: ; %bb.1: ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX908-NEXT: s_mov_b64 s[2:3], 0 -; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX908-NEXT: s_mov_b64 s[4:5], 0 +; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX908-NEXT: v_mov_b32_e32 v3, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start +; GFX908-NEXT: v_mov_b32_e32 v1, s6 +; GFX908-NEXT: .LBB4_2: ; %atomicrmw.start ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: v_mov_b32_e32 v2, v1 -; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX908-NEXT: v_mov_b32_e32 v5, v1 +; GFX908-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: buffer_wbinvl1_vol -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX908-NEXT: s_cbranch_execnz .LBB4_1 -; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX908-NEXT: s_cbranch_execnz .LBB4_2 +; GFX908-NEXT: ; %bb.3: ; %Flow +; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX908-NEXT: .LBB4_4: ; %Flow1 ; GFX908-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX908-NEXT: global_store_dword v[0:1], v1, off +; GFX908-NEXT: v_readfirstlane_b32 s0, v1 +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX908-NEXT: global_store_dword v[0:1], v0, off ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: global_atomic_fadd_ret_f32_agent: @@ -607,30 +805,45 @@ ; ; GFX10-LABEL: global_atomic_fadd_ret_f32_agent: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s4, exec_lo +; GFX10-NEXT: s_mov_b32 s3, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX10-NEXT: ; implicit-def: $vgpr1 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB4_4 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4 +; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB4_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-NEXT: .LBB4_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_mov_b32_e32 v2, v1 -; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX10-NEXT: v_mov_b32_e32 v5, v1 +; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 -; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB4_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 +; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 +; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: s_cbranch_execnz .LBB4_2 +; GFX10-NEXT: ; %bb.3: ; %Flow +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: .LBB4_4: ; %Flow1 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: global_store_dword v[0:1], v1, off +; GFX10-NEXT: v_readfirstlane_b32 s0, v1 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX10-NEXT: global_store_dword v[0:1], v0, off ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_ret_f32_agent: @@ -670,138 +883,216 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(ptr addrspace(1) %ptr) #0 { ; GFX900-LABEL: global_atomic_fadd_ret_f32_system: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[4:5], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: ; implicit-def: $vgpr1 +; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX900-NEXT: s_cbranch_execz .LBB5_4 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v0, 0 +; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX900-NEXT: s_mov_b64 s[4:5], 0 +; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) -; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB5_1: ; %atomicrmw.start +; GFX900-NEXT: v_mov_b32_e32 v1, s6 +; GFX900-NEXT: .LBB5_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_mov_b32_e32 v2, v1 -; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX900-NEXT: v_mov_b32_e32 v5, v1 +; GFX900-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: buffer_wbinvl1_vol -; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB5_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX900-NEXT: s_cbranch_execnz .LBB5_2 +; GFX900-NEXT: ; %bb.3: ; %Flow +; GFX900-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX900-NEXT: .LBB5_4: ; %Flow1 ; GFX900-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX900-NEXT: global_store_dword v[0:1], v1, off +; GFX900-NEXT: v_readfirstlane_b32 s0, v1 +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX900-NEXT: global_store_dword v[0:1], v0, off ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: global_atomic_fadd_ret_f32_system: ; GFX908: ; %bb.0: +; GFX908-NEXT: s_mov_b64 s[4:5], exec +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX908-NEXT: ; implicit-def: $vgpr1 +; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX908-NEXT: s_cbranch_execz .LBB5_4 +; GFX908-NEXT: ; %bb.1: ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX908-NEXT: s_mov_b64 s[2:3], 0 -; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX908-NEXT: s_mov_b64 s[4:5], 0 +; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX908-NEXT: v_mov_b32_e32 v3, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start +; GFX908-NEXT: v_mov_b32_e32 v1, s6 +; GFX908-NEXT: .LBB5_2: ; %atomicrmw.start ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: v_mov_b32_e32 v2, v1 -; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX908-NEXT: v_mov_b32_e32 v5, v1 +; GFX908-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: buffer_wbinvl1_vol -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX908-NEXT: s_cbranch_execnz .LBB5_1 -; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX908-NEXT: s_cbranch_execnz .LBB5_2 +; GFX908-NEXT: ; %bb.3: ; %Flow +; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX908-NEXT: .LBB5_4: ; %Flow1 ; GFX908-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX908-NEXT: global_store_dword v[0:1], v1, off +; GFX908-NEXT: v_readfirstlane_b32 s0, v1 +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX908-NEXT: global_store_dword v[0:1], v0, off ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: global_atomic_fadd_ret_f32_system: ; GFX90A: ; %bb.0: +; GFX90A-NEXT: s_mov_b64 s[4:5], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: ; implicit-def: $vgpr1 +; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB5_4 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX90A-NEXT: s_mov_b64 s[4:5], 0 +; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v3, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v1, s4 -; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start +; GFX90A-NEXT: v_mov_b32_e32 v1, s6 +; GFX90A-NEXT: .LBB5_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_mov_b32_e32 v3, v1 -; GFX90A-NEXT: v_add_f32_e32 v2, 4.0, v3 +; GFX90A-NEXT: v_mov_b32_e32 v5, v1 +; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX90A-NEXT: buffer_wbl2 ; GFX90A-NEXT: s_waitcnt vmcnt(0) -; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol -; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 -; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB5_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX90A-NEXT: s_cbranch_execnz .LBB5_2 +; GFX90A-NEXT: ; %bb.3: ; %Flow +; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX90A-NEXT: .LBB5_4: ; %Flow1 ; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX90A-NEXT: global_store_dword v[0:1], v1, off +; GFX90A-NEXT: v_readfirstlane_b32 s0, v1 +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX90A-NEXT: global_store_dword v[0:1], v0, off ; GFX90A-NEXT: s_endpgm ; ; GFX10-LABEL: global_atomic_fadd_ret_f32_system: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s4, exec_lo +; GFX10-NEXT: s_mov_b32 s3, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX10-NEXT: ; implicit-def: $vgpr1 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB5_4 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4 +; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB5_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-NEXT: .LBB5_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_mov_b32_e32 v2, v1 -; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX10-NEXT: v_mov_b32_e32 v5, v1 +; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 -; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB5_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 +; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 +; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: s_cbranch_execnz .LBB5_2 +; GFX10-NEXT: ; %bb.3: ; %Flow +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: .LBB5_4: ; %Flow1 ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: global_store_dword v[0:1], v1, off +; GFX10-NEXT: v_readfirstlane_b32 s0, v1 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX10-NEXT: global_store_dword v[0:1], v0, off ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_ret_f32_system: ; GFX11: ; %bb.0: +; GFX11-NEXT: s_mov_b32 s4, exec_lo +; GFX11-NEXT: s_mov_b32 s3, 0 +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX11-NEXT: s_mov_b32 s2, exec_lo +; GFX11-NEXT: ; implicit-def: $vgpr1 +; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX11-NEXT: s_cbranch_execz .LBB5_4 +; GFX11-NEXT: ; %bb.1: ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4 +; GFX11-NEXT: v_mov_b32_e32 v3, 0 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x0 +; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_mov_b32_e32 v1, s2 -; GFX11-NEXT: s_mov_b32 s2, 0 -; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start +; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5 +; GFX11-NEXT: .LBB5_2: ; %atomicrmw.start ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-NEXT: v_mov_b32_e32 v2, v1 -; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v1 +; GFX11-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc +; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-NEXT: s_or_b32 s2, vcc_lo, s2 -; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 -; GFX11-NEXT: s_cbranch_execnz .LBB5_1 -; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 +; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3 +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3 +; GFX11-NEXT: s_cbranch_execnz .LBB5_2 +; GFX11-NEXT: ; %bb.3: ; %Flow +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX11-NEXT: .LBB5_4: ; %Flow1 ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 -; GFX11-NEXT: global_store_b32 v[0:1], v1, off +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX11-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0 +; GFX11-NEXT: v_add_f32_e32 v0, s0, v0 +; GFX11-NEXT: global_store_b32 v[0:1], v0, off ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_endpgm @@ -813,54 +1104,86 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 { ; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget: ; GCN: ; %bb.0: +; GCN-NEXT: s_mov_b64 s[4:5], exec +; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GCN-NEXT: ; implicit-def: $vgpr1 +; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GCN-NEXT: s_cbranch_execz .LBB6_4 +; GCN-NEXT: ; %bb.1: ; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GCN-NEXT: s_mov_b64 s[2:3], 0 -; GCN-NEXT: v_mov_b32_e32 v0, 0 +; GCN-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GCN-NEXT: s_mov_b64 s[4:5], 0 +; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_load_dword s4, s[0:1], 0x0 +; GCN-NEXT: s_load_dword s6, s[0:1], 0x0 +; GCN-NEXT: v_mov_b32_e32 v3, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: v_mov_b32_e32 v1, s4 -; GCN-NEXT: .LBB6_1: ; %atomicrmw.start +; GCN-NEXT: v_mov_b32_e32 v1, s6 +; GCN-NEXT: .LBB6_2: ; %atomicrmw.start ; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 -; GCN-NEXT: v_mov_b32_e32 v2, v1 -; GCN-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GCN-NEXT: v_mov_b32_e32 v5, v1 +; GCN-NEXT: v_add_f32_e32 v4, v5, v2 ; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GCN-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: buffer_wbinvl1_vol -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GCN-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GCN-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GCN-NEXT: s_cbranch_execnz .LBB6_1 -; GCN-NEXT: ; %bb.2: ; %atomicrmw.end +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GCN-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GCN-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GCN-NEXT: s_cbranch_execnz .LBB6_2 +; GCN-NEXT: ; %bb.3: ; %Flow +; GCN-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-NEXT: .LBB6_4: ; %Flow1 ; GCN-NEXT: s_or_b64 exec, exec, s[2:3] -; GCN-NEXT: global_store_dword v[0:1], v1, off +; GCN-NEXT: v_readfirstlane_b32 s0, v1 +; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GCN-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GCN-NEXT: global_store_dword v[0:1], v0, off ; GCN-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget: ; GFX11: ; %bb.0: +; GFX11-NEXT: s_mov_b64 s[4:5], exec +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0 +; GFX11-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX11-NEXT: ; implicit-def: $vgpr1 +; GFX11-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX11-NEXT: s_cbranch_execz .LBB6_4 +; GFX11-NEXT: ; %bb.1: ; GFX11-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX11-NEXT: s_mov_b64 s[2:3], 0 -; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: s_bcnt1_i32_b64 s7, s[4:5] +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s7 +; GFX11-NEXT: s_mov_b64 s[4:5], 0 +; GFX11-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX11-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_mov_b32_e32 v1, s4 -; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start +; GFX11-NEXT: v_mov_b32_e32 v1, s6 +; GFX11-NEXT: .LBB6_2: ; %atomicrmw.start ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-NEXT: v_mov_b32_e32 v2, v1 -; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v1 +; GFX11-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc +; GFX11-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: buffer_wbinvl1_vol -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 -; GFX11-NEXT: s_or_b64 s[2:3], vcc, s[2:3] -; GFX11-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX11-NEXT: s_cbranch_execnz .LBB6_1 -; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5 +; GFX11-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX11-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX11-NEXT: s_cbranch_execnz .LBB6_2 +; GFX11-NEXT: ; %bb.3: ; %Flow +; GFX11-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX11-NEXT: .LBB6_4: ; %Flow1 ; GFX11-NEXT: s_or_b64 exec, exec, s[2:3] -; GFX11-NEXT: global_store_dword v[0:1], v1, off +; GFX11-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX11-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GFX11-NEXT: global_store_dword v[0:1], v0, off ; GFX11-NEXT: s_endpgm %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst store float %result, ptr addrspace(1) undef @@ -916,91 +1239,130 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(ptr addrspace(1) %ptr) { ; GFX900-LABEL: global_atomic_fadd_noret_f32_safe: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[2:3], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX900-NEXT: s_cbranch_execz .LBB8_3 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3] +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5 ; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v2, 0 +; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB8_1: ; %atomicrmw.start +; GFX900-NEXT: .LBB8_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX900-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX900-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: buffer_wbinvl1_vol ; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX900-NEXT: v_mov_b32_e32 v1, v0 ; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB8_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: s_cbranch_execnz .LBB8_2 +; GFX900-NEXT: .LBB8_3: ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: global_atomic_fadd_noret_f32_safe: ; GFX908: ; %bb.0: +; GFX908-NEXT: s_mov_b64 s[2:3], exec +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX908-NEXT: s_cbranch_execz .LBB8_3 +; GFX908-NEXT: ; %bb.1: ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX908-NEXT: s_bcnt1_i32_b64 s5, s[2:3] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, s5 ; GFX908-NEXT: s_mov_b64 s[2:3], 0 -; GFX908-NEXT: v_mov_b32_e32 v2, 0 +; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX908-NEXT: v_mov_b32_e32 v3, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start +; GFX908-NEXT: .LBB8_2: ; %atomicrmw.start ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX908-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX908-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX908-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: buffer_wbinvl1_vol ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX908-NEXT: v_mov_b32_e32 v1, v0 ; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX908-NEXT: s_cbranch_execnz .LBB8_1 -; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX908-NEXT: s_cbranch_execnz .LBB8_2 +; GFX908-NEXT: .LBB8_3: ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: global_atomic_fadd_noret_f32_safe: ; GFX90A: ; %bb.0: +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB8_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s5, s[2:3] +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, s5 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v3, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v1, s4 -; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB8_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX90A-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_mov_b32_e32 v1, v0 ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB8_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB8_2 +; GFX90A-NEXT: .LBB8_3: ; GFX90A-NEXT: s_endpgm ; ; GFX10-LABEL: global_atomic_fadd_noret_f32_safe: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s3, exec_lo +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB8_3 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v2, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3 +; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: .LBB8_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX10-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv @@ -1008,25 +1370,33 @@ ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB8_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: s_cbranch_execnz .LBB8_2 +; GFX10-NEXT: .LBB8_3: ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_fadd_noret_f32_safe: ; GFX11: ; %bb.0: +; GFX11-NEXT: s_mov_b32 s3, exec_lo +; GFX11-NEXT: s_mov_b32 s2, 0 +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX11-NEXT: s_mov_b32 s4, exec_lo +; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX11-NEXT: s_cbranch_execz .LBB8_3 +; GFX11-NEXT: ; %bb.1: ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 -; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3 +; GFX11-NEXT: v_mov_b32_e32 v3, 0 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s3 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x0 +; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_mov_b32_e32 v1, s2 -; GFX11-NEXT: s_mov_b32 s2, 0 -; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start +; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4 +; GFX11-NEXT: .LBB8_2: ; %atomicrmw.start ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-NEXT: v_add_f32_e32 v0, 4.0, v1 +; GFX11-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc +; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv @@ -1034,8 +1404,8 @@ ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 -; GFX11-NEXT: s_cbranch_execnz .LBB8_1 -; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_cbranch_execnz .LBB8_2 +; GFX11-NEXT: .LBB8_3: ; GFX11-NEXT: s_endpgm %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst ret void @@ -1044,26 +1414,35 @@ define amdgpu_kernel void @infer_as_before_atomic(ptr addrspace(4) %arg) #0 { ; GFX900-LABEL: infer_as_before_atomic: ; GFX900: ; %bb.0: +; GFX900-NEXT: s_mov_b64 s[2:3], exec +; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX900-NEXT: s_cbranch_execz .LBB9_3 +; GFX900-NEXT: ; %bb.1: ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3] ; GFX900-NEXT: s_mov_b64 s[2:3], 0 -; GFX900-NEXT: v_mov_b32_e32 v2, 0 +; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v2, s5 +; GFX900-NEXT: v_mov_b32_e32 v3, 0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: v_mov_b32_e32 v1, s4 -; GFX900-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX900-NEXT: .LBB9_2: ; %atomicrmw.start ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX900-NEXT: v_add_f32_e32 v0, 1.0, v1 -; GFX900-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX900-NEXT: v_add_f32_e32 v0, v1, v2 +; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX900-NEXT: v_mov_b32_e32 v1, v0 ; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX900-NEXT: s_cbranch_execnz .LBB9_1 -; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX900-NEXT: s_cbranch_execnz .LBB9_2 +; GFX900-NEXT: .LBB9_3: ; GFX900-NEXT: s_endpgm ; ; GFX908-LABEL: infer_as_before_atomic: @@ -1108,26 +1487,34 @@ ; ; GFX10-LABEL: infer_as_before_atomic: ; GFX10: ; %bb.0: +; GFX10-NEXT: s_mov_b32 s3, exec_lo +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB9_3 +; GFX10-NEXT: ; %bb.1: ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX10-NEXT: v_mov_b32_e32 v2, 0 +; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3 +; GFX10-NEXT: v_mov_b32_e32 v3, 0 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v2, s3 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, s2 -; GFX10-NEXT: s_mov_b32 s2, 0 -; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: .LBB9_2: ; %atomicrmw.start ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: v_add_f32_e32 v0, 1.0, v1 -; GFX10-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc +; GFX10-NEXT: v_add_f32_e32 v0, v1, v2 +; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 -; GFX10-NEXT: s_cbranch_execnz .LBB9_1 -; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX10-NEXT: s_cbranch_execnz .LBB9_2 +; GFX10-NEXT: .LBB9_3: ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: infer_as_before_atomic: diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -186,6 +186,10 @@ ; GCN-O1-NEXT: Cycle Info Analysis ; GCN-O1-NEXT: FunctionPass Manager ; GCN-O1-NEXT: Infer address spaces +; GCN-O1-NEXT: Dominator Tree Construction +; GCN-O1-NEXT: Cycle Info Analysis +; GCN-O1-NEXT: Uniformity Analysis +; GCN-O1-NEXT: AMDGPU atomic optimizations ; GCN-O1-NEXT: Expand Atomic instructions ; GCN-O1-NEXT: AMDGPU Promote Alloca ; GCN-O1-NEXT: Dominator Tree Construction @@ -242,14 +246,11 @@ ; GCN-O1-NEXT: Cycle Info Analysis ; GCN-O1-NEXT: Uniformity Analysis ; GCN-O1-NEXT: AMDGPU IR late optimizations -; GCN-O1-NEXT: AMDGPU atomic optimizations ; GCN-O1-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-NEXT: Function Alias Analysis Results ; GCN-O1-NEXT: Natural Loop Information ; GCN-O1-NEXT: Code sinking ; GCN-O1-NEXT: Post-Dominator Tree Construction -; GCN-O1-NEXT: Cycle Info Analysis -; GCN-O1-NEXT: Uniformity Analysis ; GCN-O1-NEXT: Unify divergent function exit nodes ; GCN-O1-NEXT: Lazy Value Information Analysis ; GCN-O1-NEXT: Lower SwitchInst's to branches @@ -458,6 +459,10 @@ ; GCN-O1-OPTS-NEXT: Cycle Info Analysis ; GCN-O1-OPTS-NEXT: FunctionPass Manager ; GCN-O1-OPTS-NEXT: Infer address spaces +; GCN-O1-OPTS-NEXT: Dominator Tree Construction +; GCN-O1-OPTS-NEXT: Cycle Info Analysis +; GCN-O1-OPTS-NEXT: Uniformity Analysis +; GCN-O1-OPTS-NEXT: AMDGPU atomic optimizations ; GCN-O1-OPTS-NEXT: Expand Atomic instructions ; GCN-O1-OPTS-NEXT: AMDGPU Promote Alloca ; GCN-O1-OPTS-NEXT: Dominator Tree Construction @@ -528,14 +533,11 @@ ; GCN-O1-OPTS-NEXT: Cycle Info Analysis ; GCN-O1-OPTS-NEXT: Uniformity Analysis ; GCN-O1-OPTS-NEXT: AMDGPU IR late optimizations -; GCN-O1-OPTS-NEXT: AMDGPU atomic optimizations ; GCN-O1-OPTS-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-OPTS-NEXT: Function Alias Analysis Results ; GCN-O1-OPTS-NEXT: Natural Loop Information ; GCN-O1-OPTS-NEXT: Code sinking ; GCN-O1-OPTS-NEXT: Post-Dominator Tree Construction -; GCN-O1-OPTS-NEXT: Cycle Info Analysis -; GCN-O1-OPTS-NEXT: Uniformity Analysis ; GCN-O1-OPTS-NEXT: Unify divergent function exit nodes ; GCN-O1-OPTS-NEXT: Lazy Value Information Analysis ; GCN-O1-OPTS-NEXT: Lower SwitchInst's to branches @@ -752,6 +754,10 @@ ; GCN-O2-NEXT: Cycle Info Analysis ; GCN-O2-NEXT: FunctionPass Manager ; GCN-O2-NEXT: Infer address spaces +; GCN-O2-NEXT: Dominator Tree Construction +; GCN-O2-NEXT: Cycle Info Analysis +; GCN-O2-NEXT: Uniformity Analysis +; GCN-O2-NEXT: AMDGPU atomic optimizations ; GCN-O2-NEXT: Expand Atomic instructions ; GCN-O2-NEXT: AMDGPU Promote Alloca ; GCN-O2-NEXT: Dominator Tree Construction @@ -830,14 +836,11 @@ ; GCN-O2-NEXT: Cycle Info Analysis ; GCN-O2-NEXT: Uniformity Analysis ; GCN-O2-NEXT: AMDGPU IR late optimizations -; GCN-O2-NEXT: AMDGPU atomic optimizations ; GCN-O2-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O2-NEXT: Function Alias Analysis Results ; GCN-O2-NEXT: Natural Loop Information ; GCN-O2-NEXT: Code sinking ; GCN-O2-NEXT: Post-Dominator Tree Construction -; GCN-O2-NEXT: Cycle Info Analysis -; GCN-O2-NEXT: Uniformity Analysis ; GCN-O2-NEXT: Unify divergent function exit nodes ; GCN-O2-NEXT: Lazy Value Information Analysis ; GCN-O2-NEXT: Lower SwitchInst's to branches @@ -1056,6 +1059,10 @@ ; GCN-O3-NEXT: Cycle Info Analysis ; GCN-O3-NEXT: FunctionPass Manager ; GCN-O3-NEXT: Infer address spaces +; GCN-O3-NEXT: Dominator Tree Construction +; GCN-O3-NEXT: Cycle Info Analysis +; GCN-O3-NEXT: Uniformity Analysis +; GCN-O3-NEXT: AMDGPU atomic optimizations ; GCN-O3-NEXT: Expand Atomic instructions ; GCN-O3-NEXT: AMDGPU Promote Alloca ; GCN-O3-NEXT: Dominator Tree Construction @@ -1146,14 +1153,11 @@ ; GCN-O3-NEXT: Cycle Info Analysis ; GCN-O3-NEXT: Uniformity Analysis ; GCN-O3-NEXT: AMDGPU IR late optimizations -; GCN-O3-NEXT: AMDGPU atomic optimizations ; GCN-O3-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O3-NEXT: Function Alias Analysis Results ; GCN-O3-NEXT: Natural Loop Information ; GCN-O3-NEXT: Code sinking ; GCN-O3-NEXT: Post-Dominator Tree Construction -; GCN-O3-NEXT: Cycle Info Analysis -; GCN-O3-NEXT: Uniformity Analysis ; GCN-O3-NEXT: Unify divergent function exit nodes ; GCN-O3-NEXT: Lazy Value Information Analysis ; GCN-O3-NEXT: Lower SwitchInst's to branches diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll --- a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll +++ b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll @@ -320,53 +320,78 @@ ; GFX7-LABEL: lds_ds_fadd: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb -; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_mov_b64 s[6:7], exec +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_lshl_b32 s4, s3, 3 -; GFX7-NEXT: s_add_i32 s4, s4, 32 -; GFX7-NEXT: v_mov_b32_e32 v0, s4 -; GFX7-NEXT: ds_read_b32 v0, v0 ; GFX7-NEXT: s_add_i32 s3, s3, 4 -; GFX7-NEXT: s_lshl_b32 s6, s3, 3 -; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: .LBB2_1: ; %atomicrmw.start +; GFX7-NEXT: ; implicit-def: $vgpr1 +; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB2_4 +; GFX7-NEXT: ; %bb.1: +; GFX7-NEXT: s_lshl_b32 s8, s3, 3 +; GFX7-NEXT: v_mov_b32_e32 v1, s8 +; GFX7-NEXT: ds_read_b32 v1, v1 +; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s6 +; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 +; GFX7-NEXT: s_mov_b64 s[6:7], 0 +; GFX7-NEXT: .LBB2_2: ; %atomicrmw.start ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_mov_b32_e32 v1, v0 -; GFX7-NEXT: v_mov_b32_e32 v0, s6 -; GFX7-NEXT: v_add_f32_e32 v2, 0x42280000, v1 +; GFX7-NEXT: v_mov_b32_e32 v3, v1 +; GFX7-NEXT: v_mov_b32_e32 v1, s8 +; GFX7-NEXT: v_add_f32_e32 v4, v3, v2 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: ds_cmpst_rtn_b32 v0, v0, v1, v2 +; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 -; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB2_1 -; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX7-NEXT: s_cbranch_execnz .LBB2_2 +; GFX7-NEXT: ; %bb.3: ; %Flow15 +; GFX7-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX7-NEXT: .LBB2_4: ; %Flow16 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX7-NEXT: s_mov_b64 s[6:7], exec +; GFX7-NEXT: v_readfirstlane_b32 s8, v1 +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB2_7 +; GFX7-NEXT: ; %bb.5: ; GFX7-NEXT: s_lshl_b32 s3, s3, 4 ; GFX7-NEXT: v_mov_b32_e32 v1, s3 -; GFX7-NEXT: ds_read_b32 v1, v1 -; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: .LBB2_3: ; %atomicrmw.start2 +; GFX7-NEXT: ds_read_b32 v2, v1 +; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 +; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX7-NEXT: s_mov_b64 s[6:7], 0 +; GFX7-NEXT: .LBB2_6: ; %atomicrmw.start2 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_add_f32_e32 v2, 0x42280000, v1 -; GFX7-NEXT: v_mov_b32_e32 v3, s3 +; GFX7-NEXT: v_add_f32_e32 v3, v2, v1 +; GFX7-NEXT: v_mov_b32_e32 v4, s3 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v1, v2 +; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1 -; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX7-NEXT: v_mov_b32_e32 v1, v2 -; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB2_3 -; GFX7-NEXT: ; %bb.4: ; %atomicrmw.end1 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2 +; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX7-NEXT: v_mov_b32_e32 v2, v3 +; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX7-NEXT: s_cbranch_execnz .LBB2_6 +; GFX7-NEXT: .LBB2_7: ; %Flow14 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX7-NEXT: v_mov_b32_e32 v1, s2 ; GFX7-NEXT: ds_read_b32 v1, v1 +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX7-NEXT: v_add_f32_e32 v0, s8, v0 ; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: .LBB2_5: ; %atomicrmw.start8 +; GFX7-NEXT: .LBB2_8: ; %atomicrmw.start8 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v2, v1 @@ -378,8 +403,8 @@ ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB2_5 -; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end7 +; GFX7-NEXT: s_cbranch_execnz .LBB2_8 +; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX7-NEXT: s_mov_b32 s3, 0xf000 @@ -391,52 +416,78 @@ ; GFX8-LABEL: lds_ds_fadd: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb -; GFX8-NEXT: s_mov_b32 m0, -1 +; GFX8-NEXT: s_mov_b64 s[6:7], exec +; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_lshl_b32 s4, s3, 3 -; GFX8-NEXT: v_mov_b32_e32 v0, s4 -; GFX8-NEXT: ds_read_b32 v0, v0 offset:32 ; GFX8-NEXT: s_add_i32 s3, s3, 4 -; GFX8-NEXT: s_lshl_b32 s6, s3, 3 -; GFX8-NEXT: s_mov_b64 s[4:5], 0 -; GFX8-NEXT: .LBB2_1: ; %atomicrmw.start +; GFX8-NEXT: ; implicit-def: $vgpr1 +; GFX8-NEXT: s_mov_b32 m0, -1 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB2_4 +; GFX8-NEXT: ; %bb.1: +; GFX8-NEXT: s_lshl_b32 s8, s3, 3 +; GFX8-NEXT: v_mov_b32_e32 v1, s8 +; GFX8-NEXT: ds_read_b32 v1, v1 +; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s6 +; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 +; GFX8-NEXT: s_mov_b64 s[6:7], 0 +; GFX8-NEXT: .LBB2_2: ; %atomicrmw.start ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v1, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_add_f32_e32 v2, 0x42280000, v1 +; GFX8-NEXT: v_mov_b32_e32 v3, v1 +; GFX8-NEXT: v_mov_b32_e32 v1, s8 +; GFX8-NEXT: v_add_f32_e32 v4, v3, v2 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: ds_cmpst_rtn_b32 v0, v0, v1, v2 +; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 -; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_cbranch_execnz .LBB2_1 -; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX8-NEXT: s_cbranch_execnz .LBB2_2 +; GFX8-NEXT: ; %bb.3: ; %Flow17 +; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX8-NEXT: .LBB2_4: ; %Flow18 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: s_mov_b64 s[6:7], exec +; GFX8-NEXT: v_readfirstlane_b32 s8, v1 +; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB2_7 +; GFX8-NEXT: ; %bb.5: ; GFX8-NEXT: s_lshl_b32 s3, s3, 4 ; GFX8-NEXT: v_mov_b32_e32 v1, s3 -; GFX8-NEXT: ds_read_b32 v1, v1 -; GFX8-NEXT: s_mov_b64 s[4:5], 0 -; GFX8-NEXT: .LBB2_3: ; %atomicrmw.start2 +; GFX8-NEXT: ds_read_b32 v2, v1 +; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 +; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX8-NEXT: s_mov_b64 s[6:7], 0 +; GFX8-NEXT: .LBB2_6: ; %atomicrmw.start2 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_add_f32_e32 v2, 0x42280000, v1 -; GFX8-NEXT: v_mov_b32_e32 v3, s3 +; GFX8-NEXT: v_add_f32_e32 v3, v2, v1 +; GFX8-NEXT: v_mov_b32_e32 v4, s3 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v3, v1, v2 +; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1 -; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX8-NEXT: v_mov_b32_e32 v1, v2 -; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_cbranch_execnz .LBB2_3 -; GFX8-NEXT: ; %bb.4: ; %atomicrmw.end1 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2 +; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX8-NEXT: v_mov_b32_e32 v2, v3 +; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX8-NEXT: s_cbranch_execnz .LBB2_6 +; GFX8-NEXT: .LBB2_7: ; %Flow16 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: ds_read_b32 v1, v1 +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX8-NEXT: v_add_f32_e32 v0, s8, v0 ; GFX8-NEXT: s_mov_b64 s[4:5], 0 -; GFX8-NEXT: .LBB2_5: ; %atomicrmw.start8 +; GFX8-NEXT: .LBB2_8: ; %atomicrmw.start8 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v2, v1 @@ -448,8 +499,8 @@ ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_cbranch_execnz .LBB2_5 -; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end7 +; GFX8-NEXT: s_cbranch_execnz .LBB2_8 +; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX8-NEXT: s_mov_b32 s3, 0xf000 @@ -636,51 +687,76 @@ ; GFX7-LABEL: lds_ds_fadd_one_as: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb -; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_mov_b64 s[6:7], exec +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_lshl_b32 s4, s3, 3 -; GFX7-NEXT: s_add_i32 s4, s4, 32 -; GFX7-NEXT: v_mov_b32_e32 v0, s4 -; GFX7-NEXT: ds_read_b32 v0, v0 ; GFX7-NEXT: s_add_i32 s3, s3, 4 -; GFX7-NEXT: s_lshl_b32 s6, s3, 3 -; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start +; GFX7-NEXT: ; implicit-def: $vgpr1 +; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB3_4 +; GFX7-NEXT: ; %bb.1: +; GFX7-NEXT: s_lshl_b32 s8, s3, 3 +; GFX7-NEXT: v_mov_b32_e32 v1, s8 +; GFX7-NEXT: ds_read_b32 v1, v1 +; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s6 +; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 +; GFX7-NEXT: s_mov_b64 s[6:7], 0 +; GFX7-NEXT: .LBB3_2: ; %atomicrmw.start ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_mov_b32_e32 v1, v0 -; GFX7-NEXT: v_mov_b32_e32 v0, s6 -; GFX7-NEXT: v_add_f32_e32 v2, 0x42280000, v1 -; GFX7-NEXT: ds_cmpst_rtn_b32 v0, v0, v1, v2 +; GFX7-NEXT: v_mov_b32_e32 v3, v1 +; GFX7-NEXT: v_mov_b32_e32 v1, s8 +; GFX7-NEXT: v_add_f32_e32 v4, v3, v2 +; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 -; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB3_1 -; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX7-NEXT: s_cbranch_execnz .LBB3_2 +; GFX7-NEXT: ; %bb.3: ; %Flow15 +; GFX7-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX7-NEXT: .LBB3_4: ; %Flow16 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX7-NEXT: s_mov_b64 s[6:7], exec +; GFX7-NEXT: v_readfirstlane_b32 s8, v1 +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 +; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7-NEXT: s_cbranch_execz .LBB3_7 +; GFX7-NEXT: ; %bb.5: ; GFX7-NEXT: s_lshl_b32 s3, s3, 4 ; GFX7-NEXT: v_mov_b32_e32 v1, s3 -; GFX7-NEXT: ds_read_b32 v1, v1 -; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: .LBB3_3: ; %atomicrmw.start2 +; GFX7-NEXT: ds_read_b32 v2, v1 +; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 +; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX7-NEXT: s_mov_b64 s[6:7], 0 +; GFX7-NEXT: .LBB3_6: ; %atomicrmw.start2 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_add_f32_e32 v2, 0x42280000, v1 -; GFX7-NEXT: v_mov_b32_e32 v3, s3 -; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v1, v2 +; GFX7-NEXT: v_add_f32_e32 v3, v2, v1 +; GFX7-NEXT: v_mov_b32_e32 v4, s3 +; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1 -; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX7-NEXT: v_mov_b32_e32 v1, v2 -; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB3_3 -; GFX7-NEXT: ; %bb.4: ; %atomicrmw.end1 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2 +; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX7-NEXT: v_mov_b32_e32 v2, v3 +; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX7-NEXT: s_cbranch_execnz .LBB3_6 +; GFX7-NEXT: .LBB3_7: ; %Flow14 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX7-NEXT: v_mov_b32_e32 v1, s2 ; GFX7-NEXT: ds_read_b32 v1, v1 +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX7-NEXT: v_add_f32_e32 v0, s8, v0 ; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: .LBB3_5: ; %atomicrmw.start8 +; GFX7-NEXT: .LBB3_8: ; %atomicrmw.start8 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_mov_b32_e32 v2, v1 @@ -691,8 +767,8 @@ ; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 ; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB3_5 -; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end7 +; GFX7-NEXT: s_cbranch_execnz .LBB3_8 +; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX7-NEXT: s_mov_b32 s3, 0xf000 @@ -704,50 +780,76 @@ ; GFX8-LABEL: lds_ds_fadd_one_as: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb -; GFX8-NEXT: s_mov_b32 m0, -1 +; GFX8-NEXT: s_mov_b64 s[6:7], exec +; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_lshl_b32 s4, s3, 3 -; GFX8-NEXT: v_mov_b32_e32 v0, s4 -; GFX8-NEXT: ds_read_b32 v0, v0 offset:32 ; GFX8-NEXT: s_add_i32 s3, s3, 4 -; GFX8-NEXT: s_lshl_b32 s6, s3, 3 -; GFX8-NEXT: s_mov_b64 s[4:5], 0 -; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start +; GFX8-NEXT: ; implicit-def: $vgpr1 +; GFX8-NEXT: s_mov_b32 m0, -1 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB3_4 +; GFX8-NEXT: ; %bb.1: +; GFX8-NEXT: s_lshl_b32 s8, s3, 3 +; GFX8-NEXT: v_mov_b32_e32 v1, s8 +; GFX8-NEXT: ds_read_b32 v1, v1 +; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s6 +; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 +; GFX8-NEXT: s_mov_b64 s[6:7], 0 +; GFX8-NEXT: .LBB3_2: ; %atomicrmw.start ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v1, v0 -; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_add_f32_e32 v2, 0x42280000, v1 -; GFX8-NEXT: ds_cmpst_rtn_b32 v0, v0, v1, v2 +; GFX8-NEXT: v_mov_b32_e32 v3, v1 +; GFX8-NEXT: v_mov_b32_e32 v1, s8 +; GFX8-NEXT: v_add_f32_e32 v4, v3, v2 +; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 -; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_cbranch_execnz .LBB3_1 -; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX8-NEXT: s_cbranch_execnz .LBB3_2 +; GFX8-NEXT: ; %bb.3: ; %Flow17 +; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX8-NEXT: .LBB3_4: ; %Flow18 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: s_mov_b64 s[6:7], exec +; GFX8-NEXT: v_readfirstlane_b32 s8, v1 +; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 +; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX8-NEXT: s_cbranch_execz .LBB3_7 +; GFX8-NEXT: ; %bb.5: ; GFX8-NEXT: s_lshl_b32 s3, s3, 4 ; GFX8-NEXT: v_mov_b32_e32 v1, s3 -; GFX8-NEXT: ds_read_b32 v1, v1 -; GFX8-NEXT: s_mov_b64 s[4:5], 0 -; GFX8-NEXT: .LBB3_3: ; %atomicrmw.start2 +; GFX8-NEXT: ds_read_b32 v2, v1 +; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 +; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX8-NEXT: s_mov_b64 s[6:7], 0 +; GFX8-NEXT: .LBB3_6: ; %atomicrmw.start2 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_add_f32_e32 v2, 0x42280000, v1 -; GFX8-NEXT: v_mov_b32_e32 v3, s3 -; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v3, v1, v2 +; GFX8-NEXT: v_add_f32_e32 v3, v2, v1 +; GFX8-NEXT: v_mov_b32_e32 v4, s3 +; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1 -; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX8-NEXT: v_mov_b32_e32 v1, v2 -; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_cbranch_execnz .LBB3_3 -; GFX8-NEXT: ; %bb.4: ; %atomicrmw.end1 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2 +; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX8-NEXT: v_mov_b32_e32 v2, v3 +; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GFX8-NEXT: s_cbranch_execnz .LBB3_6 +; GFX8-NEXT: .LBB3_7: ; %Flow16 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX8-NEXT: v_mov_b32_e32 v1, s2 ; GFX8-NEXT: ds_read_b32 v1, v1 +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX8-NEXT: v_add_f32_e32 v0, s8, v0 ; GFX8-NEXT: s_mov_b64 s[4:5], 0 -; GFX8-NEXT: .LBB3_5: ; %atomicrmw.start8 +; GFX8-NEXT: .LBB3_8: ; %atomicrmw.start8 ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v2, v1 @@ -758,8 +860,8 @@ ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2 ; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_cbranch_execnz .LBB3_5 -; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end7 +; GFX8-NEXT: s_cbranch_execnz .LBB3_8 +; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX8-NEXT: s_mov_b32 s3, 0xf000