Index: lib/Target/AMDGPU/AMDGPUISelLowering.h =================================================================== --- lib/Target/AMDGPU/AMDGPUISelLowering.h +++ lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -541,7 +541,6 @@ BUFFER_ATOMIC_CMPSWAP, BUFFER_ATOMIC_FADD, BUFFER_ATOMIC_PK_FADD, - ATOMIC_FADD, ATOMIC_PK_FADD, LAST_AMDGPU_ISD_NUMBER Index: lib/Target/AMDGPU/AMDGPUISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -4393,7 +4393,6 @@ NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP) NODE_NAME_CASE(BUFFER_ATOMIC_FADD) NODE_NAME_CASE(BUFFER_ATOMIC_PK_FADD) - NODE_NAME_CASE(ATOMIC_FADD) NODE_NAME_CASE(ATOMIC_PK_FADD) case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break; Index: lib/Target/AMDGPU/FLATInstructions.td =================================================================== --- lib/Target/AMDGPU/FLATInstructions.td +++ lib/Target/AMDGPU/FLATInstructions.td @@ -364,11 +364,12 @@ string opName, RegisterClass vdst_rc, ValueType vt, - SDPatternOperator atomic = null_frag, + SDPatternOperator atomic_rtn = null_frag, + SDPatternOperator atomic_no_rtn = null_frag, ValueType data_vt = vt, RegisterClass data_rc = vdst_rc> : - FLAT_Global_Atomic_Pseudo_NO_RTN, - FLAT_Global_Atomic_Pseudo_RTN; + FLAT_Global_Atomic_Pseudo_NO_RTN, + FLAT_Global_Atomic_Pseudo_RTN; class flat_binary_atomic_op : PatFrag< (ops node:$ptr, node:$value), @@ -556,11 +557,12 @@ let is_flat_global = 1 in { defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", - VGPR_32, i32, AMDGPUatomic_cmp_swap_global, + VGPR_32, i32, AMDGPUatomic_cmp_swap_global, null_frag, v2i32, VReg_64>; defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", VReg_64, i64, AMDGPUatomic_cmp_swap_global, + null_frag, v2i64, VReg_128>; defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", Index: lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/SIISelLowering.cpp +++ lib/Target/AMDGPU/SIISelLowering.cpp @@ -7015,13 +7015,16 @@ EVT VT = Op.getOperand(3).getValueType(); auto *M = cast(Op); - unsigned Opcode = VT.isVector() ? AMDGPUISD::ATOMIC_PK_FADD - : AMDGPUISD::ATOMIC_FADD; + if (VT.isVector()) { + return DAG.getMemIntrinsicNode( + AMDGPUISD::ATOMIC_PK_FADD, DL, Op->getVTList(), Ops, VT, + M->getMemOperand()); + } - return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT, - M->getMemOperand()); + return DAG.getAtomic(ISD::ATOMIC_LOAD_FADD, DL, VT, + DAG.getVTList(VT, MVT::Other), Ops, + M->getMemOperand()).getValue(1); } - case Intrinsic::amdgcn_end_cf: return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other, Op->getOperand(2), Chain), 0); @@ -10862,6 +10865,12 @@ // TODO: Do have these for flat. Older targets also had them for buffers. unsigned AS = RMW->getPointerAddressSpace(); + + if (AS == AMDGPUAS::GLOBAL_ADDRESS && Subtarget->hasAtomicFaddInsts()) { + return RMW->use_empty() ? AtomicExpansionKind::None : + AtomicExpansionKind::CmpXChg; + } + return (AS == AMDGPUAS::LOCAL_ADDRESS && Subtarget->hasLDSFPAtomics()) ? AtomicExpansionKind::None : AtomicExpansionKind::CmpXChg; } Index: lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.td +++ lib/Target/AMDGPU/SIInstrInfo.td @@ -225,7 +225,6 @@ [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore] >; -def SIglobal_atomic_fadd : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_FADD", f32>; def SIglobal_atomic_pk_fadd : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_PK_FADD", v2f16>; def SIpc_add_rel_offset : SDNode<"AMDGPUISD::PC_ADD_REL_OFFSET", @@ -322,7 +321,7 @@ def atomic_fadd_global_noret : PatFrag< (ops node:$ptr, node:$value), - (SIglobal_atomic_fadd node:$ptr, node:$value)> { + (atomic_load_fadd node:$ptr, node:$value)> { // FIXME: Move this let MemoryVT = f32; let IsAtomic = 1; Index: test/CodeGen/AMDGPU/global-atomics-fp.ll =================================================================== --- /dev/null +++ test/CodeGen/AMDGPU/global-atomics-fp.ll @@ -0,0 +1,29 @@ +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s +; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s + +; GCN-LABEL: {{^}}global_atomic_fadd_ret_f32: +; GCN: [[LOOP:BB[0-9]+_[0-9]+]] +; GCN: v_add_f32_e32 +; GCN: global_atomic_cmpswap +; GCN: s_andn2_b64 exec, exec, +; GCN-NEXT: s_cbranch_execnz [[LOOP]] +define amdgpu_kernel void @global_atomic_fadd_ret_f32(float addrspace(1)* %ptr) { + %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 seq_cst + store float %result, float addrspace(1)* undef + ret void +} + +; GCN-LABEL: {{^}}global_atomic_fadd_noret_f32: +; GFX900: [[LOOP:BB[0-9]+_[0-9]+]] +; GFX900: v_add_f32_e32 +; GFX900: global_atomic_cmpswap +; GFX900: s_andn2_b64 exec, exec, +; GFX900-NEXT: s_cbranch_execnz [[LOOP]] + +; GFX908-NOT: v_add_f32 +; GFX908: global_atomic_add_f32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off +; GFX908-NOT: s_cbranch_execnz +define amdgpu_kernel void @global_atomic_fadd_noret_f32(float addrspace(1)* %ptr) { + %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 seq_cst + ret void +} Index: test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll =================================================================== --- test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll +++ test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=CI %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GFX9 %s +; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -atomic-expand %s | FileCheck -check-prefix=GFX908 %s define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) { ; CI-LABEL: @test_atomicrmw_fadd_f32_flat( @@ -36,6 +37,23 @@ ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9: atomicrmw.end: ; GFX9-NEXT: ret float [[TMP6]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_flat( +; GFX908-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32* +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 +; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP6]] ; %res = atomicrmw fadd float* %ptr, float %value seq_cst ret float %res @@ -75,11 +93,71 @@ ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9: atomicrmw.end: ; GFX9-NEXT: ret float [[TMP6]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_global( +; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 +; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP6]] ; %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst ret float %res } +define void @test_atomicrmw_fadd_f32_global_no_use(float addrspace(1)* %ptr, float %value) { +; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use( +; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 +; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret void +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use( +; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 +; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret void +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(1)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst +; GFX908-NEXT: ret void +; + %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst + ret void +} + define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %value) { ; CI-LABEL: @test_atomicrmw_fadd_f32_local( ; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4 @@ -101,6 +179,10 @@ ; GFX9-LABEL: @test_atomicrmw_fadd_f32_local( ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst ; GFX9-NEXT: ret float [[RES]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_local( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst +; GFX908-NEXT: ret float [[RES]] ; %res = atomicrmw fadd float addrspace(3)* %ptr, float %value seq_cst ret float %res @@ -114,6 +196,10 @@ ; GFX9-LABEL: @test_atomicrmw_fadd_f16_flat( ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX9-NEXT: ret half [[RES]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f16_flat( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst +; GFX908-NEXT: ret half [[RES]] ; %res = atomicrmw fadd half* %ptr, half %value seq_cst ret half %res @@ -127,6 +213,10 @@ ; GFX9-LABEL: @test_atomicrmw_fadd_f16_global( ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX9-NEXT: ret half [[RES]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f16_global( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst +; GFX908-NEXT: ret half [[RES]] ; %res = atomicrmw fadd half addrspace(1)* %ptr, half %value seq_cst ret half %res @@ -140,6 +230,10 @@ ; GFX9-LABEL: @test_atomicrmw_fadd_f16_local( ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX9-NEXT: ret half [[RES]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f16_local( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst +; GFX908-NEXT: ret half [[RES]] ; %res = atomicrmw fadd half addrspace(3)* %ptr, half %value seq_cst ret half %res @@ -179,6 +273,23 @@ ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9: atomicrmw.end: ; GFX9-NEXT: ret double [[TMP6]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat( +; GFX908-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64* +; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 +; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 +; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 +; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret double [[TMP6]] ; %res = atomicrmw fadd double* %ptr, double %value seq_cst ret double %res @@ -218,6 +329,23 @@ ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9: atomicrmw.end: ; GFX9-NEXT: ret double [[TMP6]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f64_global( +; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)* +; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 +; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 +; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 +; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret double [[TMP6]] ; %res = atomicrmw fadd double addrspace(1)* %ptr, double %value seq_cst ret double %res @@ -257,6 +385,23 @@ ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9: atomicrmw.end: ; GFX9-NEXT: ret double [[TMP6]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f64_local( +; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)* +; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 +; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 +; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 +; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret double [[TMP6]] ; %res = atomicrmw fadd double addrspace(3)* %ptr, double %value seq_cst ret double %res