Index: lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -138,6 +138,10 @@ SDValue &ImmOffset) const; bool SelectMUBUFIntrinsicVOffset(SDValue Offset, SDValue &SOffset, SDValue &ImmOffset, SDValue &VOffset) const; + + bool SelectFlat(SDValue Addr, SDValue &VAddr, + SDValue &SLC, SDValue &TFE) const; + bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset, bool &Imm) const; bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, @@ -1236,6 +1240,15 @@ return true; } +bool AMDGPUDAGToDAGISel::SelectFlat(SDValue Addr, + SDValue &VAddr, + SDValue &SLC, + SDValue &TFE) const { + VAddr = Addr; + TFE = SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1); + return true; +} + /// /// \param EncodedOffset This is the immediate value that will be encoded /// directly into the instruction. On SI/CI the \p EncodedOffset @@ -1500,6 +1513,10 @@ void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { MemSDNode *Mem = cast(N); unsigned AS = Mem->getAddressSpace(); + if (AS == AMDGPUAS::FLAT_ADDRESS) { + SelectCode(N); + return; + } MVT VT = N->getSimpleValueType(0); bool Is32 = (VT == MVT::i32); Index: lib/Target/AMDGPU/AMDGPUInstructions.td =================================================================== --- lib/Target/AMDGPU/AMDGPUInstructions.td +++ lib/Target/AMDGPU/AMDGPUInstructions.td @@ -394,6 +394,12 @@ [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}] >; +class flat_binary_atomic_op : PatFrag< + (ops node:$ptr, node:$value), + (atomic_op node:$ptr, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS;}] +>; + def atomic_swap_global : global_binary_atomic_op; def atomic_add_global : global_binary_atomic_op; def atomic_and_global : global_binary_atomic_op; @@ -412,6 +418,19 @@ [{ return SDValue(N, 0).use_empty(); }] >; +def atomic_swap_flat : flat_binary_atomic_op; +def atomic_add_flat : flat_binary_atomic_op; +def atomic_and_flat : flat_binary_atomic_op; +def atomic_max_flat : flat_binary_atomic_op; +def atomic_min_flat : flat_binary_atomic_op; +def atomic_or_flat : flat_binary_atomic_op; +def atomic_sub_flat : flat_binary_atomic_op; +def atomic_umax_flat : flat_binary_atomic_op; +def atomic_umin_flat : flat_binary_atomic_op; +def atomic_xor_flat : flat_binary_atomic_op; + +def atomic_cmp_swap_flat : flat_binary_atomic_op; + //===----------------------------------------------------------------------===// // Misc Pattern Fragments //===----------------------------------------------------------------------===// Index: lib/Target/AMDGPU/CIInstructions.td =================================================================== --- lib/Target/AMDGPU/CIInstructions.td +++ lib/Target/AMDGPU/CIInstructions.td @@ -153,82 +153,84 @@ flat<0x1f, 0x1e>, "flat_store_dwordx3", VReg_96 >; defm FLAT_ATOMIC_SWAP : FLAT_ATOMIC < - flat<0x30, 0x40>, "flat_atomic_swap", VGPR_32 + flat<0x30, 0x40>, "flat_atomic_swap", VGPR_32, i32, atomic_swap_flat >; defm FLAT_ATOMIC_CMPSWAP : FLAT_ATOMIC < - flat<0x31, 0x41>, "flat_atomic_cmpswap", VGPR_32, VReg_64 + flat<0x31, 0x41>, "flat_atomic_cmpswap", VGPR_32, i32, + atomic_cmp_swap_flat, v2i32, VReg_64 >; defm FLAT_ATOMIC_ADD : FLAT_ATOMIC < - flat<0x32, 0x42>, "flat_atomic_add", VGPR_32 + flat<0x32, 0x42>, "flat_atomic_add", VGPR_32, i32, atomic_add_flat >; defm FLAT_ATOMIC_SUB : FLAT_ATOMIC < - flat<0x33, 0x43>, "flat_atomic_sub", VGPR_32 + flat<0x33, 0x43>, "flat_atomic_sub", VGPR_32, i32, atomic_sub_flat >; defm FLAT_ATOMIC_SMIN : FLAT_ATOMIC < - flat<0x35, 0x44>, "flat_atomic_smin", VGPR_32 + flat<0x35, 0x44>, "flat_atomic_smin", VGPR_32, i32, atomic_min_flat >; defm FLAT_ATOMIC_UMIN : FLAT_ATOMIC < - flat<0x36, 0x45>, "flat_atomic_umin", VGPR_32 + flat<0x36, 0x45>, "flat_atomic_umin", VGPR_32, i32, atomic_umin_flat >; defm FLAT_ATOMIC_SMAX : FLAT_ATOMIC < - flat<0x37, 0x46>, "flat_atomic_smax", VGPR_32 + flat<0x37, 0x46>, "flat_atomic_smax", VGPR_32, i32, atomic_max_flat >; defm FLAT_ATOMIC_UMAX : FLAT_ATOMIC < - flat<0x38, 0x47>, "flat_atomic_umax", VGPR_32 + flat<0x38, 0x47>, "flat_atomic_umax", VGPR_32, i32, atomic_umax_flat >; defm FLAT_ATOMIC_AND : FLAT_ATOMIC < - flat<0x39, 0x48>, "flat_atomic_and", VGPR_32 + flat<0x39, 0x48>, "flat_atomic_and", VGPR_32, i32, atomic_and_flat >; defm FLAT_ATOMIC_OR : FLAT_ATOMIC < - flat<0x3a, 0x49>, "flat_atomic_or", VGPR_32 + flat<0x3a, 0x49>, "flat_atomic_or", VGPR_32, i32, atomic_or_flat >; defm FLAT_ATOMIC_XOR : FLAT_ATOMIC < - flat<0x3b, 0x4a>, "flat_atomic_xor", VGPR_32 + flat<0x3b, 0x4a>, "flat_atomic_xor", VGPR_32, i32, atomic_xor_flat >; defm FLAT_ATOMIC_INC : FLAT_ATOMIC < - flat<0x3c, 0x4b>, "flat_atomic_inc", VGPR_32 + flat<0x3c, 0x4b>, "flat_atomic_inc", VGPR_32, i32, atomic_inc_flat >; defm FLAT_ATOMIC_DEC : FLAT_ATOMIC < - flat<0x3d, 0x4c>, "flat_atomic_dec", VGPR_32 + flat<0x3d, 0x4c>, "flat_atomic_dec", VGPR_32, i32, atomic_dec_flat >; defm FLAT_ATOMIC_SWAP_X2 : FLAT_ATOMIC < - flat<0x50, 0x60>, "flat_atomic_swap_x2", VReg_64 + flat<0x50, 0x60>, "flat_atomic_swap_x2", VReg_64, i64, atomic_swap_flat >; defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_ATOMIC < - flat<0x51, 0x61>, "flat_atomic_cmpswap_x2", VReg_64, VReg_128 + flat<0x51, 0x61>, "flat_atomic_cmpswap_x2", VReg_64, i64, + atomic_cmp_swap_flat, v2i64, VReg_128 >; defm FLAT_ATOMIC_ADD_X2 : FLAT_ATOMIC < - flat<0x52, 0x62>, "flat_atomic_add_x2", VReg_64 + flat<0x52, 0x62>, "flat_atomic_add_x2", VReg_64, i64, atomic_add_flat >; defm FLAT_ATOMIC_SUB_X2 : FLAT_ATOMIC < - flat<0x53, 0x63>, "flat_atomic_sub_x2", VReg_64 + flat<0x53, 0x63>, "flat_atomic_sub_x2", VReg_64, i64, atomic_sub_flat >; defm FLAT_ATOMIC_SMIN_X2 : FLAT_ATOMIC < - flat<0x55, 0x64>, "flat_atomic_smin_x2", VReg_64 + flat<0x55, 0x64>, "flat_atomic_smin_x2", VReg_64, i64, atomic_min_flat >; defm FLAT_ATOMIC_UMIN_X2 : FLAT_ATOMIC < - flat<0x56, 0x65>, "flat_atomic_umin_x2", VReg_64 + flat<0x56, 0x65>, "flat_atomic_umin_x2", VReg_64, i64, atomic_umin_flat >; defm FLAT_ATOMIC_SMAX_X2 : FLAT_ATOMIC < - flat<0x57, 0x66>, "flat_atomic_smax_x2", VReg_64 + flat<0x57, 0x66>, "flat_atomic_smax_x2", VReg_64, i64, atomic_max_flat >; defm FLAT_ATOMIC_UMAX_X2 : FLAT_ATOMIC < - flat<0x58, 0x67>, "flat_atomic_umax_x2", VReg_64 + flat<0x58, 0x67>, "flat_atomic_umax_x2", VReg_64, i64, atomic_umax_flat >; defm FLAT_ATOMIC_AND_X2 : FLAT_ATOMIC < - flat<0x59, 0x68>, "flat_atomic_and_x2", VReg_64 + flat<0x59, 0x68>, "flat_atomic_and_x2", VReg_64, i64, atomic_and_flat >; defm FLAT_ATOMIC_OR_X2 : FLAT_ATOMIC < - flat<0x5a, 0x69>, "flat_atomic_or_x2", VReg_64 + flat<0x5a, 0x69>, "flat_atomic_or_x2", VReg_64, i64, atomic_or_flat >; defm FLAT_ATOMIC_XOR_X2 : FLAT_ATOMIC < - flat<0x5b, 0x6a>, "flat_atomic_xor_x2", VReg_64 + flat<0x5b, 0x6a>, "flat_atomic_xor_x2", VReg_64, i64, atomic_xor_flat >; defm FLAT_ATOMIC_INC_X2 : FLAT_ATOMIC < - flat<0x5c, 0x6b>, "flat_atomic_inc_x2", VReg_64 + flat<0x5c, 0x6b>, "flat_atomic_inc_x2", VReg_64, i64, atomic_inc_flat >; defm FLAT_ATOMIC_DEC_X2 : FLAT_ATOMIC < - flat<0x5d, 0x6c>, "flat_atomic_dec_x2", VReg_64 + flat<0x5d, 0x6c>, "flat_atomic_dec_x2", VReg_64, i64, atomic_dec_flat >; } // End SubtargetPredicate = isCIVI @@ -238,22 +240,24 @@ let SubtargetPredicate = isCI, VIAssemblerPredicate = DisableInst, DisableVIDecoder = 1 in { defm FLAT_ATOMIC_FCMPSWAP : FLAT_ATOMIC < - flat<0x3e>, "flat_atomic_fcmpswap", VGPR_32, VReg_64 + flat<0x3e>, "flat_atomic_fcmpswap", VGPR_32, f32, + null_frag, v2f32, VReg_64 >; defm FLAT_ATOMIC_FMIN : FLAT_ATOMIC < - flat<0x3f>, "flat_atomic_fmin", VGPR_32 + flat<0x3f>, "flat_atomic_fmin", VGPR_32, f32 >; defm FLAT_ATOMIC_FMAX : FLAT_ATOMIC < - flat<0x40>, "flat_atomic_fmax", VGPR_32 + flat<0x40>, "flat_atomic_fmax", VGPR_32, f32 >; defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_ATOMIC < - flat<0x5e>, "flat_atomic_fcmpswap_x2", VReg_64, VReg_128 + flat<0x5e>, "flat_atomic_fcmpswap_x2", VReg_64, f64, + null_frag, v2f64, VReg_128 >; defm FLAT_ATOMIC_FMIN_X2 : FLAT_ATOMIC < - flat<0x5f>, "flat_atomic_fmin_x2", VReg_64 + flat<0x5f>, "flat_atomic_fmin_x2", VReg_64, f64 >; defm FLAT_ATOMIC_FMAX_X2 : FLAT_ATOMIC < - flat<0x60>, "flat_atomic_fmax_x2", VReg_64 + flat<0x60>, "flat_atomic_fmax_x2", VReg_64, f64 >; } // End SubtargetPredicate = isCI, VIAssemblerPredicate = DisableInst, DisableVIDecoder = 1 @@ -293,7 +297,8 @@ >; class FlatStoreAtomicPat : Pat < - // atomic store follows aotmic binop convenction so the address comes first + // atomic store follows atomic binop convention so the address comes + // first. (node i64:$addr, vt:$data), (inst $addr, $data, 1, 0, 0) >; Index: lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.td +++ lib/Target/AMDGPU/SIInstrInfo.td @@ -190,6 +190,9 @@ def atomic_inc_global : global_binary_atomic_op; def atomic_dec_global : global_binary_atomic_op; +def atomic_inc_flat : flat_binary_atomic_op; +def atomic_dec_flat : flat_binary_atomic_op; + //===----------------------------------------------------------------------===// // SDNodes and PatFrag for local loads and stores to enable s_mov_b32 m0, -1 // to be glued to the memory instructions. @@ -567,6 +570,7 @@ def MUBUFAddr32 : ComplexPattern; def MUBUFAddr64 : ComplexPattern; def MUBUFAddr64Atomic : ComplexPattern; +def FLATAtomic : ComplexPattern; def MUBUFScratch : ComplexPattern; def MUBUFOffset : ComplexPattern; def MUBUFOffsetNoGLC : ComplexPattern; @@ -3142,32 +3146,36 @@ } multiclass FLAT_ATOMIC { let mayLoad = 1, mayStore = 1, glc = 0, vdst = 0 in { - def "" : FLAT_Pseudo , AtomicNoRet ; - def _ci : FLAT_Real_ci ; - def _vi : FLAT_Real_vi ; } let glc = 1, hasPostISelHook = 1 in { - defm _RTN : FLAT_AtomicRet_m ; + defm _RTN : FLAT_AtomicRet_m < + op, (outs vdst_rc:$vdst), + (ins VReg_64:$addr, data_rc:$data, slc:$slc, tfe:$tfe), + asm_name#" $vdst, $addr, $data glc$slc$tfe", + [(set vt:$vdst, + (atomic (FLATAtomic i64:$addr, i1:$slc, i1:$tfe), data_vt:$data))] + >; } } Index: test/CodeGen/AMDGPU/flat_atomics.ll =================================================================== --- /dev/null +++ test/CodeGen/AMDGPU/flat_atomics.ll @@ -0,0 +1,968 @@ +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}atomic_add_i32_offset: +; GCN: flat_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_add_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile add i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i32_ret_offset: +; GCN: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile add i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i32_addr64_offset: +; GCN: flat_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_add_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile add i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i32_ret_addr64_offset: +; GCN: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile add i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i32: +; GCN: flat_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_add_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile add i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i32_ret: +; GCN: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile add i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i32_addr64: +; GCN: flat_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_add_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile add i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i32_ret_addr64: +; GCN: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile add i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32_offset: +; GCN: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_and_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile and i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32_ret_offset: +; GCN: flat_atomic_and [[RET:v[0-9]]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile and i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32_addr64_offset: +; GCN: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_and_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile and i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32_ret_addr64_offset: +; GCN: flat_atomic_and [[RET:v[0-9]]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile and i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32: +; GCN: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_and_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile and i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32_ret: +; GCN: flat_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile and i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32_addr64: +; GCN: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_and_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile and i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i32_ret_addr64: +; GCN: flat_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile and i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32_offset: +; GCN: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_sub_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile sub i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32_ret_offset: +; GCN: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile sub i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32_addr64_offset: +; GCN: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_sub_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile sub i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32_ret_addr64_offset: +; GCN: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile sub i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32: +; GCN: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_sub_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile sub i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32_ret: +; GCN: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile sub i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32_addr64: +; GCN: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_sub_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile sub i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i32_ret_addr64: +; GCN: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile sub i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32_offset: +; GCN: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_max_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile max i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32_ret_offset: +; GCN: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile max i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32_addr64_offset: +; GCN: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_max_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile max i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32_ret_addr64_offset: +; GCN: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile max i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32: +; GCN: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_max_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile max i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32_ret: +; GCN: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile max i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32_addr64: +; GCN: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_max_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile max i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i32_ret_addr64: +; GCN: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile max i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32_offset: +; GCN: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umax_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile umax i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32_ret_offset: +; GCN: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile umax i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32_addr64_offset: +; GCN: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umax_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile umax i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32_ret_addr64_offset: +; GCN: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile umax i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32: +; GCN: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umax_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile umax i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32_ret: +; GCN: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile umax i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32_addr64: +; GCN: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umax_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile umax i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i32_ret_addr64: +; GCN: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile umax i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32_offset: +; GCN: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_min_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile min i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32_ret_offset: +; GCN: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile min i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32_addr64_offset: +; GCN: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_min_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile min i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32_ret_addr64_offset: +; GCN: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile min i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32: +; GCN: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_min_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile min i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32_ret: +; GCN: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile min i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32_addr64: +; GCN: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_min_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile min i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i32_ret_addr64: +; GCN: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile min i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32_offset: +; GCN: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umin_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile umin i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32_ret_offset: +; GCN: flat_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umin_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile umin i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32_addr64_offset: +; GCN: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umin_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile umin i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32_ret_addr64_offset: +; GCN: flat_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umin_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile umin i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32: +; GCN: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umin_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile umin i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32_ret: +; GCN: flat_atomic_umin v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umin_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile umin i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32_addr64: +; GCN: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_umin_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile umin i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i32_ret_addr64: +; GCN: flat_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]{{$}} + define void @atomic_umin_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile umin i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32_offset: +; GCN: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_or_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile or i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32_ret_offset: +; GCN: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile or i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32_addr64_offset: +; GCN: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_or_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile or i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32_ret_addr64_offset: +; GCN: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile or i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32: +; GCN: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_or_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile or i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32_ret: +; GCN: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile or i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32_addr64: +; GCN: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_or_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile or i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i32_ret_addr64: +; GCN: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile or i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32_offset: +; GCN: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_xchg_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile xchg i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32_ret_offset: +; GCN: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile xchg i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32_addr64_offset: +; GCN: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_xchg_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile xchg i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32_ret_addr64_offset: +; GCN: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile xchg i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32: +; GCN: flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_xchg_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32_ret: +; GCN: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32_addr64: +; GCN: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_xchg_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile xchg i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i32_ret_addr64: +; GCN: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile xchg i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; CMP_SWAP + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32_offset: +; GCN: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i32_offset(i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret_offset: +; GCN: flat_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RET]] +define void @atomic_cmpxchg_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i32 %old) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst seq_cst + %flag = extractvalue { i32, i1 } %val, 0 + store i32 %flag, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32_addr64_offset: +; GCN: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index, i32 %old) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret_addr64_offset: +; GCN: flat_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RET]] +define void @atomic_cmpxchg_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index, i32 %old) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst seq_cst + %flag = extractvalue { i32, i1 } %val, 0 + store i32 %flag, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32: +; GCN: flat_atomic_cmpswap v[{{[0-9]+}}:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i32(i32 addrspace(4)* %out, i32 %in, i32 %old) { +entry: + %val = cmpxchg volatile i32 addrspace(4)* %out, i32 %old, i32 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret: +; GCN: flat_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}] glc +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RET]] +define void @atomic_cmpxchg_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i32 %old) { +entry: + %val = cmpxchg volatile i32 addrspace(4)* %out, i32 %old, i32 %in seq_cst seq_cst + %flag = extractvalue { i32, i1 } %val, 0 + store i32 %flag, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32_addr64: +; GCN: flat_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index, i32 %old) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = cmpxchg volatile i32 addrspace(4)* %ptr, i32 %old, i32 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret_addr64: +; GCN: flat_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RET]] +define void @atomic_cmpxchg_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index, i32 %old) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = cmpxchg volatile i32 addrspace(4)* %ptr, i32 %old, i32 %in seq_cst seq_cst + %flag = extractvalue { i32, i1 } %val, 0 + store i32 %flag, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32_offset: +; GCN: flat_atomic_xor v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_xor_i32_offset(i32 addrspace(4)* %out, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile xor i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32_ret_offset: +; GCN: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i32_ret_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + %val = atomicrmw volatile xor i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32_addr64_offset: +; GCN: flat_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_xor_i32_addr64_offset(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile xor i32 addrspace(4)* %gep, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32_ret_addr64_offset: +; GCN: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i32_ret_addr64_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = atomicrmw volatile xor i32 addrspace(4)* %gep, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32: +; GCN: flat_atomic_xor v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}} +define void @atomic_xor_i32(i32 addrspace(4)* %out, i32 %in) { +entry: + %val = atomicrmw volatile xor i32 addrspace(4)* %out, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32_ret: +; GCN: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i32_ret(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in) { +entry: + %val = atomicrmw volatile xor i32 addrspace(4)* %out, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32_addr64: +; GCN: flat_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}} +define void @atomic_xor_i32_addr64(i32 addrspace(4)* %out, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile xor i32 addrspace(4)* %ptr, i32 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i32_ret_addr64: +; GCN: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i32_ret_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %out2, i32 %in, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %val = atomicrmw volatile xor i32 addrspace(4)* %ptr, i32 %in seq_cst + store i32 %val, i32 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i32_offset: +; GCN: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i32_offset(i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %in, i32 4 + %val = load atomic i32, i32 addrspace(4)* %gep seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i32: +; GCN: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i32(i32 addrspace(4)* %in, i32 addrspace(4)* %out) { +entry: + %val = load atomic i32, i32 addrspace(4)* %in seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i32_addr64_offset: +; GCN: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i32_addr64_offset(i32 addrspace(4)* %in, i32 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %in, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %val = load atomic i32, i32 addrspace(4)* %gep seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i32_addr64: +; GCN: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i32_addr64(i32 addrspace(4)* %in, i32 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %in, i64 %index + %val = load atomic i32, i32 addrspace(4)* %ptr seq_cst, align 4 + store i32 %val, i32 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i32_offset: +; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +define void @atomic_store_i32_offset(i32 %in, i32 addrspace(4)* %out) { +entry: + %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4 + store atomic i32 %in, i32 addrspace(4)* %gep seq_cst, align 4 + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i32: +; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +define void @atomic_store_i32(i32 %in, i32 addrspace(4)* %out) { +entry: + store atomic i32 %in, i32 addrspace(4)* %out seq_cst, align 4 + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i32_addr64_offset: +; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +define void @atomic_store_i32_addr64_offset(i32 %in, i32 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + store atomic i32 %in, i32 addrspace(4)* %gep seq_cst, align 4 + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i32_addr64: +; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}} +define void @atomic_store_i32_addr64(i32 %in, i32 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index + store atomic i32 %in, i32 addrspace(4)* %ptr seq_cst, align 4 + ret void +} Index: test/CodeGen/AMDGPU/flat_atomics_i64.ll =================================================================== --- /dev/null +++ test/CodeGen/AMDGPU/flat_atomics_i64.ll @@ -0,0 +1,975 @@ +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}atomic_add_i64_offset: +; GCN: flat_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}{{$}} +define void @atomic_add_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i64_ret_offset: +; GCN: flat_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i64_addr64_offset: +; GCN: flat_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}{{$}} +define void @atomic_add_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i64_ret_addr64_offset: +; GCN: flat_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i64: +; GCN: flat_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_add_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i64_ret: +; GCN: flat_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i64_addr64: +; GCN: flat_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_add_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_add_i64_ret_addr64: +; GCN: flat_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_add_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile add i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64_offset: +; GCN: flat_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_and_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64_ret_offset: +; GCN: flat_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64_addr64_offset: +; GCN: flat_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_and_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64_ret_addr64_offset: +; GCN: flat_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64: +; GCN: flat_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_and_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64_ret: +; GCN: flat_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64_addr64: +; GCN: flat_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_and_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_and_i64_ret_addr64: +; GCN: flat_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_and_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile and i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64_offset: +; GCN: flat_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_sub_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64_ret_offset: +; GCN: flat_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64_addr64_offset: +; GCN: flat_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_sub_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64_ret_addr64_offset: +; GCN: flat_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64: +; GCN: flat_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_sub_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64_ret: +; GCN: flat_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64_addr64: +; GCN: flat_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_sub_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_sub_i64_ret_addr64: +; GCN: flat_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_sub_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile sub i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64_offset: +; GCN: flat_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_max_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64_ret_offset: +; GCN: flat_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64_addr64_offset: +; GCN: flat_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_max_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64_ret_addr64_offset: +; GCN: flat_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64: +; GCN: flat_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_max_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64_ret: +; GCN: flat_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64_addr64: +; GCN: flat_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_max_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_max_i64_ret_addr64: +; GCN: flat_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_max_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile max i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64_offset: +; GCN: flat_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umax_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64_ret_offset: +; GCN: flat_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64_addr64_offset: +; GCN: flat_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umax_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64_ret_addr64_offset: +; GCN: flat_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64: +; GCN: flat_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umax_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64_ret: +; GCN: flat_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64_addr64: +; GCN: flat_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umax_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umax_i64_ret_addr64: +; GCN: flat_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umax_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile umax i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64_offset: +; GCN: flat_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_min_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64_ret_offset: +; GCN: flat_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64_addr64_offset: +; GCN: flat_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_min_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64_ret_addr64_offset: +; GCN: flat_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64: +; GCN: flat_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_min_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64_ret: +; GCN: flat_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64_addr64: +; GCN: flat_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_min_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_min_i64_ret_addr64: +; GCN: flat_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_min_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile min i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64_offset: +; GCN: flat_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umin_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64_ret_offset: +; GCN: flat_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umin_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64_addr64_offset: +; GCN: flat_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umin_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64_ret_addr64_offset: +; GCN: flat_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umin_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64: +; GCN: flat_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umin_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64_ret: +; GCN: flat_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umin_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64_addr64: +; GCN: flat_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_umin_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_umin_i64_ret_addr64: +; GCN: flat_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_umin_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile umin i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64_offset: +; GCN: flat_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_or_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64_ret_offset: +; GCN: flat_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64_addr64_offset: +; GCN: flat_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_or_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64_ret_addr64_offset: +; GCN: flat_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64: +; GCN: flat_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_or_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64_ret: +; GCN: flat_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64_addr64: +; GCN: flat_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_or_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_or_i64_ret_addr64: +; GCN: flat_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_or_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile or i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64_offset: +; GCN: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xchg_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64_ret_offset: +; GCN: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64_addr64_offset: +; GCN: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xchg_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64_ret_addr64_offset: +; GCN: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64: +; GCN: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xchg_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64_ret: +; GCN: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64_addr64: +; GCN: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xchg_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xchg_i64_ret_addr64: +; GCN: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xchg_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile xchg i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64_offset: +; GCN: flat_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xor_i64_offset(i64 addrspace(4)* %out, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64_ret_offset: +; GCN: flat_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64_addr64_offset: +; GCN: flat_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xor_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %gep, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64_ret_addr64_offset: +; GCN: flat_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %gep, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64: +; GCN: flat_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xor_i64(i64 addrspace(4)* %out, i64 %in) { +entry: + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %out, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64_ret: +; GCN: flat_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in) { +entry: + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %out, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64_addr64: +; GCN: flat_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}} +define void @atomic_xor_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %ptr, i64 %in seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_xor_i64_ret_addr64: +; GCN: flat_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_xor_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %tmp0 = atomicrmw volatile xor i64 addrspace(4)* %ptr, i64 %in seq_cst + store i64 %tmp0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i64_offset: +; GCN: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i64_offset(i64 addrspace(4)* %in, i64 addrspace(4)* %out) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %in, i64 4 + %val = load atomic i64, i64 addrspace(4)* %gep seq_cst, align 8 + store i64 %val, i64 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i64: +; GCN: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i64(i64 addrspace(4)* %in, i64 addrspace(4)* %out) { +entry: + %val = load atomic i64, i64 addrspace(4)* %in seq_cst, align 8 + store i64 %val, i64 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i64_addr64_offset: +; GCN: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i64_addr64_offset(i64 addrspace(4)* %in, i64 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %in, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %val = load atomic i64, i64 addrspace(4)* %gep seq_cst, align 8 + store i64 %val, i64 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_load_i64_addr64: +; GCN: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]] +define void @atomic_load_i64_addr64(i64 addrspace(4)* %in, i64 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %in, i64 %index + %val = load atomic i64, i64 addrspace(4)* %ptr seq_cst, align 8 + store i64 %val, i64 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i64_offset: +; GCN: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}} +define void @atomic_store_i64_offset(i64 %in, i64 addrspace(4)* %out) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + store atomic i64 %in, i64 addrspace(4)* %gep seq_cst, align 8 + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i64: +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}] glc +define void @atomic_store_i64(i64 %in, i64 addrspace(4)* %out) { +entry: + store atomic i64 %in, i64 addrspace(4)* %out seq_cst, align 8 + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i64_addr64_offset: +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}} +define void @atomic_store_i64_addr64_offset(i64 %in, i64 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + store atomic i64 %in, i64 addrspace(4)* %gep seq_cst, align 8 + ret void +} + +; GCN-LABEL: {{^}}atomic_store_i64_addr64: +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}} +define void @atomic_store_i64_addr64(i64 %in, i64 addrspace(4)* %out, i64 %index) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + store atomic i64 %in, i64 addrspace(4)* %ptr seq_cst, align 8 + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_offset: +; GCN: flat_atomic_cmpswap_x2 v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i64_offset(i64 addrspace(4)* %out, i64 %in, i64 %old) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %val = cmpxchg volatile i64 addrspace(4)* %gep, i64 %old, i64 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_soffset: +; GCN: flat_atomic_cmpswap_x2 v[{{[0-9]+}}:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i64_soffset(i64 addrspace(4)* %out, i64 %in, i64 %old) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 9000 + %val = cmpxchg volatile i64 addrspace(4)* %gep, i64 %old, i64 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret_offset: +; GCN: flat_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]{{:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RET]]: +define void @atomic_cmpxchg_i64_ret_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %old) { +entry: + %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4 + %val = cmpxchg volatile i64 addrspace(4)* %gep, i64 %old, i64 %in seq_cst seq_cst + %extract0 = extractvalue { i64, i1 } %val, 0 + store i64 %extract0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_addr64_offset: +; GCN: flat_atomic_cmpswap_x2 v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i64_addr64_offset(i64 addrspace(4)* %out, i64 %in, i64 %index, i64 %old) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %val = cmpxchg volatile i64 addrspace(4)* %gep, i64 %old, i64 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret_addr64_offset: +; GCN: flat_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RET]]: +define void @atomic_cmpxchg_i64_ret_addr64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index, i64 %old) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i64 4 + %val = cmpxchg volatile i64 addrspace(4)* %gep, i64 %old, i64 %in seq_cst seq_cst + %extract0 = extractvalue { i64, i1 } %val, 0 + store i64 %extract0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64: +; GCN: flat_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i64(i64 addrspace(4)* %out, i64 %in, i64 %old) { +entry: + %val = cmpxchg volatile i64 addrspace(4)* %out, i64 %old, i64 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret: +; GCN: flat_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RET]]: +define void @atomic_cmpxchg_i64_ret(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %old) { +entry: + %val = cmpxchg volatile i64 addrspace(4)* %out, i64 %old, i64 %in seq_cst seq_cst + %extract0 = extractvalue { i64, i1 } %val, 0 + store i64 %extract0, i64 addrspace(4)* %out2 + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_addr64: +; GCN: flat_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]{{$}} +define void @atomic_cmpxchg_i64_addr64(i64 addrspace(4)* %out, i64 %in, i64 %index, i64 %old) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %val = cmpxchg volatile i64 addrspace(4)* %ptr, i64 %old, i64 %in seq_cst seq_cst + ret void +} + +; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret_addr64: +; GCN: flat_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}} +; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RET]]: +define void @atomic_cmpxchg_i64_ret_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %out2, i64 %in, i64 %index, i64 %old) { +entry: + %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index + %val = cmpxchg volatile i64 addrspace(4)* %ptr, i64 %old, i64 %in seq_cst seq_cst + %extract0 = extractvalue { i64, i1 } %val, 0 + store i64 %extract0, i64 addrspace(4)* %out2 + ret void +} Index: test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll +++ test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll @@ -3,9 +3,11 @@ declare i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* nocapture, i32) #2 declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32) #2 +declare i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* nocapture, i32) #2 declare i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* nocapture, i64) #2 declare i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* nocapture, i64) #2 +declare i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* nocapture, i64) #2 declare i32 @llvm.amdgcn.workitem.id.x() #1 @@ -108,6 +110,131 @@ ret void } +; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} +define void @flat_atomic_dec_ret_i32(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 { + %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %ptr, i32 42) + store i32 %result, i32 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} +define void @flat_atomic_dec_ret_i32_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 { + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + store i32 %result, i32 addrspace(4)* %out + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i32: +; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} +define void @flat_atomic_dec_noret_i32(i32 addrspace(4)* %ptr) nounwind { + %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %ptr, i32 42) + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i32_offset: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} +define void @flat_atomic_dec_noret_i32_offset(i32 addrspace(4)* %ptr) nounwind { + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset_addr64: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} +define void @flat_atomic_dec_ret_i32_offset_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id + %out.gep = getelementptr i32, i32 addrspace(4)* %out, i32 %id + %gep = getelementptr i32, i32 addrspace(4)* %gep.tid, i32 5 + %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + store i32 %result, i32 addrspace(4)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32_offset_addr64: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} +define void @flat_atomic_dec_noret_i32_offset_addr64(i32 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id + %gep = getelementptr i32, i32 addrspace(4)* %gep.tid, i32 5 + %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} +define void @flat_atomic_dec_ret_i64(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 { + %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %ptr, i64 42) + store i64 %result, i64 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} +define void @flat_atomic_dec_ret_i64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 { + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4 + %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + store i64 %result, i64 addrspace(4)* %out + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} +define void @flat_atomic_dec_noret_i64(i64 addrspace(4)* %ptr) nounwind { + %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %ptr, i64 42) + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i64_offset: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} +define void @flat_atomic_dec_noret_i64_offset(i64 addrspace(4)* %ptr) nounwind { + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4 + %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset_addr64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} +define void @flat_atomic_dec_ret_i64_offset_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id + %out.gep = getelementptr i64, i64 addrspace(4)* %out, i32 %id + %gep = getelementptr i64, i64 addrspace(4)* %gep.tid, i32 5 + %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + store i64 %result, i64 addrspace(4)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64_offset_addr64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} +define void @flat_atomic_dec_noret_i64_offset_addr64(i64 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id + %gep = getelementptr i64, i64 addrspace(4)* %gep.tid, i32 5 + %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + ret void +} + @lds0 = addrspace(3) global [512 x i32] undef ; SI-LABEL: {{^}}atomic_dec_shl_base_lds_0: @@ -249,3 +376,12 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readnone } attributes #2 = { nounwind argmemonly } + + + + + + + + + Index: test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll +++ test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll @@ -3,9 +3,11 @@ declare i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* nocapture, i32) #2 declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32) #2 +declare i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* nocapture, i32) #2 declare i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* nocapture, i64) #2 declare i64 @llvm.amdgcn.atomic.inc.i64.p3i64(i64 addrspace(3)* nocapture, i64) #2 +declare i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* nocapture, i64) #2 declare i32 @llvm.amdgcn.workitem.id.x() #1 @@ -231,6 +233,65 @@ ret void } +; GCN-LABEL: {{^}}flat_atomic_inc_ret_i32: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} +define void @flat_atomic_inc_ret_i32(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 { + %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %ptr, i32 42) + store i32 %result, i32 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_inc_ret_i32_offset: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} +define void @flat_atomic_inc_ret_i32_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 { + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + store i32 %result, i32 addrspace(4)* %out + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i32: +; GCN: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} +define void @flat_atomic_inc_noret_i32(i32 addrspace(4)* %ptr) nounwind { + %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %ptr, i32 42) + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i32_offset: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} +define void @flat_atomic_inc_noret_i32_offset(i32 addrspace(4)* %ptr) nounwind { + %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4 + %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_inc_ret_i32_offset_addr64: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}} +define void @flat_atomic_inc_ret_i32_offset_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id + %out.gep = getelementptr i32, i32 addrspace(4)* %out, i32 %id + %gep = getelementptr i32, i32 addrspace(4)* %gep.tid, i32 5 + %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + store i32 %result, i32 addrspace(4)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_inc_noret_i32_offset_addr64: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42 +; GCN: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}} +define void @flat_atomic_inc_noret_i32_offset_addr64(i32 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id + %gep = getelementptr i32, i32 addrspace(4)* %gep.tid, i32 5 + %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %gep, i32 42) + ret void +} + @lds1 = addrspace(3) global [512 x i64] undef, align 8 ; GCN-LABEL: {{^}}atomic_inc_shl_base_lds_0_i64: @@ -249,3 +310,74 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readnone } attributes #2 = { nounwind argmemonly } + + + + + + +; GCN-LABEL: {{^}}flat_atomic_inc_ret_i64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} +define void @flat_atomic_inc_ret_i64(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 { + %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %ptr, i64 42) + store i64 %result, i64 addrspace(4)* %out + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_inc_ret_i64_offset: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} +define void @flat_atomic_inc_ret_i64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 { + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4 + %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + store i64 %result, i64 addrspace(4)* %out + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} +define void @flat_atomic_inc_noret_i64(i64 addrspace(4)* %ptr) nounwind { + %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %ptr, i64 42) + ret void +} + +; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i64_offset: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} +define void @flat_atomic_inc_noret_i64_offset(i64 addrspace(4)* %ptr) nounwind { + %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4 + %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_inc_ret_i64_offset_addr64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}} +define void @flat_atomic_inc_ret_i64_offset_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id + %out.gep = getelementptr i64, i64 addrspace(4)* %out, i32 %id + %gep = getelementptr i64, i64 addrspace(4)* %gep.tid, i32 5 + %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + store i64 %result, i64 addrspace(4)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}flat_atomic_inc_noret_i64_offset_addr64: +; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42 +; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}} +; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}} +define void @flat_atomic_inc_noret_i64_offset_addr64(i64 addrspace(4)* %ptr) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id + %gep = getelementptr i64, i64 addrspace(4)* %gep.tid, i32 5 + %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %gep, i64 42) + ret void +}