Index: lib/Target/AMDGPU/AMDGPUInstructions.td =================================================================== --- lib/Target/AMDGPU/AMDGPUInstructions.td +++ lib/Target/AMDGPU/AMDGPUInstructions.td @@ -422,26 +422,14 @@ return cast(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS; }]>; -multiclass AtomicCmpSwapLocal { - - def _32_local : PatFrag < +class AtomicCmpSwapLocal : PatFrag< (ops node:$ptr, node:$cmp, node:$swap), (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{ AtomicSDNode *AN = cast(N); - return AN->getMemoryVT() == MVT::i32 && - AN->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS; - }]>; - - def _64_local : PatFrag< - (ops node:$ptr, node:$cmp, node:$swap), - (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{ - AtomicSDNode *AN = cast(N); - return AN->getMemoryVT() == MVT::i64 && - AN->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS; - }]>; -} + return AN->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS; +}]>; -defm atomic_cmp_swap : AtomicCmpSwapLocal ; +def atomic_cmp_swap_local : AtomicCmpSwapLocal ; multiclass global_binary_atomic_op { def "" : PatFrag< Index: lib/Target/AMDGPU/DSInstructions.td =================================================================== --- lib/Target/AMDGPU/DSInstructions.td +++ lib/Target/AMDGPU/DSInstructions.td @@ -649,35 +649,35 @@ // 32-bit atomics. -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicCmpXChg; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicCmpXChg; // 64-bit atomics. -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; -def : DSAtomicRetPat; - -def : DSAtomicCmpXChg; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; +def : DSAtomicRetPat; + +def : DSAtomicCmpXChg; //===----------------------------------------------------------------------===// // Real instructions Index: lib/Target/AMDGPU/EvergreenInstructions.td =================================================================== --- lib/Target/AMDGPU/EvergreenInstructions.td +++ lib/Target/AMDGPU/EvergreenInstructions.td @@ -660,7 +660,7 @@ [(set i32:$dst, (atomic_swap_local i32:$src0, i32:$src1))] >; def LDS_CMPST_RET : R600_LDS_1A2D_RET <0x30, "LDS_CMPST", - [(set i32:$dst, (atomic_cmp_swap_32_local i32:$src0, i32:$src1, i32:$src2))] + [(set i32:$dst, (atomic_cmp_swap_local i32:$src0, i32:$src1, i32:$src2))] >; def LDS_READ_RET : R600_LDS_1A <0x32, "LDS_READ_RET", [(set (i32 R600_Reg32:$dst), (load_local R600_Reg32:$src0))] Index: lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.td +++ lib/Target/AMDGPU/SIInstrInfo.td @@ -254,27 +254,28 @@ [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue] >; - def _local : local_binary_atomic_op (NAME#"_glue")>; -} - -defm si_atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">; -defm si_atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">; -defm si_atomic_inc : SIAtomicM0Glue2 <"INC", 1>; -defm si_atomic_dec : SIAtomicM0Glue2 <"DEC", 1>; -defm si_atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">; -defm si_atomic_load_min : SIAtomicM0Glue2 <"LOAD_MIN">; -defm si_atomic_load_max : SIAtomicM0Glue2 <"LOAD_MAX">; -defm si_atomic_load_or : SIAtomicM0Glue2 <"LOAD_OR">; -defm si_atomic_load_xor : SIAtomicM0Glue2 <"LOAD_XOR">; -defm si_atomic_load_umin : SIAtomicM0Glue2 <"LOAD_UMIN">; -defm si_atomic_load_umax : SIAtomicM0Glue2 <"LOAD_UMAX">; -defm si_atomic_swap : SIAtomicM0Glue2 <"SWAP">; - -def si_atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3, + def _local_m0 : local_binary_atomic_op (NAME#"_glue")>; +} + +defm atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">; +defm atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">; +defm atomic_inc : SIAtomicM0Glue2 <"INC", 1>; +defm atomic_dec : SIAtomicM0Glue2 <"DEC", 1>; +defm atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">; +defm atomic_load_min : SIAtomicM0Glue2 <"LOAD_MIN">; +defm atomic_load_max : SIAtomicM0Glue2 <"LOAD_MAX">; +defm atomic_load_or : SIAtomicM0Glue2 <"LOAD_OR">; +defm atomic_load_xor : SIAtomicM0Glue2 <"LOAD_XOR">; +defm atomic_load_umin : SIAtomicM0Glue2 <"LOAD_UMIN">; +defm atomic_load_umax : SIAtomicM0Glue2 <"LOAD_UMAX">; +defm atomic_swap : SIAtomicM0Glue2 <"SWAP">; + +def atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3, [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue] >; -defm si_atomic_cmp_swap : AtomicCmpSwapLocal ; +def atomic_cmp_swap_local_m0 : AtomicCmpSwapLocal; + def as_i1imm : SDNodeXFormgetTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i1);