Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1385,6 +1385,7 @@ const SDValue &getBasePtr() const { switch (getOpcode()) { case ISD::STORE: + case ISD::ATOMIC_STORE: case ISD::VP_STORE: case ISD::MSTORE: case ISD::VP_SCATTER: @@ -1457,8 +1458,12 @@ MMO->isAtomic()) && "then why are we using an AtomicSDNode?"); } - const SDValue &getBasePtr() const { return getOperand(1); } - const SDValue &getVal() const { return getOperand(2); } + const SDValue &getBasePtr() const { + return getOpcode() == ISD::ATOMIC_STORE ? getOperand(2) : getOperand(1); + } + const SDValue &getVal() const { + return getOpcode() == ISD::ATOMIC_STORE ? getOperand(1) : getOperand(2); + } /// Returns true if this SDNode represents cmpxchg atomic operation, false /// otherwise. Index: llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td =================================================================== --- llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -203,18 +203,16 @@ // separate nodes for them. This GINodeEquiv maps the non-atomic stores to // G_STORE with a non-atomic MachineMemOperand. def : GINodeEquiv { let CheckMMOIsNonAtomic = true; } - -def : GINodeEquiv { +def : GINodeEquiv { let CheckMMOIsNonAtomic = false; let CheckMMOIsAtomic = true; - let IfSignExtend = G_SEXTLOAD; - let IfZeroExtend = G_ZEXTLOAD; } -// Operands are swapped for atomic_store vs. regular store -def : GINodeEquiv { +def : GINodeEquiv { let CheckMMOIsNonAtomic = false; let CheckMMOIsAtomic = true; + let IfSignExtend = G_SEXTLOAD; + let IfZeroExtend = G_ZEXTLOAD; } def : GINodeEquiv; Index: llvm/include/llvm/Target/TargetSelectionDAG.td =================================================================== --- llvm/include/llvm/Target/TargetSelectionDAG.td +++ llvm/include/llvm/Target/TargetSelectionDAG.td @@ -315,7 +315,7 @@ ]>; def SDTAtomicStore : SDTypeProfile<0, 2, [ - SDTCisPtrTy<0>, SDTCisInt<1> + SDTCisInt<0>, SDTCisPtrTy<1> ]>; def SDTAtomicLoad : SDTypeProfile<1, 1, [ SDTCisInt<0>, SDTCisPtrTy<1> @@ -1675,7 +1675,6 @@ defm atomic_load_max : binary_atomic_op; defm atomic_load_umin : binary_atomic_op; defm atomic_load_umax : binary_atomic_op; -defm atomic_store : binary_atomic_op; defm atomic_cmp_swap : ternary_atomic_op; /// Atomic load which zeroes the excess high bits. @@ -1871,6 +1870,35 @@ MSN->getMemoryVT().getScalarType() == MVT::i32; }]>; + +def atomic_store_8 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i8; +} + +def atomic_store_16 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i16; +} + +def atomic_store_32 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i32; +} + +def atomic_store_64 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i64; +} + //===----------------------------------------------------------------------===// // Selection DAG Pattern Support. // Index: llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1043,7 +1043,7 @@ } case ISD::ATOMIC_STORE: Action = TLI.getOperationAction(Node->getOpcode(), - Node->getOperand(2).getValueType()); + Node->getOperand(1).getValueType()); break; case ISD::SELECT_CC: case ISD::STRICT_FSETCC: @@ -3080,11 +3080,10 @@ } case ISD::ATOMIC_STORE: { // There is no libcall for atomic store; fake it with ATOMIC_SWAP. - SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, - cast(Node)->getMemoryVT(), - Node->getOperand(0), - Node->getOperand(1), Node->getOperand(2), - cast(Node)->getMemOperand()); + SDValue Swap = DAG.getAtomic( + ISD::ATOMIC_SWAP, dl, cast(Node)->getMemoryVT(), + Node->getOperand(0), Node->getOperand(2), Node->getOperand(1), + cast(Node)->getMemOperand()); Results.push_back(Swap.getValue(1)); break; } Index: llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1956,9 +1956,9 @@ } SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { - SDValue Op2 = GetPromotedInteger(N->getOperand(2)); + SDValue Op2 = GetPromotedInteger(N->getOperand(1)); return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), - N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); + N->getChain(), Op2, N->getBasePtr(), N->getMemOperand()); } SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { @@ -5505,11 +5505,10 @@ SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { SDLoc dl(N); - SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, - cast(N)->getMemoryVT(), - N->getOperand(0), - N->getOperand(1), N->getOperand(2), - cast(N)->getMemOperand()); + SDValue Swap = + DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast(N)->getMemoryVT(), + N->getOperand(0), N->getOperand(2), N->getOperand(1), + cast(N)->getMemOperand()); return Swap.getValue(1); } Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4873,8 +4873,8 @@ DAG.setRoot(S); return; } - SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, - Ptr, Val, MMO); + SDValue OutChain = + DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO); setValue(&I, OutChain); DAG.setRoot(OutChain); Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -5727,7 +5727,8 @@ StoreNode->getMergedOrdering() == AtomicOrdering::Unordered || StoreNode->getMergedOrdering() == AtomicOrdering::Monotonic); - SDValue Value = StoreNode->getOpcode() == ISD::STORE + SDValue Value = (StoreNode->getOpcode() == ISD::STORE || + StoreNode->getOpcode() == ISD::ATOMIC_STORE) ? StoreNode->getOperand(1) : StoreNode->getOperand(2); SDLoc DL(Op); Index: llvm/lib/Target/AArch64/AArch64InstrAtomics.td =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -162,14 +162,14 @@ // A store operation that actually needs release semantics. class releasing_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { + : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> { let IsAtomic = 1; let IsAtomicOrderingReleaseOrStronger = 1; } // An atomic store operation that doesn't actually need to be atomic on AArch64. class relaxed_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { + : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> { let IsAtomic = 1; let IsAtomicOrderingReleaseOrStronger = 0; } Index: llvm/lib/Target/AMDGPU/AMDGPUInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -544,19 +544,18 @@ def store_hi16_#as : StoreHi16 ; def truncstorei8_hi16_#as : StoreHi16; def truncstorei16_hi16_#as : StoreHi16; - } // End let IsStore = 1, AddressSpaces = ... let IsAtomic = 1, AddressSpaces = !cast("StoreAddress_"#as).AddrSpaces in { -def atomic_store_8_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_8 node:$ptr, node:$val)>; -def atomic_store_16_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_16 node:$ptr, node:$val)>; -def atomic_store_32_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_32 node:$ptr, node:$val)>; -def atomic_store_64_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_64 node:$ptr, node:$val)>; -} +def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_8 node:$val, node:$ptr)>; +def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_16 node:$val, node:$ptr)>; +def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_32 node:$val, node:$ptr)>; +def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_64 node:$val, node:$ptr)>; +} // End let IsAtomic = 1, AddressSpaces = ... } // End foreach as multiclass noret_op { Index: llvm/lib/Target/AMDGPU/BUFInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/BUFInstructions.td +++ llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1804,14 +1804,13 @@ multiclass MUBUFStore_Atomic_Pattern { - // Store follows atomic op convention so address is first def : GCNPat < - (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), vt:$val), + (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) >; def : GCNPat < - (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset), vt:$val), + (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)), (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) >; } Index: llvm/lib/Target/AMDGPU/DSInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/DSInstructions.td +++ llvm/lib/Target/AMDGPU/DSInstructions.td @@ -806,23 +806,6 @@ } } -// Irritatingly, atomic_store reverses the order of operands from a -// normal store. -class DSAtomicWritePat : GCNPat < - (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value), - (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 0)) ->; - -multiclass DSAtomicWritePat_mc { - let OtherPredicates = [LDSRequiresM0Init] in { - def : DSAtomicWritePat(frag#"_m0")>; - } - - let OtherPredicates = [NotLDSRequiresM0Init] in { - def : DSAtomicWritePat(!cast(inst)#"_gfx9"), vt, !cast(frag)>; - } -} - defm : DSWritePat_mc ; defm : DSWritePat_mc ; defm : DSWritePat_mc ; @@ -832,12 +815,12 @@ defm : DSWritePat_mc ; } -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; let OtherPredicates = [HasD16LoadStore] in { def : DSWritePat ; Index: llvm/lib/Target/AMDGPU/FLATInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/FLATInstructions.td +++ llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -996,12 +996,6 @@ (inst $voffset, getVregSrcForVT.ret:$data, $saddr, $offset) >; -class GlobalAtomicStoreSaddrPat : GCNPat < - (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), vt:$data), - (inst $voffset, getVregSrcForVT.ret:$data, $saddr, $offset) ->; - class GlobalAtomicSaddrPat : GCNPat < (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), data_vt:$data)), @@ -1024,13 +1018,6 @@ (inst $vaddr, getVregSrcForVT.ret:$data, $offset) >; -class FlatStoreAtomicPat : GCNPat < - // atomic store follows atomic binop convention so the address comes - // first. - (node (FlatOffset i64:$vaddr, i32:$offset), vt:$data), - (inst $vaddr, getVregSrcForVT.ret:$data, $offset) ->; - class FlatStoreSignedAtomicPat : GCNPat < // atomic store follows atomic binop convention so the address comes @@ -1174,12 +1161,12 @@ def : FlatStorePat ; } -def : FlatStoreAtomicPat ; -def : FlatStoreAtomicPat ; -def : FlatStoreAtomicPat ; -def : FlatStoreAtomicPat ; -def : FlatStoreAtomicPat ; -def : FlatStoreAtomicPat ; +def : FlatStorePat ; +def : FlatStorePat ; +def : FlatStorePat ; +def : FlatStorePat ; +def : FlatStorePat ; +def : FlatStorePat ; foreach as = [ "flat", "global" ] in { defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>; @@ -1269,17 +1256,6 @@ } } -// Deal with swapped operands for atomic_store vs. regular store -multiclass GlobalFLATAtomicStorePats { - def : FlatStoreSignedAtomicPat { - let AddedComplexity = 10; - } - - def : GlobalAtomicStoreSaddrPat(!cast(inst)#"_SADDR"), node, vt> { - let AddedComplexity = 11; - } -} - multiclass GlobalFLATAtomicPatsNoRtnBase { let AddedComplexity = 11 in @@ -1444,12 +1420,12 @@ defm : GlobalFLATLoadPats_D16 ; } -defm : GlobalFLATAtomicStorePats ; -defm : GlobalFLATAtomicStorePats ; -defm : GlobalFLATAtomicStorePats ; -defm : GlobalFLATAtomicStorePats ; -defm : GlobalFLATAtomicStorePats ; -defm : GlobalFLATAtomicStorePats ; +defm : GlobalFLATStorePats ; +defm : GlobalFLATStorePats ; +defm : GlobalFLATStorePats ; +defm : GlobalFLATStorePats ; +defm : GlobalFLATStorePats ; +defm : GlobalFLATStorePats ; defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>; defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>; Index: llvm/lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -604,14 +604,14 @@ } let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces in { -def atomic_store_8_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_8_glue node:$ptr, node:$val)>; -def atomic_store_16_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_16_glue node:$ptr, node:$val)>; -def atomic_store_32_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_32_glue node:$ptr, node:$val)>; -def atomic_store_64_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_64_glue node:$ptr, node:$val)>; +def atomic_store_8_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_8_glue node:$val, node:$ptr)>; +def atomic_store_16_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_16_glue node:$val, node:$ptr)>; +def atomic_store_32_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_32_glue node:$val, node:$ptr)>; +def atomic_store_64_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_64_glue node:$val, node:$ptr)>; } // End let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces Index: llvm/lib/Target/ARM/ARMInstrInfo.td =================================================================== --- llvm/lib/Target/ARM/ARMInstrInfo.td +++ llvm/lib/Target/ARM/ARMInstrInfo.td @@ -5357,7 +5357,7 @@ def atomic_load_acquire_32 : acquiring_load; class releasing_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{ + : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr), [{ AtomicOrdering Ordering = cast(N)->getSuccessOrdering(); return isReleaseOrStronger(Ordering); }]>; @@ -6184,15 +6184,15 @@ (LDRrs ldst_so_reg:$src)>; def : ARMPat<(atomic_load_32 addrmode_imm12:$src), (LDRi12 addrmode_imm12:$src)>; -def : ARMPat<(atomic_store_8 ldst_so_reg:$ptr, GPR:$val), +def : ARMPat<(atomic_store_8 GPR:$val, ldst_so_reg:$ptr), (STRBrs GPR:$val, ldst_so_reg:$ptr)>; -def : ARMPat<(atomic_store_8 addrmode_imm12:$ptr, GPR:$val), +def : ARMPat<(atomic_store_8 GPR:$val, addrmode_imm12:$ptr), (STRBi12 GPR:$val, addrmode_imm12:$ptr)>; -def : ARMPat<(atomic_store_16 addrmode3:$ptr, GPR:$val), +def : ARMPat<(atomic_store_16 GPR:$val, addrmode3:$ptr), (STRH GPR:$val, addrmode3:$ptr)>; -def : ARMPat<(atomic_store_32 ldst_so_reg:$ptr, GPR:$val), +def : ARMPat<(atomic_store_32 GPR:$val, ldst_so_reg:$ptr), (STRrs GPR:$val, ldst_so_reg:$ptr)>; -def : ARMPat<(atomic_store_32 addrmode_imm12:$ptr, GPR:$val), +def : ARMPat<(atomic_store_32 GPR:$val, addrmode_imm12:$ptr), (STRi12 GPR:$val, addrmode_imm12:$ptr)>; Index: llvm/lib/Target/ARM/ARMInstrThumb.td =================================================================== --- llvm/lib/Target/ARM/ARMInstrThumb.td +++ llvm/lib/Target/ARM/ARMInstrThumb.td @@ -1713,17 +1713,17 @@ (tLDRi t_addrmode_is4:$src)>; def : T1Pat<(atomic_load_32 t_addrmode_rr:$src), (tLDRr t_addrmode_rr:$src)>; -def : T1Pat<(atomic_store_8 t_addrmode_is1:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_8 tGPR:$val, t_addrmode_is1:$ptr), (tSTRBi tGPR:$val, t_addrmode_is1:$ptr)>; -def : T1Pat<(atomic_store_8 t_addrmode_rr:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_8 tGPR:$val, t_addrmode_rr:$ptr), (tSTRBr tGPR:$val, t_addrmode_rr:$ptr)>; -def : T1Pat<(atomic_store_16 t_addrmode_is2:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_16 tGPR:$val, t_addrmode_is2:$ptr), (tSTRHi tGPR:$val, t_addrmode_is2:$ptr)>; -def : T1Pat<(atomic_store_16 t_addrmode_rr:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_16 tGPR:$val, t_addrmode_rr:$ptr), (tSTRHr tGPR:$val, t_addrmode_rr:$ptr)>; -def : T1Pat<(atomic_store_32 t_addrmode_is4:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_32 tGPR:$val, t_addrmode_is4:$ptr), (tSTRi tGPR:$val, t_addrmode_is4:$ptr)>; -def : T1Pat<(atomic_store_32 t_addrmode_rr:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_32 tGPR:$val, t_addrmode_rr:$ptr), (tSTRr tGPR:$val, t_addrmode_rr:$ptr)>; // Large immediate handling. Index: llvm/lib/Target/ARM/ARMInstrThumb2.td =================================================================== --- llvm/lib/Target/ARM/ARMInstrThumb2.td +++ llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -4893,23 +4893,23 @@ (t2LDRi8 t2addrmode_negimm8:$addr)>; def : T2Pat<(atomic_load_32 t2addrmode_so_reg:$addr), (t2LDRs t2addrmode_so_reg:$addr)>; -def : T2Pat<(atomic_store_8 t2addrmode_imm12:$addr, GPR:$val), +def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_imm12:$addr), (t2STRBi12 GPR:$val, t2addrmode_imm12:$addr)>; -def : T2Pat<(atomic_store_8 t2addrmode_negimm8:$addr, GPR:$val), +def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_negimm8:$addr), (t2STRBi8 GPR:$val, t2addrmode_negimm8:$addr)>; -def : T2Pat<(atomic_store_8 t2addrmode_so_reg:$addr, GPR:$val), +def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_so_reg:$addr), (t2STRBs GPR:$val, t2addrmode_so_reg:$addr)>; -def : T2Pat<(atomic_store_16 t2addrmode_imm12:$addr, GPR:$val), +def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_imm12:$addr), (t2STRHi12 GPR:$val, t2addrmode_imm12:$addr)>; -def : T2Pat<(atomic_store_16 t2addrmode_negimm8:$addr, GPR:$val), +def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_negimm8:$addr), (t2STRHi8 GPR:$val, t2addrmode_negimm8:$addr)>; -def : T2Pat<(atomic_store_16 t2addrmode_so_reg:$addr, GPR:$val), +def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_so_reg:$addr), (t2STRHs GPR:$val, t2addrmode_so_reg:$addr)>; -def : T2Pat<(atomic_store_32 t2addrmode_imm12:$addr, GPR:$val), +def : T2Pat<(atomic_store_32 GPR:$val,t2addrmode_imm12:$addr), (t2STRi12 GPR:$val, t2addrmode_imm12:$addr)>; -def : T2Pat<(atomic_store_32 t2addrmode_negimm8:$addr, GPR:$val), +def : T2Pat<(atomic_store_32 GPR:$val, t2addrmode_negimm8:$addr), (t2STRi8 GPR:$val, t2addrmode_negimm8:$addr)>; -def : T2Pat<(atomic_store_32 t2addrmode_so_reg:$addr, GPR:$val), +def : T2Pat<(atomic_store_32 GPR:$val, t2addrmode_so_reg:$addr), (t2STRs GPR:$val, t2addrmode_so_reg:$addr)>; let AddedComplexity = 8, Predicates = [IsThumb, HasAcquireRelease, HasV7Clrex] in { Index: llvm/lib/Target/AVR/AVRInstrInfo.td =================================================================== --- llvm/lib/Target/AVR/AVRInstrInfo.td +++ llvm/lib/Target/AVR/AVRInstrInfo.td @@ -1468,9 +1468,7 @@ (ins PTRRC : $rd, DRC : $rr), - "atomic_op", [(Op i16 - : $rd, DRC - : $rr)]>; + "atomic_op", [(Op DRC:$rr, i16:$rd)]>; class AtomicLoadOp : Pseudo<(outs DRC:$rd), Index: llvm/lib/Target/Hexagon/HexagonPatterns.td =================================================================== --- llvm/lib/Target/Hexagon/HexagonPatterns.td +++ llvm/lib/Target/Hexagon/HexagonPatterns.td @@ -2593,19 +2593,6 @@ : Pat<(Store Value:$val, Addr:$addr), (MI Addr:$addr, (ValueMod Value:$val))>; -// Regular stores in the DAG have two operands: value and address. -// Atomic stores also have two, but they are reversed: address, value. -// To use atomic stores with the patterns, they need to have their operands -// swapped. This relies on the knowledge that the F.Fragment uses names -// "ptr" and "val". -class AtomSt - : PatFrag<(ops node:$val, node:$ptr), !head(F.Fragments), F.PredicateCode, - F.OperandTransform> { - let IsAtomic = F.IsAtomic; - let MemoryVT = F.MemoryVT; -} - - def IMM_BYTE : SDNodeXForm; def: Storea_pat; def: Storea_pat; - def: Storea_pat, I32, addrgp, S2_storerbgp>; - def: Storea_pat, I32, addrgp, S2_storerhgp>; - def: Storea_pat, I32, addrgp, S2_storerigp>; - def: Storea_pat, V4I8, addrgp, S2_storerigp>; - def: Storea_pat, V2I16, addrgp, S2_storerigp>; - def: Storea_pat, I64, addrgp, S2_storerdgp>; - def: Storea_pat, V8I8, addrgp, S2_storerdgp>; - def: Storea_pat, V4I16, addrgp, S2_storerdgp>; - def: Storea_pat, V2I32, addrgp, S2_storerdgp>; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; def: Stoream_pat; def: Stoream_pat; @@ -2755,15 +2742,15 @@ def: Storea_pat; def: Storea_pat; def: Storea_pat; - def: Storea_pat, I32, anyimm0, PS_storerbabs>; - def: Storea_pat, I32, anyimm1, PS_storerhabs>; - def: Storea_pat, I32, anyimm2, PS_storeriabs>; - def: Storea_pat, V4I8, anyimm2, PS_storeriabs>; - def: Storea_pat, V2I16, anyimm2, PS_storeriabs>; - def: Storea_pat, I64, anyimm3, PS_storerdabs>; - def: Storea_pat, V8I8, anyimm3, PS_storerdabs>; - def: Storea_pat, V4I16, anyimm3, PS_storerdabs>; - def: Storea_pat, V2I32, anyimm3, PS_storerdabs>; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; def: Stoream_pat; def: Stoream_pat; @@ -2918,15 +2905,15 @@ defm: Storexim_pat; defm: Storexim_pat; - defm: Storexi_pat, I32, anyimm0, S2_storerb_io>; - defm: Storexi_pat, I32, anyimm1, S2_storerh_io>; - defm: Storexi_pat, I32, anyimm2, S2_storeri_io>; - defm: Storexi_pat, V4I8, anyimm2, S2_storeri_io>; - defm: Storexi_pat, V2I16, anyimm2, S2_storeri_io>; - defm: Storexi_pat, I64, anyimm3, S2_storerd_io>; - defm: Storexi_pat, V8I8, anyimm3, S2_storerd_io>; - defm: Storexi_pat, V4I16, anyimm3, S2_storerd_io>; - defm: Storexi_pat, V2I32, anyimm3, S2_storerd_io>; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; } // Reg+Reg @@ -2977,15 +2964,15 @@ def: Storexim_base_pat; def: Storexim_base_pat; - def: Storexi_base_pat, I32, S2_storerb_io>; - def: Storexi_base_pat, I32, S2_storerh_io>; - def: Storexi_base_pat, I32, S2_storeri_io>; - def: Storexi_base_pat, V4I8, S2_storeri_io>; - def: Storexi_base_pat, V2I16, S2_storeri_io>; - def: Storexi_base_pat, I64, S2_storerd_io>; - def: Storexi_base_pat, V8I8, S2_storerd_io>; - def: Storexi_base_pat, V4I16, S2_storerd_io>; - def: Storexi_base_pat, V2I32, S2_storerd_io>; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; } Index: llvm/lib/Target/LoongArch/LoongArchInstrInfo.td =================================================================== --- llvm/lib/Target/LoongArch/LoongArchInstrInfo.td +++ llvm/lib/Target/LoongArch/LoongArchInstrInfo.td @@ -1642,13 +1642,13 @@ defm : LdPat; class release_seqcst_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{ + : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr), [{ AtomicOrdering Ordering = cast(N)->getSuccessOrdering(); return isReleaseOrStronger(Ordering); }]>; class unordered_monotonic_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{ + : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr), [{ AtomicOrdering Ordering = cast(N)->getSuccessOrdering(); return !isReleaseOrStronger(Ordering); }]>; @@ -1660,23 +1660,13 @@ def atomic_store_unordered_monotonic_64 : unordered_monotonic_store; -/// AtomicStores - -multiclass AtomicStPat { - def : Pat<(StoreOp BaseAddr:$ptr, (vt StTy:$val)), - (Inst StTy:$val, BaseAddr:$ptr, 0)>; - def : Pat<(StoreOp (AddLike BaseAddr:$ptr, simm12:$imm12), (vt StTy:$val)), - (Inst StTy:$val, BaseAddr:$ptr, simm12:$imm12)>; -} - -defm : AtomicStPat; -defm : AtomicStPat; -defm : AtomicStPat, +defm : StPat; +defm : StPat; +defm : StPat, Requires<[IsLA32]>; def PseudoAtomicStoreW - : Pseudo<(outs GPR:$dst), (ins GPR:$rj, GPR:$rk)>, + : Pseudo<(outs GPR:$dst), (ins GPR:$rk, GPR:$rj)>, PseudoInstExpansion<(AMSWAP__DB_W R0, GPR:$rk, GPRMemAtomic:$rj)>; def : Pat<(atomic_store_release_seqcst_32 GPR:$rj, GPR:$rk), @@ -1684,15 +1674,15 @@ let Predicates = [IsLA64] in { def PseudoAtomicStoreD - : Pseudo<(outs GPR:$dst), (ins GPR:$rj, GPR:$rk)>, + : Pseudo<(outs GPR:$dst), (ins GPR:$rk, GPR:$rj)>, PseudoInstExpansion<(AMSWAP__DB_D R0, GPR:$rk, GPRMemAtomic:$rj)>; def : Pat<(atomic_store_release_seqcst_64 GPR:$rj, GPR:$rk), (PseudoAtomicStoreD GPR:$rj, GPR:$rk)>; defm : LdPat; -defm : AtomicStPat; -defm : AtomicStPat; +defm : StPat; +defm : StPat; } // Predicates = [IsLA64] /// Atomic Ops Index: llvm/lib/Target/Mips/Mips64InstrInfo.td =================================================================== --- llvm/lib/Target/Mips/Mips64InstrInfo.td +++ llvm/lib/Target/Mips/Mips64InstrInfo.td @@ -894,13 +894,13 @@ def : MipsPat<(atomic_load_64 addr:$a), (LD addr:$a)>, ISA_MIPS3, GPR_64; // Atomic store patterns. -def : MipsPat<(atomic_store_8 addr:$a, GPR64:$v), (SB64 GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_8 GPR64:$v, addr:$a), (SB64 GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; -def : MipsPat<(atomic_store_16 addr:$a, GPR64:$v), (SH64 GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_16 GPR64:$v, addr:$a), (SH64 GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; -def : MipsPat<(atomic_store_32 addr:$a, GPR64:$v), (SW64 GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_32 GPR64:$v, addr:$a), (SW64 GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; -def : MipsPat<(atomic_store_64 addr:$a, GPR64:$v), (SD GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_64 GPR64:$v, addr:$a), (SD GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; // Patterns used for matching away redundant sign extensions. Index: llvm/lib/Target/Mips/MipsInstrInfo.td =================================================================== --- llvm/lib/Target/Mips/MipsInstrInfo.td +++ llvm/lib/Target/Mips/MipsInstrInfo.td @@ -3357,11 +3357,11 @@ def : MipsPat<(atomic_load_32 addr:$a), (LW addr:$a)>, ISA_MIPS1; // Atomic store patterns. - def : MipsPat<(atomic_store_8 addr:$a, GPR32:$v), (SB GPR32:$v, addr:$a)>, + def : MipsPat<(atomic_store_8 GPR32:$v, addr:$a), (SB GPR32:$v, addr:$a)>, ISA_MIPS1; - def : MipsPat<(atomic_store_16 addr:$a, GPR32:$v), (SH GPR32:$v, addr:$a)>, + def : MipsPat<(atomic_store_16 GPR32:$v, addr:$a), (SH GPR32:$v, addr:$a)>, ISA_MIPS1; - def : MipsPat<(atomic_store_32 addr:$a, GPR32:$v), (SW GPR32:$v, addr:$a)>, + def : MipsPat<(atomic_store_32 GPR32:$v, addr:$a), (SW GPR32:$v, addr:$a)>, ISA_MIPS1; } Index: llvm/lib/Target/PowerPC/PPCISelLowering.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -11045,14 +11045,14 @@ SmallVector Ops{ N->getOperand(0), DAG.getConstant(Intrinsic::ppc_atomic_store_i128, dl, MVT::i32)}; - SDValue Val = N->getOperand(2); + SDValue Val = N->getOperand(1); SDValue ValLo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, Val); SDValue ValHi = DAG.getNode(ISD::SRL, dl, MVT::i128, Val, DAG.getConstant(64, dl, MVT::i32)); ValHi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, ValHi); Ops.push_back(ValLo); Ops.push_back(ValHi); - Ops.push_back(N->getOperand(1)); + Ops.push_back(N->getOperand(2)); return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, Tys, Ops, MemVT, N->getMemOperand()); } Index: llvm/lib/Target/PowerPC/PPCInstr64Bit.td =================================================================== --- llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -1970,8 +1970,8 @@ def : Pat<(atomic_load_64 DSForm:$src), (LD memrix:$src)>; def : Pat<(atomic_load_64 XForm:$src), (LDX memrr:$src)>; -def : Pat<(atomic_store_64 DSForm:$ptr, i64:$val), (STD g8rc:$val, memrix:$ptr)>; -def : Pat<(atomic_store_64 XForm:$ptr, i64:$val), (STDX g8rc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_64 i64:$val, DSForm:$ptr), (STD g8rc:$val, memrix:$ptr)>; +def : Pat<(atomic_store_64 i64:$val, XForm:$ptr), (STDX g8rc:$val, memrr:$ptr)>; let Predicates = [IsISA3_0, In64BitMode] in { def : Pat<(i64 (int_ppc_cmpeqb g8rc:$a, g8rc:$b)), Index: llvm/lib/Target/PowerPC/PPCInstrInfo.td =================================================================== --- llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -5027,12 +5027,12 @@ def : Pat<(atomic_load_32 XForm:$src), (LWZX memrr:$src)>; // Atomic stores -def : Pat<(atomic_store_8 DForm:$ptr, i32:$val), (STB gprc:$val, memri:$ptr)>; -def : Pat<(atomic_store_16 DForm:$ptr, i32:$val), (STH gprc:$val, memri:$ptr)>; -def : Pat<(atomic_store_32 DForm:$ptr, i32:$val), (STW gprc:$val, memri:$ptr)>; -def : Pat<(atomic_store_8 XForm:$ptr, i32:$val), (STBX gprc:$val, memrr:$ptr)>; -def : Pat<(atomic_store_16 XForm:$ptr, i32:$val), (STHX gprc:$val, memrr:$ptr)>; -def : Pat<(atomic_store_32 XForm:$ptr, i32:$val), (STWX gprc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_8 i32:$val, DForm:$ptr), (STB gprc:$val, memri:$ptr)>; +def : Pat<(atomic_store_16 i32:$val, DForm:$ptr), (STH gprc:$val, memri:$ptr)>; +def : Pat<(atomic_store_32 i32:$val, DForm:$ptr), (STW gprc:$val, memri:$ptr)>; +def : Pat<(atomic_store_8 i32:$val, XForm:$ptr), (STBX gprc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_16 i32:$val, XForm:$ptr), (STHX gprc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_32 i32:$val, XForm:$ptr), (STWX gprc:$val, memrr:$ptr)>; let Predicates = [IsISA3_0] in { Index: llvm/lib/Target/PowerPC/PPCInstrP10.td =================================================================== --- llvm/lib/Target/PowerPC/PPCInstrP10.td +++ llvm/lib/Target/PowerPC/PPCInstrP10.td @@ -1236,19 +1236,19 @@ (PLDpc $ga, 0)>; // Atomic Store - def : Pat<(atomic_store_8 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS), + def : Pat<(atomic_store_8 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTBpc $RS, $ga, 0)>; - def : Pat<(atomic_store_16 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS), + def : Pat<(atomic_store_16 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTHpc $RS, $ga, 0)>; - def : Pat<(atomic_store_32 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS), + def : Pat<(atomic_store_32 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTWpc $RS, $ga, 0)>; - def : Pat<(atomic_store_8 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_8 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTB8pc $RS, $ga, 0)>; - def : Pat<(atomic_store_16 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_16 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTH8pc $RS, $ga, 0)>; - def : Pat<(atomic_store_32 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_32 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTW8pc $RS, $ga, 0)>; - def : Pat<(atomic_store_64 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_64 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTDpc $RS, $ga, 0)>; // Special Cases For PPCstore_scal_int_from_vsr @@ -2276,10 +2276,10 @@ def : Pat<(atomic_load_64 PDForm:$src), (PLD memri34:$src)>; // Atomic Store - def : Pat<(atomic_store_8 PDForm:$dst, i32:$RS), (PSTB $RS, memri34:$dst)>; - def : Pat<(atomic_store_16 PDForm:$dst, i32:$RS), (PSTH $RS, memri34:$dst)>; - def : Pat<(atomic_store_32 PDForm:$dst, i32:$RS), (PSTW $RS, memri34:$dst)>; - def : Pat<(atomic_store_64 PDForm:$dst, i64:$RS), (PSTD $RS, memri34:$dst)>; + def : Pat<(atomic_store_8 i32:$RS, PDForm:$dst), (PSTB $RS, memri34:$dst)>; + def : Pat<(atomic_store_16 i32:$RS, PDForm:$dst), (PSTH $RS, memri34:$dst)>; + def : Pat<(atomic_store_32 i32:$RS, PDForm:$dst), (PSTW $RS, memri34:$dst)>; + def : Pat<(atomic_store_64 i64:$RS, PDForm:$dst), (PSTD $RS, memri34:$dst)>; // Prefixed fpext to v2f64 def : Pat<(v4f32 (PPCldvsxlh PDForm:$src)), Index: llvm/lib/Target/RISCV/RISCVInstrInfoA.td =================================================================== --- llvm/lib/Target/RISCV/RISCVInstrInfoA.td +++ llvm/lib/Target/RISCV/RISCVInstrInfoA.td @@ -46,8 +46,7 @@ class AtomicStPat - : Pat<(StoreOp (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12), - (vt StTy:$rs2)), + : Pat<(StoreOp (vt StTy:$rs2), (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12)), (Inst StTy:$rs2, GPR:$rs1, simm12:$imm12)>; //===----------------------------------------------------------------------===// Index: llvm/lib/Target/Sparc/SparcInstr64Bit.td =================================================================== --- llvm/lib/Target/Sparc/SparcInstr64Bit.td +++ llvm/lib/Target/Sparc/SparcInstr64Bit.td @@ -492,8 +492,8 @@ def : Pat<(i64 (atomic_load_64 ADDRri:$src)), (LDXri ADDRri:$src)>; // atomic_store_64 val, addr -> store val, addr -def : Pat<(atomic_store_64 ADDRrr:$dst, i64:$val), (STXrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_64 ADDRri:$dst, i64:$val), (STXri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_64 i64:$val, ADDRrr:$dst), (STXrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_64 i64:$val, ADDRri:$dst), (STXri ADDRri:$dst, $val)>; } // Predicates = [Is64Bit] Index: llvm/lib/Target/Sparc/SparcInstrInfo.td =================================================================== --- llvm/lib/Target/Sparc/SparcInstrInfo.td +++ llvm/lib/Target/Sparc/SparcInstrInfo.td @@ -1868,12 +1868,12 @@ def : Pat<(i32 (atomic_load_32 ADDRri:$src)), (LDri ADDRri:$src)>; // atomic_store val, addr -> store val, addr -def : Pat<(atomic_store_8 ADDRrr:$dst, i32:$val), (STBrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_8 ADDRri:$dst, i32:$val), (STBri ADDRri:$dst, $val)>; -def : Pat<(atomic_store_16 ADDRrr:$dst, i32:$val), (STHrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_16 ADDRri:$dst, i32:$val), (STHri ADDRri:$dst, $val)>; -def : Pat<(atomic_store_32 ADDRrr:$dst, i32:$val), (STrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_32 ADDRri:$dst, i32:$val), (STri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_8 i32:$val, ADDRrr:$dst), (STBrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_8 i32:$val, ADDRri:$dst), (STBri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_16 i32:$val, ADDRrr:$dst), (STHrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_16 i32:$val, ADDRri:$dst), (STHri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_32 i32:$val, ADDRrr:$dst), (STrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_32 i32:$val, ADDRri:$dst), (STri ADDRri:$dst, $val)>; // A register pair with zero upper half. // The upper part is done with ORrr instead of `COPY G0` Index: llvm/lib/Target/SystemZ/SystemZISelLowering.cpp =================================================================== --- llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -5961,9 +5961,8 @@ case ISD::ATOMIC_STORE: { SDLoc DL(N); SDVTList Tys = DAG.getVTList(MVT::Other); - SDValue Ops[] = { N->getOperand(0), - lowerI128ToGR128(DAG, N->getOperand(2)), - N->getOperand(1) }; + SDValue Ops[] = {N->getOperand(0), lowerI128ToGR128(DAG, N->getOperand(1)), + N->getOperand(2)}; MachineMemOperand *MMO = cast(N)->getMemOperand(); SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128, DL, Tys, Ops, MVT::i128, MMO); Index: llvm/lib/Target/VE/VEInstrInfo.td =================================================================== --- llvm/lib/Target/VE/VEInstrInfo.td +++ llvm/lib/Target/VE/VEInstrInfo.td @@ -1864,10 +1864,10 @@ multiclass ATMSTm { - def : Pat<(from ADDRrri:$addr, ty:$src), (torri MEMrri:$addr, $src)>; - def : Pat<(from ADDRrii:$addr, ty:$src), (torii MEMrii:$addr, $src)>; - def : Pat<(from ADDRzri:$addr, ty:$src), (tozri MEMzri:$addr, $src)>; - def : Pat<(from ADDRzii:$addr, ty:$src), (tozii MEMzii:$addr, $src)>; + def : Pat<(from ty:$src, ADDRrri:$addr), (torri MEMrri:$addr, $src)>; + def : Pat<(from ty:$src, ADDRrii:$addr), (torii MEMrii:$addr, $src)>; + def : Pat<(from ty:$src, ADDRzri:$addr), (tozri MEMzri:$addr, $src)>; + def : Pat<(from ty:$src, ADDRzii:$addr), (tozii MEMzii:$addr, $src)>; } defm : ATMSTm; defm : ATMSTm; @@ -1880,14 +1880,14 @@ RM torii, RM tozri, RM tozii> { - def : Pat<(from ADDRrri:$addr, (i32 (trunc i64:$src))), - (torri MEMrri:$addr, (l2i $src))>; - def : Pat<(from ADDRrii:$addr, (i32 (trunc i64:$src))), - (torii MEMrii:$addr, (l2i $src))>; - def : Pat<(from ADDRzri:$addr, (i32 (trunc i64:$src))), - (tozri MEMzri:$addr, (l2i $src))>; - def : Pat<(from ADDRzii:$addr, (i32 (trunc i64:$src))), - (tozii MEMzii:$addr, (l2i $src))>; + def : Pat<(from (i32 (trunc i64:$src)), ADDRrri:$addr), + (torri MEMrri:$addr, (EXTRACT_SUBREG $src, sub_i32))>; + def : Pat<(from (i32 (trunc i64:$src)), ADDRrii:$addr), + (torii MEMrii:$addr, (EXTRACT_SUBREG $src, sub_i32))>; + def : Pat<(from (i32 (trunc i64:$src)), ADDRzri:$addr), + (tozri MEMzri:$addr, (EXTRACT_SUBREG $src, sub_i32))>; + def : Pat<(from (i32 (trunc i64:$src)), ADDRzii:$addr), + (tozii MEMzii:$addr, (EXTRACT_SUBREG $src, sub_i32))>; } defm : TRATMSTm; defm : TRATMSTm; Index: llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td =================================================================== --- llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td +++ llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td @@ -192,16 +192,17 @@ defm ATOMIC_STORE_I32 : AtomicStore; defm ATOMIC_STORE_I64 : AtomicStore; -// We need an 'atomic' version of store patterns because store and atomic_store -// nodes have different operand orders: -// store: (store $val, $ptr) -// atomic_store: (store $ptr, $val) +// We used to need an 'atomic' version of store patterns because store and atomic_store +// nodes have different operand orders. +// +// TODO: This is no longer true and atomic_store and store patterns +// can be unified. multiclass AStorePat { - def : Pat<(kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val), + def : Pat<(kind ty:$val, (AddrOps32 offset32_op:$offset, I32:$addr)), (!cast(inst#_A32) 0, $offset, $addr, $val)>, Requires<[HasAddr32, HasAtomics]>; - def : Pat<(kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val), + def : Pat<(kind ty:$val, (AddrOps64 offset64_op:$offset, I64:$addr)), (!cast(inst#_A64) 0, $offset, $addr, $val)>, Requires<[HasAddr64, HasAtomics]>; } @@ -221,8 +222,8 @@ // instructions, we just need to match bare atomic stores. On the other hand, // truncating stores from i64 values are once truncated to i32 first. class trunc_astore_64 : - PatFrag<(ops node:$addr, node:$val), - (kind node:$addr, (i32 (trunc (i64 node:$val))))>; + PatFrag<(ops node:$val, node:$addr), + (kind (i32 (trunc (i64 node:$val))), node:$addr)>; def trunc_astore_8_64 : trunc_astore_64; def trunc_astore_16_64 : trunc_astore_64; def trunc_astore_32_64 : trunc_astore_64; Index: llvm/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.cpp +++ llvm/lib/Target/X86/X86ISelLowering.cpp @@ -31168,8 +31168,8 @@ if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) { SDValue Chain; if (Subtarget.hasSSE1()) { - SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, - Node->getOperand(2)); + SDValue SclToVec = + DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal()); MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32; SclToVec = DAG.getBitcast(StVT, SclToVec); SDVTList Tys = DAG.getVTList(MVT::Other); @@ -31183,9 +31183,8 @@ int SPFI = cast(StackPtr.getNode())->getIndex(); MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); - Chain = - DAG.getStore(Node->getChain(), dl, Node->getOperand(2), StackPtr, - MPI, MaybeAlign(), MachineMemOperand::MOStore); + Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr, + MPI, MaybeAlign(), MachineMemOperand::MOStore); SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); SDValue LdOps[] = {Chain, StackPtr}; SDValue Value = DAG.getMemIntrinsicNode( @@ -31214,11 +31213,9 @@ // Convert seq_cst store -> xchg // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b) // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment. - SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, - Node->getMemoryVT(), - Node->getOperand(0), - Node->getOperand(1), Node->getOperand(2), - Node->getMemOperand()); + SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(), + Node->getOperand(0), Node->getOperand(2), + Node->getOperand(1), Node->getMemOperand()); return Swap.getValue(1); } Index: llvm/lib/Target/X86/X86InstrCompiler.td =================================================================== --- llvm/lib/Target/X86/X86InstrCompiler.td +++ llvm/lib/Target/X86/X86InstrCompiler.td @@ -1071,30 +1071,28 @@ * (see below the RELEASE_MOV* / ACQUIRE_MOV* pseudo-instructions) */ multiclass RELEASE_BINOP_MI { - def : Pat<(atomic_store_8 addr:$dst, - (op (atomic_load_8 addr:$dst), (i8 imm:$src))), + def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 imm:$src)), + addr:$dst), (!cast(Name#"8mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_16 addr:$dst, - (op (atomic_load_16 addr:$dst), (i16 imm:$src))), + def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 imm:$src)), + addr:$dst), (!cast(Name#"16mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_32 addr:$dst, - (op (atomic_load_32 addr:$dst), (i32 imm:$src))), + def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 imm:$src)), + addr:$dst), (!cast(Name#"32mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_64 addr:$dst, - (op (atomic_load_64 addr:$dst), (i64immSExt32:$src))), + def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64immSExt32:$src)), + addr:$dst), (!cast(Name#"64mi32") addr:$dst, (i64immSExt32:$src))>; - - def : Pat<(atomic_store_8 addr:$dst, - (op (atomic_load_8 addr:$dst), (i8 GR8:$src))), + def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 GR8:$src)), addr:$dst), (!cast(Name#"8mr") addr:$dst, GR8:$src)>; - def : Pat<(atomic_store_16 addr:$dst, - (op (atomic_load_16 addr:$dst), (i16 GR16:$src))), + def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 GR16:$src)), + addr:$dst), (!cast(Name#"16mr") addr:$dst, GR16:$src)>; - def : Pat<(atomic_store_32 addr:$dst, - (op (atomic_load_32 addr:$dst), (i32 GR32:$src))), + def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 GR32:$src)), + addr:$dst), (!cast(Name#"32mr") addr:$dst, GR32:$src)>; - def : Pat<(atomic_store_64 addr:$dst, - (op (atomic_load_64 addr:$dst), (i64 GR64:$src))), + def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64 GR64:$src)), + addr:$dst), (!cast(Name#"64mr") addr:$dst, GR64:$src)>; } defm : RELEASE_BINOP_MI<"ADD", add>; @@ -1131,13 +1129,13 @@ multiclass RELEASE_UNOP { - def : Pat<(atomic_store_8 addr:$dst, dag8), + def : Pat<(atomic_store_8 dag8, addr:$dst), (!cast(Name#8m) addr:$dst)>; - def : Pat<(atomic_store_16 addr:$dst, dag16), + def : Pat<(atomic_store_16 dag16, addr:$dst), (!cast(Name#16m) addr:$dst)>; - def : Pat<(atomic_store_32 addr:$dst, dag32), + def : Pat<(atomic_store_32 dag32, addr:$dst), (!cast(Name#32m) addr:$dst)>; - def : Pat<(atomic_store_64 addr:$dst, dag64), + def : Pat<(atomic_store_64 dag64, addr:$dst), (!cast(Name#64m) addr:$dst)>; } @@ -1165,22 +1163,22 @@ (not (i32 (atomic_load_32 addr:$dst))), (not (i64 (atomic_load_64 addr:$dst)))>; -def : Pat<(atomic_store_8 addr:$dst, (i8 imm:$src)), +def : Pat<(atomic_store_8 (i8 imm:$src), addr:$dst), (MOV8mi addr:$dst, imm:$src)>; -def : Pat<(atomic_store_16 addr:$dst, (i16 imm:$src)), +def : Pat<(atomic_store_16 (i16 imm:$src), addr:$dst), (MOV16mi addr:$dst, imm:$src)>; -def : Pat<(atomic_store_32 addr:$dst, (i32 imm:$src)), +def : Pat<(atomic_store_32 (i32 imm:$src), addr:$dst), (MOV32mi addr:$dst, imm:$src)>; -def : Pat<(atomic_store_64 addr:$dst, (i64immSExt32:$src)), +def : Pat<(atomic_store_64 (i64immSExt32:$src), addr:$dst), (MOV64mi32 addr:$dst, i64immSExt32:$src)>; -def : Pat<(atomic_store_8 addr:$dst, GR8:$src), +def : Pat<(atomic_store_8 GR8:$src, addr:$dst), (MOV8mr addr:$dst, GR8:$src)>; -def : Pat<(atomic_store_16 addr:$dst, GR16:$src), +def : Pat<(atomic_store_16 GR16:$src, addr:$dst), (MOV16mr addr:$dst, GR16:$src)>; -def : Pat<(atomic_store_32 addr:$dst, GR32:$src), +def : Pat<(atomic_store_32 GR32:$src, addr:$dst), (MOV32mr addr:$dst, GR32:$src)>; -def : Pat<(atomic_store_64 addr:$dst, GR64:$src), +def : Pat<(atomic_store_64 GR64:$src, addr:$dst), (MOV64mr addr:$dst, GR64:$src)>; def : Pat<(i8 (atomic_load_8 addr:$src)), (MOV8rm addr:$src)>; @@ -1189,18 +1187,18 @@ def : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>; // Floating point loads/stores. -def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), +def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), (MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>; -def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), +def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), (VMOVSSmr addr:$dst, FR32:$src)>, Requires<[UseAVX]>; -def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), +def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), (VMOVSSZmr addr:$dst, FR32:$src)>, Requires<[HasAVX512]>; -def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), +def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), (MOVSDmr addr:$dst, FR64:$src)>, Requires<[UseSSE2]>; -def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), +def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[UseAVX]>; -def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), +def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[HasAVX512]>; def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), Index: llvm/test/TableGen/GlobalISelEmitter-atomic_store.td =================================================================== --- llvm/test/TableGen/GlobalISelEmitter-atomic_store.td +++ llvm/test/TableGen/GlobalISelEmitter-atomic_store.td @@ -5,20 +5,17 @@ def ST_ATOM_B32 : I<(outs), (ins GPR32Op:$val, GPR32Op:$ptr), []>; -// Check that the pattern for atomic_store inverts the operands to -// match the order of G_STORE. - // GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE, // GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/1, // GISEL-NEXT: GIM_CheckAtomicOrderingOrStrongerThan, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::Unordered, -// GISEL-NEXT: // MIs[0] ptr -// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0, // GISEL-NEXT: // MIs[0] val // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, -// GISEL-NEXT: // (atomic_store iPTR:{ *:[iPTR] }:$ptr, i32:{ *:[i32] }:$val)<> => (ST_ATOM_B32 GPR32Op:{ *:[i32] }:$val, GPR32Op:{ *:[i32] }:$ptr) +// GISEL-NEXT: // MIs[0] ptr +// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0, +// GISEL-NEXT: // (atomic_store i32:{ *:[i32] }:$val, iPTR:{ *:[iPTR] }:$ptr)<> => (ST_ATOM_B32 GPR32Op:{ *:[i32] }:$val, GPR32Op:{ *:[i32] }:$ptr) // GISEL-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ST_ATOM_B32, def : Pat< // (atomic_store_8 iPTR:$ptr, i32:$val), - (atomic_store_8 iPTR:$ptr, i32:$val), + (atomic_store_8 i32:$val, iPTR:$ptr), (ST_ATOM_B32 GPR32Op:$val, GPR32Op:$ptr) >; Index: llvm/utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- llvm/utils/TableGen/GlobalISelEmitter.cpp +++ llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -787,13 +787,11 @@ } } - bool IsAtomic = false; if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) InsnMatcher.addPredicate("NotAtomic"); else if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { - IsAtomic = true; InsnMatcher.addPredicate( "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); } @@ -847,27 +845,6 @@ } } - // Hack around an unfortunate mistake in how atomic store (and really - // atomicrmw in general) operands were ordered. A ISD::STORE used the order - // , order. ISD::ATOMIC_STORE used the opposite, - // , . In GlobalISel there's just the one store - // opcode, so we need to swap the operands here to get the right type check. - if (IsAtomic && SrcGIOrNull->TheDef->getName() == "G_STORE") { - assert(NumChildren == 2 && "wrong operands for atomic store"); - - const TreePatternNode *PtrChild = Src->getChild(0); - const TreePatternNode *ValueChild = Src->getChild(1); - - if (auto Error = importChildMatcher(Rule, InsnMatcher, PtrChild, true, - false, 1, TempOpIdx)) - return std::move(Error); - - if (auto Error = importChildMatcher(Rule, InsnMatcher, ValueChild, false, - false, 0, TempOpIdx)) - return std::move(Error); - return InsnMatcher; - } - // Match the used operands (i.e. the children of the operator). bool IsIntrinsic = SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" ||