Index: llvm/include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1364,6 +1364,7 @@ const SDValue &getBasePtr() const { switch (getOpcode()) { case ISD::STORE: + case ISD::ATOMIC_STORE: case ISD::VP_STORE: case ISD::MSTORE: case ISD::VP_SCATTER: @@ -1430,8 +1431,12 @@ MMO->isAtomic()) && "then why are we using an AtomicSDNode?"); } - const SDValue &getBasePtr() const { return getOperand(1); } - const SDValue &getVal() const { return getOperand(2); } + const SDValue &getBasePtr() const { + return getOpcode() == ISD::ATOMIC_STORE ? getOperand(2) : getOperand(1); + } + const SDValue &getVal() const { + return getOpcode() == ISD::ATOMIC_STORE ? getOperand(1) : getOperand(2); + } /// Returns true if this SDNode represents cmpxchg atomic operation, false /// otherwise. Index: llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td =================================================================== --- llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -180,14 +180,13 @@ // separate nodes for them. This GINodeEquiv maps the non-atomic stores to // G_STORE with a non-atomic MachineMemOperand. def : GINodeEquiv { let CheckMMOIsNonAtomic = true; } - -def : GINodeEquiv { +def : GINodeEquiv { let CheckMMOIsNonAtomic = false; let CheckMMOIsAtomic = true; } -// Operands are swapped for atomic_store vs. regular store -def : GINodeEquiv { + +def : GINodeEquiv { let CheckMMOIsNonAtomic = false; let CheckMMOIsAtomic = true; } Index: llvm/include/llvm/Target/TargetSelectionDAG.td =================================================================== --- llvm/include/llvm/Target/TargetSelectionDAG.td +++ llvm/include/llvm/Target/TargetSelectionDAG.td @@ -291,7 +291,7 @@ ]>; def SDTAtomicStore : SDTypeProfile<0, 2, [ - SDTCisPtrTy<0>, SDTCisInt<1> + SDTCisInt<0>, SDTCisPtrTy<1> ]>; def SDTAtomicLoad : SDTypeProfile<1, 1, [ SDTCisInt<0>, SDTCisPtrTy<1> @@ -1600,7 +1600,6 @@ defm atomic_load_max : binary_atomic_op; defm atomic_load_umin : binary_atomic_op; defm atomic_load_umax : binary_atomic_op; -defm atomic_store : binary_atomic_op; defm atomic_cmp_swap : ternary_atomic_op; def atomic_load_8 : @@ -1628,6 +1627,34 @@ let MemoryVT = i64; } +def atomic_store_8 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i8; +} + +def atomic_store_16 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i16; +} + +def atomic_store_32 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i32; +} + +def atomic_store_64 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = i64; +} + //===----------------------------------------------------------------------===// // Selection DAG Pattern Support. // Index: llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1027,7 +1027,7 @@ } case ISD::ATOMIC_STORE: Action = TLI.getOperationAction(Node->getOpcode(), - Node->getOperand(2).getValueType()); + Node->getOperand(1).getValueType()); break; case ISD::SELECT_CC: case ISD::STRICT_FSETCC: @@ -2779,7 +2779,7 @@ SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast(Node)->getMemoryVT(), Node->getOperand(0), - Node->getOperand(1), Node->getOperand(2), + Node->getOperand(2), Node->getOperand(1), cast(Node)->getMemOperand()); Results.push_back(Swap.getValue(1)); break; Index: llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1763,9 +1763,9 @@ } SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { - SDValue Op2 = GetPromotedInteger(N->getOperand(2)); + SDValue Op2 = GetPromotedInteger(N->getOperand(1)); return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), - N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); + N->getChain(), Op2, N->getBasePtr(), N->getMemOperand()); } SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { @@ -4998,7 +4998,7 @@ SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast(N)->getMemoryVT(), N->getOperand(0), - N->getOperand(1), N->getOperand(2), + N->getOperand(2), N->getOperand(1), cast(N)->getMemOperand()); return Swap.getValue(1); } Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4737,7 +4737,7 @@ return; } SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, - Ptr, Val, MMO); + Val, Ptr, MMO); DAG.setRoot(OutChain); Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -4976,7 +4976,8 @@ StoreNode->getMergedOrdering() == AtomicOrdering::Unordered || StoreNode->getMergedOrdering() == AtomicOrdering::Monotonic); - SDValue Value = StoreNode->getOpcode() == ISD::STORE + SDValue Value = (StoreNode->getOpcode() == ISD::STORE || + StoreNode->getOpcode() == ISD::ATOMIC_STORE) ? StoreNode->getOperand(1) : StoreNode->getOperand(2); SDLoc DL(Op); Index: llvm/lib/Target/AArch64/AArch64InstrAtomics.td =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -140,14 +140,14 @@ // A store operation that actually needs release semantics. class releasing_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { + : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> { let IsAtomic = 1; let IsAtomicOrderingReleaseOrStronger = 1; } // An atomic store operation that doesn't actually need to be atomic on AArch64. class relaxed_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { + : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> { let IsAtomic = 1; let IsAtomicOrderingReleaseOrStronger = 0; } Index: llvm/lib/Target/AMDGPU/AMDGPUInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -516,19 +516,18 @@ def store_hi16_#as : StoreHi16 ; def truncstorei8_hi16_#as : StoreHi16; def truncstorei16_hi16_#as : StoreHi16; - } // End let IsStore = 1, AddressSpaces = ... let IsAtomic = 1, AddressSpaces = !cast("StoreAddress_"#as).AddrSpaces in { -def atomic_store_8_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_8 node:$ptr, node:$val)>; -def atomic_store_16_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_16 node:$ptr, node:$val)>; -def atomic_store_32_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_32 node:$ptr, node:$val)>; -def atomic_store_64_#as : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_64 node:$ptr, node:$val)>; -} +def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_8 node:$val, node:$ptr)>; +def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_16 node:$val, node:$ptr)>; +def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_32 node:$val, node:$ptr)>; +def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_64 node:$val, node:$ptr)>; +} // End let IsAtomic = 1, AddressSpaces = ... } // End foreach as // TODO: Add GISelPredicateCode for the ret and noret PatFrags once Index: llvm/lib/Target/AMDGPU/BUFInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/BUFInstructions.td +++ llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1782,14 +1782,13 @@ multiclass MUBUFStore_Atomic_Pattern { - // Store follows atomic op convention so address is first def : GCNPat < - (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), + (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset)), (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) >; def : GCNPat < - (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), + (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset)), (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) >; } Index: llvm/lib/Target/AMDGPU/DSInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/DSInstructions.td +++ llvm/lib/Target/AMDGPU/DSInstructions.td @@ -758,23 +758,6 @@ } } -// Irritatingly, atomic_store reverses the order of operands from a -// normal store. -class DSAtomicWritePat : GCNPat < - (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value), - (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 0)) ->; - -multiclass DSAtomicWritePat_mc { - let OtherPredicates = [LDSRequiresM0Init] in { - def : DSAtomicWritePat(frag#"_m0")>; - } - - let OtherPredicates = [NotLDSRequiresM0Init] in { - def : DSAtomicWritePat(!cast(inst)#"_gfx9"), vt, !cast(frag)>; - } -} - defm : DSWritePat_mc ; defm : DSWritePat_mc ; defm : DSWritePat_mc ; @@ -784,12 +767,12 @@ defm : DSWritePat_mc ; } -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; -defm : DSAtomicWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; +defm : DSWritePat_mc ; let OtherPredicates = [D16PreservesUnusedBits] in { def : DSWritePat ; Index: llvm/lib/Target/AMDGPU/FLATInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/FLATInstructions.td +++ llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -997,21 +997,6 @@ (inst $vaddr, getVregSrcForVT.ret:$data, $offset) >; -class FlatStoreAtomicPat : GCNPat < - // atomic store follows atomic binop convention so the address comes - // first. - (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), - (inst $vaddr, getVregSrcForVT.ret:$data, $offset) ->; - -class FlatStoreSignedAtomicPat : GCNPat < - // atomic store follows atomic binop convention so the address comes - // first. - (node (GlobalOffset i64:$vaddr, i16:$offset), data_vt:$data), - (inst $vaddr, getVregSrcForVT.ret:$data, $offset) ->; - class FlatAtomicPatNoRtn : GCNPat < (node (FlatOffset i64:$vaddr, i16:$offset), vt:$data), (inst VReg_64:$vaddr, getVregSrcForVT.ret:$data, $offset) @@ -1124,8 +1109,8 @@ def : FlatStorePat ; } -def : FlatStoreAtomicPat ; -def : FlatStoreAtomicPat ; +def : FlatStorePat ; +def : FlatStorePat ; foreach as = [ "flat", "global" ] in { defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>; @@ -1213,17 +1198,6 @@ } } -// Deal with swapped operands for atomic_store vs. regular store -multiclass GlobalFLATAtomicStorePats { - def : FlatStoreSignedAtomicPat { - let AddedComplexity = 10; - } - - def : GlobalAtomicStoreSaddrPat(!cast(inst)#"_SADDR"), node, vt> { - let AddedComplexity = 11; - } -} - multiclass GlobalFLATAtomicPatsRtn { def : FlatSignedAtomicPat (nortn_inst_name#"_RTN"), node, vt, data_vt> { @@ -1369,8 +1343,8 @@ defm : GlobalFLATLoadPats_D16 ; } -defm : GlobalFLATAtomicStorePats ; -defm : GlobalFLATAtomicStorePats ; +defm : GlobalFLATStorePats ; +defm : GlobalFLATStorePats ; defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>; defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>; Index: llvm/lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -586,14 +586,14 @@ } let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces in { -def atomic_store_8_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_8_glue node:$ptr, node:$val)>; -def atomic_store_16_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_16_glue node:$ptr, node:$val)>; -def atomic_store_32_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_32_glue node:$ptr, node:$val)>; -def atomic_store_64_local_m0 : PatFrag<(ops node:$ptr, node:$val), - (atomic_store_64_glue node:$ptr, node:$val)>; +def atomic_store_8_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_8_glue node:$val, node:$ptr)>; +def atomic_store_16_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_16_glue node:$val, node:$ptr)>; +def atomic_store_32_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_32_glue node:$val, node:$ptr)>; +def atomic_store_64_local_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_64_glue node:$val, node:$ptr)>; } // End let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces Index: llvm/lib/Target/ARM/ARMInstrInfo.td =================================================================== --- llvm/lib/Target/ARM/ARMInstrInfo.td +++ llvm/lib/Target/ARM/ARMInstrInfo.td @@ -5320,7 +5320,7 @@ def atomic_load_acquire_32 : acquiring_load; class releasing_store - : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{ + : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr), [{ AtomicOrdering Ordering = cast(N)->getSuccessOrdering(); return isReleaseOrStronger(Ordering); }]>; @@ -6143,15 +6143,15 @@ (LDRrs ldst_so_reg:$src)>; def : ARMPat<(atomic_load_32 addrmode_imm12:$src), (LDRi12 addrmode_imm12:$src)>; -def : ARMPat<(atomic_store_8 ldst_so_reg:$ptr, GPR:$val), +def : ARMPat<(atomic_store_8 GPR:$val, ldst_so_reg:$ptr), (STRBrs GPR:$val, ldst_so_reg:$ptr)>; -def : ARMPat<(atomic_store_8 addrmode_imm12:$ptr, GPR:$val), +def : ARMPat<(atomic_store_8 GPR:$val, addrmode_imm12:$ptr), (STRBi12 GPR:$val, addrmode_imm12:$ptr)>; -def : ARMPat<(atomic_store_16 addrmode3:$ptr, GPR:$val), +def : ARMPat<(atomic_store_16 GPR:$val, addrmode3:$ptr), (STRH GPR:$val, addrmode3:$ptr)>; -def : ARMPat<(atomic_store_32 ldst_so_reg:$ptr, GPR:$val), +def : ARMPat<(atomic_store_32 GPR:$val, ldst_so_reg:$ptr), (STRrs GPR:$val, ldst_so_reg:$ptr)>; -def : ARMPat<(atomic_store_32 addrmode_imm12:$ptr, GPR:$val), +def : ARMPat<(atomic_store_32 GPR:$val, addrmode_imm12:$ptr), (STRi12 GPR:$val, addrmode_imm12:$ptr)>; Index: llvm/lib/Target/ARM/ARMInstrThumb.td =================================================================== --- llvm/lib/Target/ARM/ARMInstrThumb.td +++ llvm/lib/Target/ARM/ARMInstrThumb.td @@ -1697,17 +1697,17 @@ (tLDRi t_addrmode_is4:$src)>; def : T1Pat<(atomic_load_32 t_addrmode_rr:$src), (tLDRr t_addrmode_rr:$src)>; -def : T1Pat<(atomic_store_8 t_addrmode_is1:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_8 tGPR:$val, t_addrmode_is1:$ptr), (tSTRBi tGPR:$val, t_addrmode_is1:$ptr)>; -def : T1Pat<(atomic_store_8 t_addrmode_rr:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_8 tGPR:$val, t_addrmode_rr:$ptr), (tSTRBr tGPR:$val, t_addrmode_rr:$ptr)>; -def : T1Pat<(atomic_store_16 t_addrmode_is2:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_16 tGPR:$val, t_addrmode_is2:$ptr), (tSTRHi tGPR:$val, t_addrmode_is2:$ptr)>; -def : T1Pat<(atomic_store_16 t_addrmode_rr:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_16 tGPR:$val, t_addrmode_rr:$ptr), (tSTRHr tGPR:$val, t_addrmode_rr:$ptr)>; -def : T1Pat<(atomic_store_32 t_addrmode_is4:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_32 tGPR:$val, t_addrmode_is4:$ptr), (tSTRi tGPR:$val, t_addrmode_is4:$ptr)>; -def : T1Pat<(atomic_store_32 t_addrmode_rr:$ptr, tGPR:$val), +def : T1Pat<(atomic_store_32 tGPR:$val, t_addrmode_rr:$ptr), (tSTRr tGPR:$val, t_addrmode_rr:$ptr)>; // Large immediate handling. Index: llvm/lib/Target/ARM/ARMInstrThumb2.td =================================================================== --- llvm/lib/Target/ARM/ARMInstrThumb2.td +++ llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -4803,23 +4803,23 @@ (t2LDRi8 t2addrmode_negimm8:$addr)>; def : T2Pat<(atomic_load_32 t2addrmode_so_reg:$addr), (t2LDRs t2addrmode_so_reg:$addr)>; -def : T2Pat<(atomic_store_8 t2addrmode_imm12:$addr, GPR:$val), +def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_imm12:$addr), (t2STRBi12 GPR:$val, t2addrmode_imm12:$addr)>; -def : T2Pat<(atomic_store_8 t2addrmode_negimm8:$addr, GPR:$val), +def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_negimm8:$addr), (t2STRBi8 GPR:$val, t2addrmode_negimm8:$addr)>; -def : T2Pat<(atomic_store_8 t2addrmode_so_reg:$addr, GPR:$val), +def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_so_reg:$addr), (t2STRBs GPR:$val, t2addrmode_so_reg:$addr)>; -def : T2Pat<(atomic_store_16 t2addrmode_imm12:$addr, GPR:$val), +def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_imm12:$addr), (t2STRHi12 GPR:$val, t2addrmode_imm12:$addr)>; -def : T2Pat<(atomic_store_16 t2addrmode_negimm8:$addr, GPR:$val), +def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_negimm8:$addr), (t2STRHi8 GPR:$val, t2addrmode_negimm8:$addr)>; -def : T2Pat<(atomic_store_16 t2addrmode_so_reg:$addr, GPR:$val), +def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_so_reg:$addr), (t2STRHs GPR:$val, t2addrmode_so_reg:$addr)>; -def : T2Pat<(atomic_store_32 t2addrmode_imm12:$addr, GPR:$val), +def : T2Pat<(atomic_store_32 GPR:$val,t2addrmode_imm12:$addr), (t2STRi12 GPR:$val, t2addrmode_imm12:$addr)>; -def : T2Pat<(atomic_store_32 t2addrmode_negimm8:$addr, GPR:$val), +def : T2Pat<(atomic_store_32 GPR:$val, t2addrmode_negimm8:$addr), (t2STRi8 GPR:$val, t2addrmode_negimm8:$addr)>; -def : T2Pat<(atomic_store_32 t2addrmode_so_reg:$addr, GPR:$val), +def : T2Pat<(atomic_store_32 GPR:$val, t2addrmode_so_reg:$addr), (t2STRs GPR:$val, t2addrmode_so_reg:$addr)>; let AddedComplexity = 8, Predicates = [IsThumb, HasAcquireRelease, HasV7Clrex] in { Index: llvm/lib/Target/AVR/AVRInstrInfo.td =================================================================== --- llvm/lib/Target/AVR/AVRInstrInfo.td +++ llvm/lib/Target/AVR/AVRInstrInfo.td @@ -1442,9 +1442,7 @@ (ins PTRRC : $rd, DRC : $rr), - "atomic_op", [(Op i16 - : $rd, DRC - : $rr)]>; + "atomic_op", [(Op DRC:$rr, i16:$rd)]>; class AtomicLoadOp : Pseudo<(outs DRC:$rd), Index: llvm/lib/Target/Hexagon/HexagonPatterns.td =================================================================== --- llvm/lib/Target/Hexagon/HexagonPatterns.td +++ llvm/lib/Target/Hexagon/HexagonPatterns.td @@ -2447,19 +2447,6 @@ : Pat<(Store Value:$val, Addr:$addr), (MI Addr:$addr, (ValueMod Value:$val))>; -// Regular stores in the DAG have two operands: value and address. -// Atomic stores also have two, but they are reversed: address, value. -// To use atomic stores with the patterns, they need to have their operands -// swapped. This relies on the knowledge that the F.Fragment uses names -// "ptr" and "val". -class AtomSt - : PatFrag<(ops node:$val, node:$ptr), !head(F.Fragments), F.PredicateCode, - F.OperandTransform> { - let IsAtomic = F.IsAtomic; - let MemoryVT = F.MemoryVT; -} - - def IMM_BYTE : SDNodeXForm; def: Storea_pat; def: Storea_pat; - def: Storea_pat, I32, addrgp, S2_storerbgp>; - def: Storea_pat, I32, addrgp, S2_storerhgp>; - def: Storea_pat, I32, addrgp, S2_storerigp>; - def: Storea_pat, V4I8, addrgp, S2_storerigp>; - def: Storea_pat, V2I16, addrgp, S2_storerigp>; - def: Storea_pat, I64, addrgp, S2_storerdgp>; - def: Storea_pat, V8I8, addrgp, S2_storerdgp>; - def: Storea_pat, V4I16, addrgp, S2_storerdgp>; - def: Storea_pat, V2I32, addrgp, S2_storerdgp>; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; def: Stoream_pat; def: Stoream_pat; @@ -2609,15 +2596,15 @@ def: Storea_pat; def: Storea_pat; def: Storea_pat; - def: Storea_pat, I32, anyimm0, PS_storerbabs>; - def: Storea_pat, I32, anyimm1, PS_storerhabs>; - def: Storea_pat, I32, anyimm2, PS_storeriabs>; - def: Storea_pat, V4I8, anyimm2, PS_storeriabs>; - def: Storea_pat, V2I16, anyimm2, PS_storeriabs>; - def: Storea_pat, I64, anyimm3, PS_storerdabs>; - def: Storea_pat, V8I8, anyimm3, PS_storerdabs>; - def: Storea_pat, V4I16, anyimm3, PS_storerdabs>; - def: Storea_pat, V2I32, anyimm3, PS_storerdabs>; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; + def: Storea_pat; def: Stoream_pat; def: Stoream_pat; @@ -2772,15 +2759,15 @@ defm: Storexim_pat; defm: Storexim_pat; - defm: Storexi_pat, I32, anyimm0, S2_storerb_io>; - defm: Storexi_pat, I32, anyimm1, S2_storerh_io>; - defm: Storexi_pat, I32, anyimm2, S2_storeri_io>; - defm: Storexi_pat, V4I8, anyimm2, S2_storeri_io>; - defm: Storexi_pat, V2I16, anyimm2, S2_storeri_io>; - defm: Storexi_pat, I64, anyimm3, S2_storerd_io>; - defm: Storexi_pat, V8I8, anyimm3, S2_storerd_io>; - defm: Storexi_pat, V4I16, anyimm3, S2_storerd_io>; - defm: Storexi_pat, V2I32, anyimm3, S2_storerd_io>; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; + defm: Storexi_pat; } // Reg+Reg @@ -2831,15 +2818,15 @@ def: Storexim_base_pat; def: Storexim_base_pat; - def: Storexi_base_pat, I32, S2_storerb_io>; - def: Storexi_base_pat, I32, S2_storerh_io>; - def: Storexi_base_pat, I32, S2_storeri_io>; - def: Storexi_base_pat, V4I8, S2_storeri_io>; - def: Storexi_base_pat, V2I16, S2_storeri_io>; - def: Storexi_base_pat, I64, S2_storerd_io>; - def: Storexi_base_pat, V8I8, S2_storerd_io>; - def: Storexi_base_pat, V4I16, S2_storerd_io>; - def: Storexi_base_pat, V2I32, S2_storerd_io>; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; + def: Storexi_base_pat; } Index: llvm/lib/Target/Mips/Mips64InstrInfo.td =================================================================== --- llvm/lib/Target/Mips/Mips64InstrInfo.td +++ llvm/lib/Target/Mips/Mips64InstrInfo.td @@ -894,13 +894,13 @@ def : MipsPat<(atomic_load_64 addr:$a), (LD addr:$a)>, ISA_MIPS3, GPR_64; // Atomic store patterns. -def : MipsPat<(atomic_store_8 addr:$a, GPR64:$v), (SB64 GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_8 GPR64:$v, addr:$a), (SB64 GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; -def : MipsPat<(atomic_store_16 addr:$a, GPR64:$v), (SH64 GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_16 GPR64:$v, addr:$a), (SH64 GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; -def : MipsPat<(atomic_store_32 addr:$a, GPR64:$v), (SW64 GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_32 GPR64:$v, addr:$a), (SW64 GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; -def : MipsPat<(atomic_store_64 addr:$a, GPR64:$v), (SD GPR64:$v, addr:$a)>, +def : MipsPat<(atomic_store_64 GPR64:$v, addr:$a), (SD GPR64:$v, addr:$a)>, ISA_MIPS3, GPR_64; // Patterns used for matching away redundant sign extensions. Index: llvm/lib/Target/Mips/MipsInstrInfo.td =================================================================== --- llvm/lib/Target/Mips/MipsInstrInfo.td +++ llvm/lib/Target/Mips/MipsInstrInfo.td @@ -3353,11 +3353,11 @@ def : MipsPat<(atomic_load_32 addr:$a), (LW addr:$a)>, ISA_MIPS1; // Atomic store patterns. - def : MipsPat<(atomic_store_8 addr:$a, GPR32:$v), (SB GPR32:$v, addr:$a)>, + def : MipsPat<(atomic_store_8 GPR32:$v, addr:$a), (SB GPR32:$v, addr:$a)>, ISA_MIPS1; - def : MipsPat<(atomic_store_16 addr:$a, GPR32:$v), (SH GPR32:$v, addr:$a)>, + def : MipsPat<(atomic_store_16 GPR32:$v, addr:$a), (SH GPR32:$v, addr:$a)>, ISA_MIPS1; - def : MipsPat<(atomic_store_32 addr:$a, GPR32:$v), (SW GPR32:$v, addr:$a)>, + def : MipsPat<(atomic_store_32 GPR32:$v, addr:$a), (SW GPR32:$v, addr:$a)>, ISA_MIPS1; } Index: llvm/lib/Target/PowerPC/PPCISelLowering.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -10749,14 +10749,14 @@ SmallVector Ops{ N->getOperand(0), DAG.getConstant(Intrinsic::ppc_atomic_store_i128, dl, MVT::i32)}; - SDValue Val = N->getOperand(2); + SDValue Val = N->getOperand(1); SDValue ValLo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, Val); SDValue ValHi = DAG.getNode(ISD::SRL, dl, MVT::i128, Val, DAG.getConstant(64, dl, MVT::i32)); ValHi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, ValHi); Ops.push_back(ValLo); Ops.push_back(ValHi); - Ops.push_back(N->getOperand(1)); + Ops.push_back(N->getOperand(2)); return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, Tys, Ops, MemVT, N->getMemOperand()); } Index: llvm/lib/Target/PowerPC/PPCInstr64Bit.td =================================================================== --- llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -1885,8 +1885,8 @@ def : Pat<(atomic_load_64 DSForm:$src), (LD memrix:$src)>; def : Pat<(atomic_load_64 XForm:$src), (LDX memrr:$src)>; -def : Pat<(atomic_store_64 DSForm:$ptr, i64:$val), (STD g8rc:$val, memrix:$ptr)>; -def : Pat<(atomic_store_64 XForm:$ptr, i64:$val), (STDX g8rc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_64 i64:$val, DSForm:$ptr), (STD g8rc:$val, memrix:$ptr)>; +def : Pat<(atomic_store_64 i64:$val, XForm:$ptr), (STDX g8rc:$val, memrr:$ptr)>; let Predicates = [IsISA3_0, In64BitMode] in { def : Pat<(i64 (int_ppc_cmpeqb g8rc:$a, g8rc:$b)), Index: llvm/lib/Target/PowerPC/PPCInstrInfo.td =================================================================== --- llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -5345,12 +5345,12 @@ def : Pat<(atomic_load_32 XForm:$src), (LWZX memrr:$src)>; // Atomic stores -def : Pat<(atomic_store_8 DForm:$ptr, i32:$val), (STB gprc:$val, memri:$ptr)>; -def : Pat<(atomic_store_16 DForm:$ptr, i32:$val), (STH gprc:$val, memri:$ptr)>; -def : Pat<(atomic_store_32 DForm:$ptr, i32:$val), (STW gprc:$val, memri:$ptr)>; -def : Pat<(atomic_store_8 XForm:$ptr, i32:$val), (STBX gprc:$val, memrr:$ptr)>; -def : Pat<(atomic_store_16 XForm:$ptr, i32:$val), (STHX gprc:$val, memrr:$ptr)>; -def : Pat<(atomic_store_32 XForm:$ptr, i32:$val), (STWX gprc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_8 i32:$val, DForm:$ptr), (STB gprc:$val, memri:$ptr)>; +def : Pat<(atomic_store_16 i32:$val, DForm:$ptr), (STH gprc:$val, memri:$ptr)>; +def : Pat<(atomic_store_32 i32:$val, DForm:$ptr), (STW gprc:$val, memri:$ptr)>; +def : Pat<(atomic_store_8 i32:$val, XForm:$ptr), (STBX gprc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_16 i32:$val, XForm:$ptr), (STHX gprc:$val, memrr:$ptr)>; +def : Pat<(atomic_store_32 i32:$val, XForm:$ptr), (STWX gprc:$val, memrr:$ptr)>; let Predicates = [IsISA3_0] in { Index: llvm/lib/Target/PowerPC/PPCInstrP10.td =================================================================== --- llvm/lib/Target/PowerPC/PPCInstrP10.td +++ llvm/lib/Target/PowerPC/PPCInstrP10.td @@ -1247,19 +1247,19 @@ (PLDpc $ga, 0)>; // Atomic Store - def : Pat<(atomic_store_8 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS), + def : Pat<(atomic_store_8 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTBpc $RS, $ga, 0)>; - def : Pat<(atomic_store_16 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS), + def : Pat<(atomic_store_16 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTHpc $RS, $ga, 0)>; - def : Pat<(atomic_store_32 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS), + def : Pat<(atomic_store_32 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTWpc $RS, $ga, 0)>; - def : Pat<(atomic_store_8 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_8 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTB8pc $RS, $ga, 0)>; - def : Pat<(atomic_store_16 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_16 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTH8pc $RS, $ga, 0)>; - def : Pat<(atomic_store_32 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_32 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTW8pc $RS, $ga, 0)>; - def : Pat<(atomic_store_64 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS), + def : Pat<(atomic_store_64 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)), (PSTDpc $RS, $ga, 0)>; // Special Cases For PPCstore_scal_int_from_vsr @@ -2235,10 +2235,10 @@ def : Pat<(atomic_load_64 PDForm:$src), (PLD memri34:$src)>; // Atomic Store - def : Pat<(atomic_store_8 PDForm:$dst, i32:$RS), (PSTB $RS, memri34:$dst)>; - def : Pat<(atomic_store_16 PDForm:$dst, i32:$RS), (PSTH $RS, memri34:$dst)>; - def : Pat<(atomic_store_32 PDForm:$dst, i32:$RS), (PSTW $RS, memri34:$dst)>; - def : Pat<(atomic_store_64 PDForm:$dst, i64:$RS), (PSTD $RS, memri34:$dst)>; + def : Pat<(atomic_store_8 i32:$RS, PDForm:$dst), (PSTB $RS, memri34:$dst)>; + def : Pat<(atomic_store_16 i32:$RS, PDForm:$dst), (PSTH $RS, memri34:$dst)>; + def : Pat<(atomic_store_32 i32:$RS, PDForm:$dst), (PSTW $RS, memri34:$dst)>; + def : Pat<(atomic_store_64 i64:$RS, PDForm:$dst), (PSTD $RS, memri34:$dst)>; // Prefixed fpext to v2f64 def : Pat<(v4f32 (PPCldvsxlh PDForm:$src)), Index: llvm/lib/Target/RISCV/RISCVInstrInfoA.td =================================================================== --- llvm/lib/Target/RISCV/RISCVInstrInfoA.td +++ llvm/lib/Target/RISCV/RISCVInstrInfoA.td @@ -45,11 +45,11 @@ multiclass AtomicStPat { - def : Pat<(StoreOp BaseAddr:$rs1, (vt StTy:$rs2)), + def : Pat<(StoreOp (vt StTy:$rs2), BaseAddr:$rs1), (Inst StTy:$rs2, BaseAddr:$rs1, 0)>; - def : Pat<(StoreOp (add BaseAddr:$rs1, simm12:$imm12), (vt StTy:$rs2)), + def : Pat<(StoreOp (vt StTy:$rs2), (add BaseAddr:$rs1, simm12:$imm12)), (Inst StTy:$rs2, BaseAddr:$rs1, simm12:$imm12)>; - def : Pat<(StoreOp (IsOrAdd AddrFI:$rs1, simm12:$imm12), (vt StTy:$rs2)), + def : Pat<(StoreOp (vt StTy:$rs2), (IsOrAdd AddrFI:$rs1, simm12:$imm12)), (Inst StTy:$rs2, AddrFI:$rs1, simm12:$imm12)>; } Index: llvm/lib/Target/Sparc/SparcInstr64Bit.td =================================================================== --- llvm/lib/Target/Sparc/SparcInstr64Bit.td +++ llvm/lib/Target/Sparc/SparcInstr64Bit.td @@ -502,8 +502,8 @@ def : Pat<(i64 (atomic_load_64 ADDRri:$src)), (LDXri ADDRri:$src)>; // atomic_store_64 val, addr -> store val, addr -def : Pat<(atomic_store_64 ADDRrr:$dst, i64:$val), (STXrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_64 ADDRri:$dst, i64:$val), (STXri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_64 i64:$val, ADDRrr:$dst), (STXrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_64 i64:$val, ADDRri:$dst), (STXri ADDRri:$dst, $val)>; } // Predicates = [Is64Bit] Index: llvm/lib/Target/Sparc/SparcInstrInfo.td =================================================================== --- llvm/lib/Target/Sparc/SparcInstrInfo.td +++ llvm/lib/Target/Sparc/SparcInstrInfo.td @@ -1759,12 +1759,12 @@ def : Pat<(i32 (atomic_load_32 ADDRri:$src)), (LDri ADDRri:$src)>; // atomic_store val, addr -> store val, addr -def : Pat<(atomic_store_8 ADDRrr:$dst, i32:$val), (STBrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_8 ADDRri:$dst, i32:$val), (STBri ADDRri:$dst, $val)>; -def : Pat<(atomic_store_16 ADDRrr:$dst, i32:$val), (STHrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_16 ADDRri:$dst, i32:$val), (STHri ADDRri:$dst, $val)>; -def : Pat<(atomic_store_32 ADDRrr:$dst, i32:$val), (STrr ADDRrr:$dst, $val)>; -def : Pat<(atomic_store_32 ADDRri:$dst, i32:$val), (STri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_8 i32:$val, ADDRrr:$dst), (STBrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_8 i32:$val, ADDRri:$dst), (STBri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_16 i32:$val, ADDRrr:$dst), (STHrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_16 i32:$val, ADDRri:$dst), (STHri ADDRri:$dst, $val)>; +def : Pat<(atomic_store_32 i32:$val, ADDRrr:$dst), (STrr ADDRrr:$dst, $val)>; +def : Pat<(atomic_store_32 i32:$val, ADDRri:$dst), (STri ADDRri:$dst, $val)>; // extract_vector def : Pat<(extractelt (v2i32 IntPair:$Rn), 0), Index: llvm/lib/Target/SystemZ/SystemZISelLowering.cpp =================================================================== --- llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -5748,8 +5748,8 @@ SDLoc DL(N); SDVTList Tys = DAG.getVTList(MVT::Other); SDValue Ops[] = { N->getOperand(0), - lowerI128ToGR128(DAG, N->getOperand(2)), - N->getOperand(1) }; + lowerI128ToGR128(DAG, N->getOperand(1)), + N->getOperand(2) }; MachineMemOperand *MMO = cast(N)->getMemOperand(); SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128, DL, Tys, Ops, MVT::i128, MMO); Index: llvm/lib/Target/VE/VEInstrInfo.td =================================================================== --- llvm/lib/Target/VE/VEInstrInfo.td +++ llvm/lib/Target/VE/VEInstrInfo.td @@ -1874,10 +1874,10 @@ multiclass ATMSTm { - def : Pat<(from ADDRrri:$addr, ty:$src), (torri MEMrri:$addr, $src)>; - def : Pat<(from ADDRrii:$addr, ty:$src), (torii MEMrii:$addr, $src)>; - def : Pat<(from ADDRzri:$addr, ty:$src), (tozri MEMzri:$addr, $src)>; - def : Pat<(from ADDRzii:$addr, ty:$src), (tozii MEMzii:$addr, $src)>; + def : Pat<(from ty:$src, ADDRrri:$addr), (torri MEMrri:$addr, $src)>; + def : Pat<(from ty:$src, ADDRrii:$addr), (torii MEMrii:$addr, $src)>; + def : Pat<(from ty:$src, ADDRzri:$addr), (tozri MEMzri:$addr, $src)>; + def : Pat<(from ty:$src, ADDRzii:$addr), (tozii MEMzii:$addr, $src)>; } defm : ATMSTm; defm : ATMSTm; @@ -1890,13 +1890,13 @@ RM torii, RM tozri, RM tozii> { - def : Pat<(from ADDRrri:$addr, (i32 (trunc i64:$src))), + def : Pat<(from (i32 (trunc i64:$src)), ADDRrri:$addr), (torri MEMrri:$addr, (EXTRACT_SUBREG $src, sub_i32))>; - def : Pat<(from ADDRrii:$addr, (i32 (trunc i64:$src))), + def : Pat<(from (i32 (trunc i64:$src)), ADDRrii:$addr), (torii MEMrii:$addr, (EXTRACT_SUBREG $src, sub_i32))>; - def : Pat<(from ADDRzri:$addr, (i32 (trunc i64:$src))), + def : Pat<(from (i32 (trunc i64:$src)), ADDRzri:$addr), (tozri MEMzri:$addr, (EXTRACT_SUBREG $src, sub_i32))>; - def : Pat<(from ADDRzii:$addr, (i32 (trunc i64:$src))), + def : Pat<(from (i32 (trunc i64:$src)), ADDRzii:$addr), (tozii MEMzii:$addr, (EXTRACT_SUBREG $src, sub_i32))>; } defm : TRATMSTm; Index: llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td =================================================================== --- llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td +++ llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td @@ -352,10 +352,10 @@ // Select stores with no constant offset. multiclass AStorePatNoOffset { - def : Pat<(kind I32:$addr, ty:$val), + def : Pat<(kind ty:$val, I32:$addr), (!cast(inst#_A32) 0, 0, I32:$addr, ty:$val)>, Requires<[HasAddr32, HasAtomics]>; - def : Pat<(kind I64:$addr, ty:$val), + def : Pat<(kind ty:$val, I64:$addr), (!cast(inst#_A64) 0, 0, I64:$addr, ty:$val)>, Requires<[HasAddr64, HasAtomics]>; } @@ -367,10 +367,10 @@ // Pattern with address + immediate offset multiclass AStorePatImmOff { - def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val), + def : Pat<(kind ty:$val, (operand I32:$addr, imm:$off)), (!cast(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, Requires<[HasAddr32, HasAtomics]>; - def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val), + def : Pat<(kind ty:$val, (operand I64:$addr, imm:$off)), (!cast(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, Requires<[HasAddr64, HasAtomics]>; } @@ -379,10 +379,10 @@ // Select stores with just a constant offset. multiclass AStorePatOffsetOnly { - def : Pat<(kind imm:$off, ty:$val), + def : Pat<(kind ty:$val, imm:$off), (!cast(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, Requires<[HasAddr32, HasAtomics]>; - def : Pat<(kind imm:$off, ty:$val), + def : Pat<(kind ty:$val, imm:$off), (!cast(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, Requires<[HasAddr64, HasAtomics]>; } @@ -390,10 +390,10 @@ defm : AStorePatOffsetOnly; multiclass AStorePatGlobalAddrOffOnly { - def : Pat<(kind (WebAssemblyWrapper tglobaladdr:$off), ty:$val), + def : Pat<(kind ty:$val, (WebAssemblyWrapper tglobaladdr:$off)), (!cast(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, Requires<[HasAddr32, HasAtomics, IsNotPIC]>; - def : Pat<(kind (WebAssemblyWrapper tglobaladdr:$off), ty:$val), + def : Pat<(kind ty:$val, (WebAssemblyWrapper tglobaladdr:$off)), (!cast(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, Requires<[HasAddr64, HasAtomics, IsNotPIC]>; } @@ -414,8 +414,8 @@ // instructions, we just need to match bare atomic stores. On the other hand, // truncating stores from i64 values are once truncated to i32 first. class trunc_astore_64 : - PatFrag<(ops node:$addr, node:$val), - (kind node:$addr, (i32 (trunc (i64 node:$val))))>; + PatFrag<(ops node:$val, node:$addr), + (kind (i32 (trunc (i64 node:$val))), node:$addr)>; def trunc_astore_8_64 : trunc_astore_64; def trunc_astore_16_64 : trunc_astore_64; def trunc_astore_32_64 : trunc_astore_64; Index: llvm/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/lib/Target/X86/X86ISelLowering.cpp +++ llvm/lib/Target/X86/X86ISelLowering.cpp @@ -31302,7 +31302,7 @@ SDValue Chain; if (Subtarget.hasSSE1()) { SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, - Node->getOperand(2)); + Node->getVal()); MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32; SclToVec = DAG.getBitcast(StVT, SclToVec); SDVTList Tys = DAG.getVTList(MVT::Other); @@ -31317,7 +31317,7 @@ MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); Chain = - DAG.getStore(Node->getChain(), dl, Node->getOperand(2), StackPtr, + DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr, MPI, MaybeAlign(), MachineMemOperand::MOStore); SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); SDValue LdOps[] = {Chain, StackPtr}; @@ -31350,7 +31350,7 @@ SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(), Node->getOperand(0), - Node->getOperand(1), Node->getOperand(2), + Node->getOperand(2), Node->getOperand(1), Node->getMemOperand()); return Swap.getValue(1); } Index: llvm/lib/Target/X86/X86InstrCompiler.td =================================================================== --- llvm/lib/Target/X86/X86InstrCompiler.td +++ llvm/lib/Target/X86/X86InstrCompiler.td @@ -1022,30 +1022,28 @@ * (see below the RELEASE_MOV* / ACQUIRE_MOV* pseudo-instructions) */ multiclass RELEASE_BINOP_MI { - def : Pat<(atomic_store_8 addr:$dst, - (op (atomic_load_8 addr:$dst), (i8 imm:$src))), + def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 imm:$src)), + addr:$dst), (!cast(Name#"8mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_16 addr:$dst, - (op (atomic_load_16 addr:$dst), (i16 imm:$src))), + def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 imm:$src)), + addr:$dst), (!cast(Name#"16mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_32 addr:$dst, - (op (atomic_load_32 addr:$dst), (i32 imm:$src))), + def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 imm:$src)), + addr:$dst), (!cast(Name#"32mi") addr:$dst, imm:$src)>; - def : Pat<(atomic_store_64 addr:$dst, - (op (atomic_load_64 addr:$dst), (i64immSExt32:$src))), + def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64immSExt32:$src)), + addr:$dst), (!cast(Name#"64mi32") addr:$dst, (i64immSExt32:$src))>; - - def : Pat<(atomic_store_8 addr:$dst, - (op (atomic_load_8 addr:$dst), (i8 GR8:$src))), + def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 GR8:$src)), addr:$dst), (!cast(Name#"8mr") addr:$dst, GR8:$src)>; - def : Pat<(atomic_store_16 addr:$dst, - (op (atomic_load_16 addr:$dst), (i16 GR16:$src))), + def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 GR16:$src)), + addr:$dst), (!cast(Name#"16mr") addr:$dst, GR16:$src)>; - def : Pat<(atomic_store_32 addr:$dst, - (op (atomic_load_32 addr:$dst), (i32 GR32:$src))), + def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 GR32:$src)), + addr:$dst), (!cast(Name#"32mr") addr:$dst, GR32:$src)>; - def : Pat<(atomic_store_64 addr:$dst, - (op (atomic_load_64 addr:$dst), (i64 GR64:$src))), + def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64 GR64:$src)), + addr:$dst), (!cast(Name#"64mr") addr:$dst, GR64:$src)>; } defm : RELEASE_BINOP_MI<"ADD", add>; @@ -1082,13 +1080,13 @@ multiclass RELEASE_UNOP { - def : Pat<(atomic_store_8 addr:$dst, dag8), + def : Pat<(atomic_store_8 dag8, addr:$dst), (!cast(Name#8m) addr:$dst)>; - def : Pat<(atomic_store_16 addr:$dst, dag16), + def : Pat<(atomic_store_16 dag16, addr:$dst), (!cast(Name#16m) addr:$dst)>; - def : Pat<(atomic_store_32 addr:$dst, dag32), + def : Pat<(atomic_store_32 dag32, addr:$dst), (!cast(Name#32m) addr:$dst)>; - def : Pat<(atomic_store_64 addr:$dst, dag64), + def : Pat<(atomic_store_64 dag64, addr:$dst), (!cast(Name#64m) addr:$dst)>; } @@ -1116,22 +1114,22 @@ (not (i32 (atomic_load_32 addr:$dst))), (not (i64 (atomic_load_64 addr:$dst)))>; -def : Pat<(atomic_store_8 addr:$dst, (i8 imm:$src)), +def : Pat<(atomic_store_8 (i8 imm:$src), addr:$dst), (MOV8mi addr:$dst, imm:$src)>; -def : Pat<(atomic_store_16 addr:$dst, (i16 imm:$src)), +def : Pat<(atomic_store_16 (i16 imm:$src), addr:$dst), (MOV16mi addr:$dst, imm:$src)>; -def : Pat<(atomic_store_32 addr:$dst, (i32 imm:$src)), +def : Pat<(atomic_store_32 (i32 imm:$src), addr:$dst), (MOV32mi addr:$dst, imm:$src)>; -def : Pat<(atomic_store_64 addr:$dst, (i64immSExt32:$src)), +def : Pat<(atomic_store_64 (i64immSExt32:$src), addr:$dst), (MOV64mi32 addr:$dst, i64immSExt32:$src)>; -def : Pat<(atomic_store_8 addr:$dst, GR8:$src), +def : Pat<(atomic_store_8 GR8:$src, addr:$dst), (MOV8mr addr:$dst, GR8:$src)>; -def : Pat<(atomic_store_16 addr:$dst, GR16:$src), +def : Pat<(atomic_store_16 GR16:$src, addr:$dst), (MOV16mr addr:$dst, GR16:$src)>; -def : Pat<(atomic_store_32 addr:$dst, GR32:$src), +def : Pat<(atomic_store_32 GR32:$src, addr:$dst), (MOV32mr addr:$dst, GR32:$src)>; -def : Pat<(atomic_store_64 addr:$dst, GR64:$src), +def : Pat<(atomic_store_64 GR64:$src, addr:$dst), (MOV64mr addr:$dst, GR64:$src)>; def : Pat<(i8 (atomic_load_8 addr:$src)), (MOV8rm addr:$src)>; @@ -1140,18 +1138,18 @@ def : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>; // Floating point loads/stores. -def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), +def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), (MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>; -def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), +def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), (VMOVSSmr addr:$dst, FR32:$src)>, Requires<[UseAVX]>; -def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), +def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), (VMOVSSZmr addr:$dst, FR32:$src)>, Requires<[HasAVX512]>; -def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), +def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), (MOVSDmr addr:$dst, FR64:$src)>, Requires<[UseSSE2]>; -def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), +def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[UseAVX]>; -def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), +def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst), (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[HasAVX512]>; def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), Index: llvm/test/TableGen/GlobalISelEmitter-atomic_store.td =================================================================== --- llvm/test/TableGen/GlobalISelEmitter-atomic_store.td +++ llvm/test/TableGen/GlobalISelEmitter-atomic_store.td @@ -5,20 +5,17 @@ def ST_ATOM_B32 : I<(outs), (ins GPR32Op:$val, GPR32Op:$ptr), []>; -// Check that the pattern for atomic_store inverts the operands to -// match the order of G_STORE. - // GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE, // GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/1, // GISEL-NEXT: GIM_CheckAtomicOrderingOrStrongerThan, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::Unordered, -// GISEL-NEXT: // MIs[0] ptr -// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0, // GISEL-NEXT: // MIs[0] val // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, -// GISEL-NEXT: // (atomic_store iPTR:{ *:[iPTR] }:$ptr, i32:{ *:[i32] }:$val)<> => (ST_ATOM_B32 GPR32Op:{ *:[i32] }:$val, GPR32Op:{ *:[i32] }:$ptr) +// GISEL-NEXT: // MIs[0] ptr +// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0, +// GISEL-NEXT: // (atomic_store i32:{ *:[i32] }:$val, iPTR:{ *:[iPTR] }:$ptr)<> => (ST_ATOM_B32 GPR32Op:{ *:[i32] }:$val, GPR32Op:{ *:[i32] }:$ptr) // GISEL-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ST_ATOM_B32, def : Pat< // (atomic_store_8 iPTR:$ptr, i32:$val), - (atomic_store_8 iPTR:$ptr, i32:$val), + (atomic_store_8 i32:$val, iPTR:$ptr), (ST_ATOM_B32 GPR32Op:$val, GPR32Op:$ptr) >; Index: llvm/utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- llvm/utils/TableGen/GlobalISelEmitter.cpp +++ llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -4019,11 +4019,9 @@ } } - bool IsAtomic = false; if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) InsnMatcher.addPredicate("NotAtomic"); else if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { - IsAtomic = true; InsnMatcher.addPredicate( "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); } @@ -4077,27 +4075,6 @@ } } - // Hack around an unfortunate mistake in how atomic store (and really - // atomicrmw in general) operands were ordered. A ISD::STORE used the order - // , order. ISD::ATOMIC_STORE used the opposite, - // , . In GlobalISel there's just the one store - // opcode, so we need to swap the operands here to get the right type check. - if (IsAtomic && SrcGIOrNull->TheDef->getName() == "G_STORE") { - assert(NumChildren == 2 && "wrong operands for atomic store"); - - TreePatternNode *PtrChild = Src->getChild(0); - TreePatternNode *ValueChild = Src->getChild(1); - - if (auto Error = importChildMatcher(Rule, InsnMatcher, PtrChild, true, - false, 1, TempOpIdx)) - return std::move(Error); - - if (auto Error = importChildMatcher(Rule, InsnMatcher, ValueChild, false, - false, 0, TempOpIdx)) - return std::move(Error); - return InsnMatcher; - } - // Match the used operands (i.e. the children of the operator). bool IsIntrinsic = SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" ||