Index: include/llvm/CodeGen/MachineFunction.h =================================================================== --- include/llvm/CodeGen/MachineFunction.h +++ include/llvm/CodeGen/MachineFunction.h @@ -557,11 +557,13 @@ /// getMachineMemOperand - Allocate a new MachineMemOperand. /// MachineMemOperands are owned by the MachineFunction and need not be /// explicitly deallocated. - MachineMemOperand *getMachineMemOperand(MachinePointerInfo PtrInfo, - MachineMemOperand::Flags f, - uint64_t s, unsigned base_alignment, - const AAMDNodes &AAInfo = AAMDNodes(), - const MDNode *Ranges = nullptr); + MachineMemOperand *getMachineMemOperand( + MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, + unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), + const MDNode *Ranges = nullptr, + SynchronizationScope SynchScope = CrossThread, + AtomicOrdering Ordering = AtomicOrdering::NotAtomic, + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); /// getMachineMemOperand - Allocate a new MachineMemOperand by copying /// an existing one, adjusting by an offset and using the given size. Index: include/llvm/CodeGen/MachineMemOperand.h =================================================================== --- include/llvm/CodeGen/MachineMemOperand.h +++ include/llvm/CodeGen/MachineMemOperand.h @@ -19,8 +19,10 @@ #include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Value.h" // PointerLikeTypeTraits +#include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/DataTypes.h" namespace llvm { @@ -115,20 +117,52 @@ }; private: + /// Atomic information for this memory operation. + struct MachineAtomicInfo { + /// Synchronization scope for this memory operation. + unsigned SynchScope : 1; // enum SynchronizationScope + /// Atomic ordering requirements for this memory operation. For cmpxchg + /// atomic operations, atomic ordering requirements when store occurs. + unsigned Ordering : 4; // enum AtomicOrdering + /// For cmpxchg atomic operations, atomic ordering requirements when store + /// does not occur. + unsigned FailureOrdering : 4; // enum AtomicOrdering + /// Reserved/unused. + unsigned Reserved : 23; + }; + MachinePointerInfo PtrInfo; uint64_t Size; Flags FlagVals; uint16_t BaseAlignLog2; // log_2(base_alignment) + 1 + MachineAtomicInfo AtomicInfo; AAMDNodes AAInfo; const MDNode *Ranges; + /// Initialize atomic information for this memory operation. + void InitAtomicInfo(SynchronizationScope SynchScope, AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) { + AtomicInfo.SynchScope = static_cast(SynchScope); + assert(getSynchScope() == SynchScope && "Value truncated"); + AtomicInfo.Ordering = static_cast(Ordering); + assert(getOrdering() == Ordering && "Value truncated"); + AtomicInfo.FailureOrdering = static_cast(FailureOrdering); + assert(getFailureOrdering() == FailureOrdering && "Value truncated"); + } + public: /// Construct a MachineMemOperand object with the specified PtrInfo, flags, - /// size, and base alignment. + /// size, and base alignment. For atomic operations the synchronization scope + /// and atomic ordering requirements must also be specified. For cmpxchg + /// atomic operations the atomic ordering requirements when store does not + /// occur must also be specified. MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), - const MDNode *Ranges = nullptr); + const MDNode *Ranges = nullptr, + SynchronizationScope SynchScope = CrossThread, + AtomicOrdering Ordering = AtomicOrdering::NotAtomic, + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); const MachinePointerInfo &getPointerInfo() const { return PtrInfo; } @@ -176,6 +210,28 @@ /// Return the range tag for the memory reference. const MDNode *getRanges() const { return Ranges; } + /// Return the synchronization scope for this memory operation. + SynchronizationScope getSynchScope() const { + return static_cast(AtomicInfo.SynchScope); + } + + /// Return the atomic ordering requirements for this memory operation. + AtomicOrdering getOrdering() const { + return static_cast(AtomicInfo.Ordering); + } + + /// For cmpxchg atomic operations, return the atomic ordering requirements + /// when store occurs. + AtomicOrdering getSuccessOrdering() const { + return getOrdering(); + } + + /// For cmpxchg atomic operations, return the atomic ordering requirements + /// when store does not occur. + AtomicOrdering getFailureOrdering() const { + return static_cast(AtomicInfo.FailureOrdering); + } + bool isLoad() const { return FlagVals & MOLoad; } bool isStore() const { return FlagVals & MOStore; } bool isVolatile() const { return FlagVals & MOVolatile; } @@ -183,6 +239,10 @@ bool isDereferenceable() const { return FlagVals & MODereferenceable; } bool isInvariant() const { return FlagVals & MOInvariant; } + /// Returns true if this operation has an atomic ordering requirement of + /// unordered or higher, false otherwise. + bool isAtomic() const { return getOrdering() != AtomicOrdering::NotAtomic; } + /// Returns true if this memory operation doesn't have any ordering /// constraints other than normal aliasing. Volatile and atomic memory /// operations can't be reordered. Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -856,10 +856,7 @@ SynchronizationScope SynchScope); SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, - SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); + SDValue Cmp, SDValue Swp, MachineMemOperand *MMO); /// Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. @@ -868,26 +865,18 @@ unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, - SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope); + SDValue Ptr, SDValue Val, MachineMemOperand *MMO); /// Gets a node for an atomic op, produces result and chain and /// takes 1 operand. SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT, - SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope); + SDValue Chain, SDValue Ptr, MachineMemOperand *MMO); /// Gets a node for an atomic op, produces result and chain and takes N /// operands. SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTList, ArrayRef Ops, - MachineMemOperand *MMO, AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope); - SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, - SDVTList VTList, ArrayRef Ops, - MachineMemOperand *MMO, AtomicOrdering Ordering, - SynchronizationScope SynchScope); + MachineMemOperand *MMO); /// Creates a MemIntrinsicNode that may produce a /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, Index: include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- include/llvm/CodeGen/SelectionDAGNodes.h +++ include/llvm/CodeGen/SelectionDAGNodes.h @@ -424,10 +424,8 @@ uint16_t IsNonTemporal : 1; uint16_t IsDereferenceable : 1; uint16_t IsInvariant : 1; - uint16_t SynchScope : 1; // enum SynchronizationScope - uint16_t Ordering : 4; // enum AtomicOrdering }; - enum { NumMemSDNodeBits = NumSDNodeBits + 9 }; + enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; class LSBaseSDNodeBitfields { friend class LSBaseSDNode; @@ -1108,13 +1106,6 @@ bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } bool isInvariant() const { return MemSDNodeBits.IsInvariant; } - AtomicOrdering getOrdering() const { - return static_cast(MemSDNodeBits.Ordering); - } - SynchronizationScope getSynchScope() const { - return static_cast(MemSDNodeBits.SynchScope); - } - // Returns the offset from the location of the access. int64_t getSrcValueOffset() const { return MMO->getOffset(); } @@ -1124,6 +1115,12 @@ /// Returns the Ranges that describes the dereference. const MDNode *getRanges() const { return MMO->getRanges(); } + /// Return the synchronization scope for this memory operation. + SynchronizationScope getSynchScope() const { return MMO->getSynchScope(); } + + /// Return the atomic ordering requirements for this memory operation. + AtomicOrdering getOrdering() const { return MMO->getOrdering(); } + /// Return the type of the in-memory value. EVT getMemoryVT() const { return MemoryVT; } @@ -1186,40 +1183,24 @@ /// This is an SDNode representing atomic operations. class AtomicSDNode : public MemSDNode { - /// For cmpxchg instructions, the ordering requirements when a store does not - /// occur. - AtomicOrdering FailureOrdering; - - void InitAtomic(AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) { - MemSDNodeBits.Ordering = static_cast(SuccessOrdering); - assert(getOrdering() == SuccessOrdering && "Value truncated"); - MemSDNodeBits.SynchScope = static_cast(SynchScope); - assert(getSynchScope() == SynchScope && "Value truncated"); - this->FailureOrdering = FailureOrdering; - } - public: AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, - EVT MemVT, MachineMemOperand *MMO, - AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) - : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { - InitAtomic(SuccessOrdering, FailureOrdering, SynchScope); - } + EVT MemVT, MachineMemOperand *MMO) + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {} const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } + /// For cmpxchg atomic operations, return the atomic ordering requirements + /// when store occurs. AtomicOrdering getSuccessOrdering() const { - return getOrdering(); + return MMO->getSuccessOrdering(); } - // Not quite enough room in SubclassData for everything, so failure gets its - // own field. + /// For cmpxchg atomic operations, return the atomic ordering requirements + /// when store does not occur. AtomicOrdering getFailureOrdering() const { - return FailureOrdering; + return MMO->getFailureOrdering(); } bool isCompareAndSwap() const { Index: lib/CodeGen/MachineFunction.cpp =================================================================== --- lib/CodeGen/MachineFunction.cpp +++ lib/CodeGen/MachineFunction.cpp @@ -306,9 +306,12 @@ MachineMemOperand *MachineFunction::getMachineMemOperand( MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, - unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges) { + unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges, + SynchronizationScope SynchScope, AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) { return new (Allocator) - MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges); + MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges, + SynchScope, Ordering, FailureOrdering); } MachineMemOperand * @@ -318,13 +321,15 @@ return new (Allocator) MachineMemOperand(MachinePointerInfo(MMO->getValue(), MMO->getOffset()+Offset), - MMO->getFlags(), Size, - MMO->getBaseAlignment()); + MMO->getFlags(), Size, MMO->getBaseAlignment(), + AAMDNodes(), nullptr, MMO->getSynchScope(), + MMO->getOrdering(), MMO->getFailureOrdering()); return new (Allocator) MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(), MMO->getOffset()+Offset), - MMO->getFlags(), Size, - MMO->getBaseAlignment()); + MMO->getFlags(), Size, MMO->getBaseAlignment(), + AAMDNodes(), nullptr, MMO->getSynchScope(), + MMO->getOrdering(), MMO->getFailureOrdering()); } MachineInstr::mmo_iterator @@ -355,7 +360,9 @@ getMachineMemOperand((*I)->getPointerInfo(), (*I)->getFlags() & ~MachineMemOperand::MOStore, (*I)->getSize(), (*I)->getBaseAlignment(), - (*I)->getAAInfo()); + (*I)->getAAInfo(), nullptr, + (*I)->getSynchScope(), (*I)->getOrdering(), + (*I)->getFailureOrdering()); Result[Index] = JustLoad; } ++Index; @@ -387,7 +394,9 @@ getMachineMemOperand((*I)->getPointerInfo(), (*I)->getFlags() & ~MachineMemOperand::MOLoad, (*I)->getSize(), (*I)->getBaseAlignment(), - (*I)->getAAInfo()); + (*I)->getAAInfo(), nullptr, + (*I)->getSynchScope(), (*I)->getOrdering(), + (*I)->getFailureOrdering()); Result[Index] = JustStore; } ++Index; Index: lib/CodeGen/MachineInstr.cpp =================================================================== --- lib/CodeGen/MachineInstr.cpp +++ lib/CodeGen/MachineInstr.cpp @@ -537,7 +537,10 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, uint64_t s, unsigned int a, const AAMDNodes &AAInfo, - const MDNode *Ranges) + const MDNode *Ranges, + SynchronizationScope SynchScope, + AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlignLog2(Log2_32(a) + 1), AAInfo(AAInfo), Ranges(Ranges) { assert((PtrInfo.V.isNull() || PtrInfo.V.is() || @@ -545,6 +548,7 @@ "invalid pointer value"); assert(getBaseAlignment() == a && "Alignment is not a power of 2!"); assert((isLoad() || isStore()) && "Not a load/store!"); + InitAtomicInfo(SynchScope, Ordering, FailureOrdering); } /// Profile - Gather unique data for the object. Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2834,10 +2834,7 @@ SDValue Swap = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs, Node->getOperand(0), Node->getOperand(1), Zero, Zero, - cast(Node)->getMemOperand(), - cast(Node)->getOrdering(), - cast(Node)->getOrdering(), - cast(Node)->getSynchScope()); + cast(Node)->getMemOperand()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); break; @@ -2848,9 +2845,7 @@ cast(Node)->getMemoryVT(), Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), - cast(Node)->getMemOperand(), - cast(Node)->getOrdering(), - cast(Node)->getSynchScope()); + cast(Node)->getMemOperand()); Results.push_back(Swap.getValue(1)); break; } @@ -2862,10 +2857,7 @@ SDValue Res = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs, Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), - Node->getOperand(3), cast(Node)->getMemOperand(), - cast(Node)->getSuccessOrdering(), - cast(Node)->getFailureOrdering(), - cast(Node)->getSynchScope()); + Node->getOperand(3), cast(Node)->getMemOperand()); SDValue ExtRes = Res; SDValue LHS = Res; Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -183,8 +183,7 @@ SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), ResVT, N->getChain(), N->getBasePtr(), - N->getMemOperand(), N->getOrdering(), - N->getSynchScope()); + N->getMemOperand()); // Legalize the chain result - switch anything that used the old chain to // use the new one. ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); @@ -196,8 +195,7 @@ SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, N->getMemOperand(), N->getOrdering(), - N->getSynchScope()); + Op2, N->getMemOperand()); // Legalize the chain result - switch anything that used the old chain to // use the new one. ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); @@ -220,8 +218,7 @@ SDValue Res = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), - N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(), - N->getSynchScope()); + N->getMemOperand()); ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); return Res.getValue(1); @@ -233,8 +230,7 @@ DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); SDValue Res = DAG.getAtomicCmpSwap( N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), - N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(), - N->getFailureOrdering(), N->getSynchScope()); + N->getBasePtr(), Op2, Op3, N->getMemOperand()); // Update the use to N with the newly created Res. for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) ReplaceValueWith(SDValue(N, i), Res.getValue(i)); @@ -992,8 +988,7 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { SDValue Op2 = GetPromotedInteger(N->getOperand(2)); return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), - N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), - N->getOrdering(), N->getSynchScope()); + N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); } SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { @@ -1362,8 +1357,7 @@ SDValue Tmp = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), - AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(), - AN->getSynchScope()); + AN->getMemOperand()); // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine // success simply by comparing the loaded value against the ingoing @@ -2714,10 +2708,7 @@ SDValue Swap = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, cast(N)->getMemoryVT(), VTs, N->getOperand(0), - N->getOperand(1), Zero, Zero, cast(N)->getMemOperand(), - cast(N)->getOrdering(), - cast(N)->getOrdering(), - cast(N)->getSynchScope()); + N->getOperand(1), Zero, Zero, cast(N)->getMemOperand()); ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); @@ -3205,9 +3196,7 @@ cast(N)->getMemoryVT(), N->getOperand(0), N->getOperand(1), N->getOperand(2), - cast(N)->getMemOperand(), - cast(N)->getOrdering(), - cast(N)->getSynchScope()); + cast(N)->getMemOperand()); return Swap.getValue(1); } Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -4797,10 +4797,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTList, ArrayRef Ops, - MachineMemOperand *MMO, - AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) { + MachineMemOperand *MMO) { FoldingSetNodeID ID; ID.AddInteger(MemVT.getRawBits()); AddNodeIDNode(ID, Opcode, VTList, Ops); @@ -4812,8 +4809,7 @@ } auto *N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), - VTList, MemVT, MMO, SuccessOrdering, - FailureOrdering, SynchScope); + VTList, MemVT, MMO); createOperands(N, Ops); CSEMap.InsertNode(N, IP); @@ -4821,14 +4817,6 @@ return SDValue(N, 0); } -SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, - SDVTList VTList, ArrayRef Ops, - MachineMemOperand *MMO, AtomicOrdering Ordering, - SynchronizationScope SynchScope) { - return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering, - Ordering, SynchScope); -} - SDValue SelectionDAG::getAtomicCmpSwap( unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, @@ -4848,26 +4836,23 @@ auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad | MachineMemOperand::MOStore; MachineMemOperand *MMO = - MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); + MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, + AAMDNodes(), nullptr, SynchScope, SuccessOrdering, + FailureOrdering); - return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO, - SuccessOrdering, FailureOrdering, SynchScope); + return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO); } SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, - MachineMemOperand *MMO, - AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SynchronizationScope SynchScope) { + MachineMemOperand *MMO) { assert(Opcode == ISD::ATOMIC_CMP_SWAP || Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; - return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, - SuccessOrdering, FailureOrdering, SynchScope); + return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); } SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, @@ -4893,16 +4878,15 @@ MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, - MemVT.getStoreSize(), Alignment); + MemVT.getStoreSize(), Alignment, AAMDNodes(), + nullptr, SynchScope, Ordering); - return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, - Ordering, SynchScope); + return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO); } SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, - MachineMemOperand *MMO, AtomicOrdering Ordering, - SynchronizationScope SynchScope) { + MachineMemOperand *MMO) { assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || @@ -4922,18 +4906,17 @@ SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : getVTList(VT, MVT::Other); SDValue Ops[] = {Chain, Ptr, Val}; - return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope); + return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); } SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, - MachineMemOperand *MMO, AtomicOrdering Ordering, - SynchronizationScope SynchScope) { + MachineMemOperand *MMO) { assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); SDVTList VTs = getVTList(VT, MVT::Other); SDValue Ops[] = {Chain, Ptr}; - return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope); + return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); } /// getMergeValues - Create a MERGE_VALUES node from the given operands. Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4008,13 +4008,13 @@ MachineMemOperand::MOLoad, VT.getStoreSize(), I.getAlignment() ? I.getAlignment() : - DAG.getEVTAlignment(VT)); + DAG.getEVTAlignment(VT), + AAMDNodes(), nullptr, Scope, Order); InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, - getValue(I.getPointerOperand()), MMO, - Order, Scope); + getValue(I.getPointerOperand()), MMO); SDValue OutChain = L.getValue(1); Index: lib/Target/SystemZ/SystemZISelLowering.cpp =================================================================== --- lib/Target/SystemZ/SystemZISelLowering.cpp +++ lib/Target/SystemZ/SystemZISelLowering.cpp @@ -3310,8 +3310,7 @@ if (NegSrc2.getNode()) return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, Node->getChain(), Node->getBasePtr(), NegSrc2, - Node->getMemOperand(), Node->getOrdering(), - Node->getSynchScope()); + Node->getMemOperand()); // Use the node as-is. return Op; Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -21551,8 +21551,7 @@ AtomicSDNode *AN = cast(N.getNode()); RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS); return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS, - RHS, AN->getMemOperand(), AN->getOrdering(), - AN->getSynchScope()); + RHS, AN->getMemOperand()); } assert(Opc == ISD::ATOMIC_LOAD_ADD && "Used AtomicRMW ops other than Add should have been expanded!"); @@ -21583,9 +21582,7 @@ cast(Node)->getMemoryVT(), Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), - cast(Node)->getMemOperand(), - cast(Node)->getOrdering(), - cast(Node)->getSynchScope()); + cast(Node)->getMemOperand()); return Swap.getValue(1); } // Other atomic stores have a simple pattern.