Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -636,23 +636,24 @@ SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value* PtrVal, - unsigned Alignment, AtomicOrdering Ordering, + unsigned Alignment, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result and chain and @@ -660,11 +661,11 @@ SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a @@ -676,17 +677,20 @@ const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align = 0, bool Vol = false, - bool ReadMem = true, bool WriteMem = true); + bool ReadMem = true, bool WriteMem = true, + unsigned TargetFlags = 0); SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align = 0, bool Vol = false, - bool ReadMem = true, bool WriteMem = true); + bool ReadMem = true, bool WriteMem = true, + unsigned TargetFlags = 0); SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, - EVT MemVT, MachineMemOperand *MMO); + EVT MemVT, MachineMemOperand *MMO, + unsigned TargetFlags = 0); /// getMergeValues - Create a MERGE_VALUES node from the given operands. SDValue getMergeValues(const SDValue *Ops, unsigned NumOps, DebugLoc dl); Index: include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- include/llvm/CodeGen/SelectionDAGNodes.h +++ include/llvm/CodeGen/SelectionDAGNodes.h @@ -1013,15 +1013,20 @@ class AtomicSDNode : public MemSDNode { SDUse Ops[4]; - void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) { + void InitAtomic(AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) { // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. assert((Ordering & 15) == Ordering && "Ordering may not require more than 4 bits!"); + assert((TargetFlags & 3) == TargetFlags && + "TargetFlags may not require more than 2 bits!"); assert((SynchScope & 1) == SynchScope && "SynchScope may not require more than 1 bit!"); SubclassData |= Ordering << 8; + SubclassData |= TargetFlags; SubclassData |= SynchScope << 12; assert(getOrdering() == Ordering && "Ordering encoding error!"); + assert(getTargetFlags() == TargetFlags && "TargetFlags encoding error!"); assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); } @@ -1037,28 +1042,34 @@ AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, TargetFlags, SynchScope); InitOperands(Ops, Chain, Ptr, Cmp, Swp); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, TargetFlags, SynchScope); InitOperands(Ops, Chain, Ptr, Val); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, TargetFlags, SynchScope); InitOperands(Ops, Chain, Ptr); } + /// getTargetFlags - Return target-specific flags. + unsigned getTargetFlags() const { return SubclassData & 3; } + const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } @@ -1094,10 +1105,17 @@ public: MemIntrinsicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, - EVT MemoryVT, MachineMemOperand *MMO) + EVT MemoryVT, MachineMemOperand *MMO, + unsigned TargetFlags = 0) : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, MMO) { + assert((TargetFlags & 3) == TargetFlags && + "TargetFlags may not require more than 2 bits!"); + SubclassData |= TargetFlags; } + /// getTargetFlags - Return target-specific flags. + unsigned getTargetFlags() const { return SubclassData & 3; } + // Methods to support isa and dyn_cast static bool classof(const SDNode *N) { // We lower some target intrinsics to their target opcode Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2787,6 +2787,7 @@ Node->getOperand(1), Zero, Zero, cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getTargetFlags(), cast(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); @@ -2800,6 +2801,7 @@ Node->getOperand(1), Node->getOperand(2), cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getTargetFlags(), cast(Node)->getSynchScope()); Results.push_back(Swap.getValue(1)); break; Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -169,7 +169,8 @@ SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), ResVT, N->getChain(), N->getBasePtr(), - N->getMemOperand(), N->getOrdering(), + N->getMemOperand(), + N->getOrdering(), N->getTargetFlags(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -182,7 +183,8 @@ SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, N->getMemOperand(), N->getOrdering(), + Op2, N->getMemOperand(), + N->getOrdering(), N->getTargetFlags(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -195,7 +197,8 @@ SDValue Op3 = GetPromotedInteger(N->getOperand(3)); SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, Op3, N->getMemOperand(), N->getOrdering(), + Op2, Op3, N->getMemOperand(), + N->getOrdering(), N->getTargetFlags(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -853,7 +856,8 @@ SDValue Op2 = GetPromotedInteger(N->getOperand(2)); return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), - N->getOrdering(), N->getSynchScope()); + N->getOrdering(), N->getTargetFlags(), + N->getSynchScope()); } SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { @@ -2435,6 +2439,7 @@ N->getOperand(1), Zero, Zero, cast(N)->getMemOperand(), cast(N)->getOrdering(), + cast(N)->getTargetFlags(), cast(N)->getSynchScope()); ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); ReplaceValueWith(SDValue(N, 1), Swap.getValue(1)); @@ -2859,6 +2864,7 @@ N->getOperand(1), N->getOperand(2), cast(N)->getMemOperand(), cast(N)->getOrdering(), + cast(N)->getTargetFlags(), cast(N)->getSynchScope()); return Swap.getValue(1); } Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -4051,7 +4051,7 @@ SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4072,14 +4072,14 @@ MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, - Ordering, SynchScope); + Ordering, TargetFlags, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); @@ -4099,7 +4099,7 @@ } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO, Ordering, - SynchScope); + TargetFlags, SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4110,7 +4110,7 @@ SDValue Ptr, SDValue Val, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4133,14 +4133,14 @@ MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, - Ordering, SynchScope); + Ordering, TargetFlags, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || @@ -4171,8 +4171,8 @@ return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, Val, MMO, - Ordering, SynchScope); + Ptr, Val, MMO, Ordering, + TargetFlags, SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4183,7 +4183,7 @@ SDValue Ptr, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4206,14 +4206,14 @@ MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, - Ordering, SynchScope); + Ordering, TargetFlags, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); @@ -4229,7 +4229,8 @@ return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, MMO, Ordering, SynchScope); + Ptr, MMO, Ordering, TargetFlags, + SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4255,10 +4256,11 @@ const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, - bool ReadMem, bool WriteMem) { + bool ReadMem, bool WriteMem, + unsigned TargetFlags) { return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, MemVT, PtrInfo, Align, Vol, - ReadMem, WriteMem); + ReadMem, WriteMem, TargetFlags); } SDValue @@ -4266,7 +4268,8 @@ const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, - bool ReadMem, bool WriteMem) { + bool ReadMem, bool WriteMem, + unsigned TargetFlags) { if (Align == 0) // Ensure that codegen never sees alignment 0 Align = getEVTAlignment(MemVT); @@ -4281,13 +4284,14 @@ MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); - return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); + return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO, TargetFlags); } SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, - EVT MemVT, MachineMemOperand *MMO) { + EVT MemVT, MachineMemOperand *MMO, + unsigned TargetFlags) { assert((Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || @@ -4310,11 +4314,11 @@ } N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, - MemVT, MMO); + MemVT, MMO, TargetFlags); CSEMap.InsertNode(N, IP); } else { N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, - MemVT, MMO); + MemVT, MMO, TargetFlags); } AllNodes.push_back(N); return SDValue(N, 0); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3423,9 +3423,25 @@ return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3); } +static unsigned GetAtomicTargetFlags(const Instruction &I) { + const MDNode* TargetFlagsInfo = I.getMetadata("targetflags"); + + if (!TargetFlagsInfo) + return 0; + + assert((TargetFlagsInfo->getNumOperands() > 0) && + "'targetflags' requires 1 operand!"); + const ConstantInt *CI = + dyn_cast(TargetFlagsInfo->getOperand(0)); + assert(CI && "'targetflags' not a constant integer!"); + + return CI->getZExtValue(); +} + void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3443,6 +3459,7 @@ getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); SDValue OutChain = L.getValue(1); @@ -3473,6 +3490,7 @@ case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; } AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3489,6 +3507,7 @@ getValue(I.getValOperand()), I.getPointerOperand(), 0 /* Alignment */, TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); SDValue OutChain = L.getValue(1); @@ -3513,6 +3532,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3527,6 +3547,7 @@ getValue(I.getPointerOperand()), I.getPointerOperand(), I.getAlignment(), TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); SDValue OutChain = L.getValue(1); @@ -3543,6 +3564,7 @@ DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3563,6 +3585,7 @@ getValue(I.getValueOperand()), I.getPointerOperand(), I.getAlignment(), TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); if (TLI.getInsertFencesForAtomic()) Index: lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.cpp +++ lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2414,6 +2414,7 @@ Chain, // Chain AtomicOp.getOperand(1), // Pointer AtomicNode->getMemOperand(), Acquire, + AtomicNode->getTargetFlags(), FenceScope); if (AtomicNode->getOpcode() == ISD::ATOMIC_LOAD) @@ -2447,6 +2448,7 @@ AtomicNode->getOperand(1), // Pointer AtomicNode->getOperand(2), // Value AtomicNode->getMemOperand(), Release, + AtomicNode->getTargetFlags(), FenceScope); } Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -11927,9 +11927,10 @@ DAG.getTargetConstant(size, MVT::i8), cpIn.getValue(1) }; SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); - MachineMemOperand *MMO = cast(Op)->getMemOperand(); + const AtomicSDNode *AT = cast(Op); SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, - Ops, 5, T, MMO); + Ops, 5, T, AT->getMemOperand(), + AT->getTargetFlags()); SDValue cpOut = DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1)); return cpOut; @@ -11987,6 +11988,7 @@ cast(Node)->getSrcValue(), cast(Node)->getAlignment(), cast(Node)->getOrdering(), + cast(Node)->getTargetFlags(), cast(Node)->getSynchScope()); } @@ -12008,6 +12010,7 @@ Node->getOperand(1), Node->getOperand(2), cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getTargetFlags(), cast(Node)->getSynchScope()); return Swap.getValue(1); } @@ -12180,6 +12183,7 @@ Node->getOperand(1), Zero, Zero, cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getTargetFlags(), cast(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); @@ -12200,9 +12204,10 @@ Node->getOperand(2), DAG.getIntPtrConstant(1)); SDValue Ops[] = { Chain, In1, In2L, In2H }; SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); - SDValue Result = - DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, - cast(Node)->getMemOperand()); + const AtomicSDNode *AT = cast(Node); + SDValue Result = DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, + AT->getMemOperand(), + AT->getTargetFlags()); SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)}; Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); Results.push_back(Result.getValue(2)); @@ -12315,11 +12320,12 @@ N->getOperand(1), swapInH.getValue(1) }; SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); - MachineMemOperand *MMO = cast(N)->getMemOperand(); + const AtomicSDNode *AT = cast(N); unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG : X86ISD::LCMPXCHG8_DAG; - SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, - Ops, 3, T, MMO); + SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, 3, T, + AT->getMemOperand(), + AT->getTargetFlags()); SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, Regs64bit ? X86::RAX : X86::EAX, HalfT, Result.getValue(1));