Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -636,23 +636,24 @@ SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value* PtrVal, - unsigned Alignment, AtomicOrdering Ordering, + unsigned Alignment, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result and chain and @@ -660,11 +661,11 @@ SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope); /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a @@ -676,17 +677,20 @@ const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align = 0, bool Vol = false, - bool ReadMem = true, bool WriteMem = true); + bool ReadMem = true, bool WriteMem = true, + HLEHint Hint = HLENone); SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align = 0, bool Vol = false, - bool ReadMem = true, bool WriteMem = true); + bool ReadMem = true, bool WriteMem = true, + HLEHint Hint = HLENone); SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, - EVT MemVT, MachineMemOperand *MMO); + EVT MemVT, MachineMemOperand *MMO, + HLEHint Hint = HLENone); /// getMergeValues - Create a MERGE_VALUES node from the given operands. SDValue getMergeValues(const SDValue *Ops, unsigned NumOps, DebugLoc dl); Index: include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- include/llvm/CodeGen/SelectionDAGNodes.h +++ include/llvm/CodeGen/SelectionDAGNodes.h @@ -1013,15 +1013,20 @@ class AtomicSDNode : public MemSDNode { SDUse Ops[4]; - void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) { + void InitAtomic(AtomicOrdering Ordering, HLEHint Hint, + SynchronizationScope SynchScope) { // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. assert((Ordering & 15) == Ordering && "Ordering may not require more than 4 bits!"); + assert((Hint & 3) == Hint && + "HLE hint may not require more than 2 bits!"); assert((SynchScope & 1) == SynchScope && "SynchScope may not require more than 1 bit!"); SubclassData |= Ordering << 8; + SubclassData |= Hint; SubclassData |= SynchScope << 12; assert(getOrdering() == Ordering && "Ordering encoding error!"); + assert(getHLEHint() == Hint && "TargetFlags encoding error!"); assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); } @@ -1037,28 +1042,34 @@ AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, HLEHint Hint, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, Hint, SynchScope); InitOperands(Ops, Chain, Ptr, Cmp, Swp); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, HLEHint Hint, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, Hint, SynchScope); InitOperands(Ops, Chain, Ptr, Val); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, HLEHint Hint, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, Hint, SynchScope); InitOperands(Ops, Chain, Ptr); } + /// getHLEHint - Return HLE hint. + HLEHint getHLEHint() const { return HLEHint(SubclassData & 3); } + const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } @@ -1094,10 +1105,17 @@ public: MemIntrinsicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, - EVT MemoryVT, MachineMemOperand *MMO) + EVT MemoryVT, MachineMemOperand *MMO, + HLEHint Hint = HLENone) : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, MMO) { + assert((Hint & 3) == Hint && + "HLE hint may not require more than 2 bits!"); + SubclassData |= Hint; } + /// getHLEHint - Return HLE hint. + HLEHint getHLEHint() const { return HLEHint(SubclassData & 3); } + // Methods to support isa and dyn_cast static bool classof(const SDNode *N) { // We lower some target intrinsics to their target opcode Index: include/llvm/IR/Instructions.h =================================================================== --- include/llvm/IR/Instructions.h +++ include/llvm/IR/Instructions.h @@ -51,6 +51,12 @@ CrossThread = 1 }; +enum HLEHint { + HLENone = 0, + HLEAcquire = 1, + HLERelease = 2 +}; + //===----------------------------------------------------------------------===// // AllocaInst Class //===----------------------------------------------------------------------===// Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2787,6 +2787,7 @@ Node->getOperand(1), Zero, Zero, cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getHLEHint(), cast(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); @@ -2800,6 +2801,7 @@ Node->getOperand(1), Node->getOperand(2), cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getHLEHint(), cast(Node)->getSynchScope()); Results.push_back(Swap.getValue(1)); break; Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -169,7 +169,8 @@ SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), ResVT, N->getChain(), N->getBasePtr(), - N->getMemOperand(), N->getOrdering(), + N->getMemOperand(), + N->getOrdering(), N->getHLEHint(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -182,7 +183,8 @@ SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, N->getMemOperand(), N->getOrdering(), + Op2, N->getMemOperand(), + N->getOrdering(), N->getHLEHint(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -195,7 +197,8 @@ SDValue Op3 = GetPromotedInteger(N->getOperand(3)); SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, Op3, N->getMemOperand(), N->getOrdering(), + Op2, Op3, N->getMemOperand(), + N->getOrdering(), N->getHLEHint(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -855,7 +858,8 @@ SDValue Op2 = GetPromotedInteger(N->getOperand(2)); return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), - N->getOrdering(), N->getSynchScope()); + N->getOrdering(), N->getHLEHint(), + N->getSynchScope()); } SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { @@ -2438,6 +2442,7 @@ N->getOperand(1), Zero, Zero, cast(N)->getMemOperand(), cast(N)->getOrdering(), + cast(N)->getHLEHint(), cast(N)->getSynchScope()); ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); ReplaceValueWith(SDValue(N, 1), Swap.getValue(1)); @@ -2862,6 +2867,7 @@ N->getOperand(1), N->getOperand(2), cast(N)->getMemOperand(), cast(N)->getOrdering(), + cast(N)->getHLEHint(), cast(N)->getSynchScope()); return Swap.getValue(1); } Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -4056,7 +4056,7 @@ SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4077,14 +4077,14 @@ MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, - Ordering, SynchScope); + Ordering, Hint, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope) { assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); @@ -4104,7 +4104,7 @@ } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO, Ordering, - SynchScope); + Hint, SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4115,7 +4115,7 @@ SDValue Ptr, SDValue Val, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4138,14 +4138,14 @@ MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, - Ordering, SynchScope); + Ordering, Hint, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope) { assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || @@ -4176,8 +4176,8 @@ return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, Val, MMO, - Ordering, SynchScope); + Ptr, Val, MMO, Ordering, + Hint, SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4188,7 +4188,7 @@ SDValue Ptr, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4211,14 +4211,14 @@ MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, - Ordering, SynchScope); + Ordering, Hint, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, HLEHint Hint, SynchronizationScope SynchScope) { assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); @@ -4234,7 +4234,8 @@ return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, MMO, Ordering, SynchScope); + Ptr, MMO, Ordering, Hint, + SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4260,10 +4261,11 @@ const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, - bool ReadMem, bool WriteMem) { + bool ReadMem, bool WriteMem, + HLEHint Hint) { return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, MemVT, PtrInfo, Align, Vol, - ReadMem, WriteMem); + ReadMem, WriteMem, Hint); } SDValue @@ -4271,7 +4273,8 @@ const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, - bool ReadMem, bool WriteMem) { + bool ReadMem, bool WriteMem, + HLEHint Hint) { if (Align == 0) // Ensure that codegen never sees alignment 0 Align = getEVTAlignment(MemVT); @@ -4286,13 +4289,14 @@ MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); - return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); + return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO, Hint); } SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, - EVT MemVT, MachineMemOperand *MMO) { + EVT MemVT, MachineMemOperand *MMO, + HLEHint Hint) { assert((Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || @@ -4315,11 +4319,11 @@ } N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, - MemVT, MMO); + MemVT, MMO, Hint); CSEMap.InsertNode(N, IP); } else { N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, - MemVT, MMO); + MemVT, MMO, Hint); } AllNodes.push_back(N); return SDValue(N, 0); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3423,9 +3423,32 @@ return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3); } +static HLEHint GetHLEHint(const Instruction &I) { + const MDNode* HLEHintInfo = I.getMetadata("hle.lock"); + + if (!HLEHintInfo) + return HLENone; + + assert((HLEHintInfo->getNumOperands() > 0) && + "'hle.lock' requires 1 operand!"); + const MDString *string = + dyn_cast(HLEHintInfo->getOperand(0)); + assert(string && "'hle.lock' requires 1 string hint!"); + + if (string->getString() == "acquire") + return HLEAcquire; + else if (string->getString() == "release") + return HLERelease; + + llvm_unreachable("Unknown 'hle.lock' hint string!"); + + return HLENone; +} + void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + HLEHint Hint = GetHLEHint(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3443,6 +3466,7 @@ getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Hint, Scope); SDValue OutChain = L.getValue(1); @@ -3473,6 +3497,7 @@ case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; } AtomicOrdering Order = I.getOrdering(); + HLEHint Hint = GetHLEHint(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3489,6 +3514,7 @@ getValue(I.getValOperand()), I.getPointerOperand(), 0 /* Alignment */, TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Hint, Scope); SDValue OutChain = L.getValue(1); @@ -3513,6 +3539,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + HLEHint Hint = GetHLEHint(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3527,6 +3554,7 @@ getValue(I.getPointerOperand()), I.getPointerOperand(), I.getAlignment(), TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Hint, Scope); SDValue OutChain = L.getValue(1); @@ -3543,6 +3571,7 @@ DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + HLEHint Hint = GetHLEHint(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3563,6 +3592,7 @@ getValue(I.getValueOperand()), I.getPointerOperand(), I.getAlignment(), TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Hint, Scope); if (TLI.getInsertFencesForAtomic()) Index: lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.cpp +++ lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2413,6 +2413,7 @@ Chain, // Chain AtomicOp.getOperand(1), // Pointer AtomicNode->getMemOperand(), Acquire, + AtomicNode->getHLEHint(), FenceScope); if (AtomicNode->getOpcode() == ISD::ATOMIC_LOAD) @@ -2446,6 +2447,7 @@ AtomicNode->getOperand(1), // Pointer AtomicNode->getOperand(2), // Value AtomicNode->getMemOperand(), Release, + AtomicNode->getHLEHint(), FenceScope); } Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -11957,9 +11957,10 @@ DAG.getTargetConstant(size, MVT::i8), cpIn.getValue(1) }; SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); - MachineMemOperand *MMO = cast(Op)->getMemOperand(); + const AtomicSDNode *AT = cast(Op); SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, - Ops, 5, T, MMO); + Ops, 5, T, AT->getMemOperand(), + AT->getHLEHint()); SDValue cpOut = DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1)); return cpOut; @@ -12017,6 +12018,7 @@ cast(Node)->getSrcValue(), cast(Node)->getAlignment(), cast(Node)->getOrdering(), + cast(Node)->getHLEHint(), cast(Node)->getSynchScope()); } @@ -12038,6 +12040,7 @@ Node->getOperand(1), Node->getOperand(2), cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getHLEHint(), cast(Node)->getSynchScope()); return Swap.getValue(1); } @@ -12210,6 +12213,7 @@ Node->getOperand(1), Zero, Zero, cast(Node)->getMemOperand(), cast(Node)->getOrdering(), + cast(Node)->getHLEHint(), cast(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); @@ -12230,9 +12234,10 @@ Node->getOperand(2), DAG.getIntPtrConstant(1)); SDValue Ops[] = { Chain, In1, In2L, In2H }; SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); - SDValue Result = - DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, - cast(Node)->getMemOperand()); + const AtomicSDNode *AT = cast(Node); + SDValue Result = DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, + AT->getMemOperand(), + AT->getHLEHint()); SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)}; Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); Results.push_back(Result.getValue(2)); @@ -12346,11 +12351,12 @@ N->getOperand(1), swapInH.getValue(1) }; SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); - MachineMemOperand *MMO = cast(N)->getMemOperand(); + const AtomicSDNode *AT = cast(N); unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG : X86ISD::LCMPXCHG8_DAG; - SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, - Ops, 3, T, MMO); + SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, 3, T, + AT->getMemOperand(), + AT->getHLEHint()); SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, Regs64bit ? X86::RAX : X86::EAX, HalfT, Result.getValue(1));