Add HLE hint in AtomicSDNode and MemInrinsicSDNode

[darkbuck]

Summary

Hi All,

The attached patch introduces new field, HLEHint, in AtomicSDNode and MemIntrinsicSDNode. This new field is used to store HLE hint, acquire, release or none. This field is populated from HLE metadata attached to atomic IR.

Yours

• Michael

[darkbuck]

previous TargetFlags is revised to HLEHint

[darkbuck]

Hi All

The patch is rebased on trunk. There is no significant change.

Thanks for review

• Michael

[chandlerc]

Unsure if you or others are still interested in this patch, but clearing it off my dashboard for now. Please update it if you need me to take a look.

include/llvm/CodeGen/SelectionDAG.h

   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Cmp, SDValue Swp,
                     MachinePointerInfo PtrInfo, unsigned Alignment,
-                    AtomicOrdering Ordering,
+                    AtomicOrdering Ordering, HLEHint Hint,
                     SynchronizationScope SynchScope);
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Cmp, SDValue Swp,
                     MachineMemOperand *MMO,
-                    AtomicOrdering Ordering,
+                    AtomicOrdering Ordering, HLEHint Hint,
                     SynchronizationScope SynchScope);
 
   /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
   /// and chain and takes 2 operands.
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Val, const Value* PtrVal,
-                    unsigned Alignment, AtomicOrdering Ordering,
+                    unsigned Alignment,
+                    AtomicOrdering Ordering, HLEHint Hint,
                     SynchronizationScope SynchScope);
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Val, MachineMemOperand *MMO,
-                    AtomicOrdering Ordering,
+                    AtomicOrdering Ordering, HLEHint Hint,
                     SynchronizationScope SynchScope);
 
   /// getAtomic - Gets a node for an atomic op, produces result and chain and

   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,
                     SDValue Chain, SDValue Ptr, const Value* PtrVal,
                     unsigned Alignment,
-                    AtomicOrdering Ordering,
+                    AtomicOrdering Ordering, HLEHint Hint,
                     SynchronizationScope SynchScope);
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,
                     SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,
-                    AtomicOrdering Ordering,
+                    AtomicOrdering Ordering, HLEHint Hint,
                     SynchronizationScope SynchScope);
 
   /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a

                               const SDValue *Ops, unsigned NumOps,
                               EVT MemVT, MachinePointerInfo PtrInfo,
                               unsigned Align = 0, bool Vol = false,
-                              bool ReadMem = true, bool WriteMem = true);
+                              bool ReadMem = true, bool WriteMem = true,
+                              HLEHint Hint = HLENone);
 
   SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
                               const SDValue *Ops, unsigned NumOps,
                               EVT MemVT, MachinePointerInfo PtrInfo,
                               unsigned Align = 0, bool Vol = false,
-                              bool ReadMem = true, bool WriteMem = true);
+                              bool ReadMem = true, bool WriteMem = true,
+                              HLEHint Hint = HLENone);
 
   SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
                               const SDValue *Ops, unsigned NumOps,
-                              EVT MemVT, MachineMemOperand *MMO);
+                              EVT MemVT, MachineMemOperand *MMO,
+                              HLEHint Hint = HLENone);
 
   /// getMergeValues - Create a MERGE_VALUES node from the given operands.
   SDValue getMergeValues(const SDValue *Ops, unsigned NumOps, DebugLoc dl);

include/llvm/CodeGen/SelectionDAGNodes.h

 class AtomicSDNode : public MemSDNode {
   SDUse Ops[4];
 
-  void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) {
+  void InitAtomic(AtomicOrdering Ordering, HLEHint Hint,
+                  SynchronizationScope SynchScope) {
     // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp.
     assert((Ordering & 15) == Ordering &&
            "Ordering may not require more than 4 bits!");
+    assert((Hint & 3) == Hint &&
+           "HLE hint may not require more than 2 bits!");
     assert((SynchScope & 1) == SynchScope &&
            "SynchScope may not require more than 1 bit!");
     SubclassData |= Ordering << 8;
+    SubclassData |= Hint;
     SubclassData |= SynchScope << 12;
     assert(getOrdering() == Ordering && "Ordering encoding error!");
+    assert(getHLEHint() == Hint && "TargetFlags encoding error!");
     assert(getSynchScope() == SynchScope && "Synch-scope encoding error!");
   }
 

   AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
                SDValue Chain, SDValue Ptr,
                SDValue Cmp, SDValue Swp, MachineMemOperand *MMO,
-               AtomicOrdering Ordering, SynchronizationScope SynchScope)
+               AtomicOrdering Ordering, HLEHint Hint,
+               SynchronizationScope SynchScope)
     : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
-    InitAtomic(Ordering, SynchScope);
+    InitAtomic(Ordering, Hint, SynchScope);
     InitOperands(Ops, Chain, Ptr, Cmp, Swp);
   }
   AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
                SDValue Chain, SDValue Ptr,
                SDValue Val, MachineMemOperand *MMO,
-               AtomicOrdering Ordering, SynchronizationScope SynchScope)
+               AtomicOrdering Ordering, HLEHint Hint,
+               SynchronizationScope SynchScope)
     : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
-    InitAtomic(Ordering, SynchScope);
+    InitAtomic(Ordering, Hint, SynchScope);
     InitOperands(Ops, Chain, Ptr, Val);
   }
   AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
                SDValue Chain, SDValue Ptr,
                MachineMemOperand *MMO,
-               AtomicOrdering Ordering, SynchronizationScope SynchScope)
+               AtomicOrdering Ordering, HLEHint Hint,
+               SynchronizationScope SynchScope)
     : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
-    InitAtomic(Ordering, SynchScope);
+    InitAtomic(Ordering, Hint, SynchScope);
     InitOperands(Ops, Chain, Ptr);
   }
 
+  /// getHLEHint - Return HLE hint.
+  HLEHint getHLEHint() const { return HLEHint(SubclassData & 3); }
+
   const SDValue &getBasePtr() const { return getOperand(1); }
   const SDValue &getVal() const { return getOperand(2); }
 

 public:
   MemIntrinsicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs,
                      const SDValue *Ops, unsigned NumOps,
-                     EVT MemoryVT, MachineMemOperand *MMO)
+                     EVT MemoryVT, MachineMemOperand *MMO,
+                     HLEHint Hint = HLENone)
     : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, MMO) {
+    assert((Hint & 3) == Hint &&
+           "HLE hint may not require more than 2 bits!");
+    SubclassData |= Hint;
   }
 
+  /// getHLEHint - Return HLE hint.
+  HLEHint getHLEHint() const { return HLEHint(SubclassData & 3); }
+
   // Methods to support isa and dyn_cast
   static bool classof(const SDNode *N) {
     // We lower some target intrinsics to their target opcode

include/llvm/IR/Instructions.h

   CrossThread = 1
 };
 
+enum HLEHint {
+  HLENone = 0,
+  HLEAcquire = 1,
+  HLERelease = 2
+};
+
 //===----------------------------------------------------------------------===//
 //                                AllocaInst Class
 //===----------------------------------------------------------------------===//

lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

                                  Node->getOperand(1), Zero, Zero,
                                  cast<AtomicSDNode>(Node)->getMemOperand(),
                                  cast<AtomicSDNode>(Node)->getOrdering(),
+                                 cast<AtomicSDNode>(Node)->getHLEHint(),
                                  cast<AtomicSDNode>(Node)->getSynchScope());
     Results.push_back(Swap.getValue(0));
     Results.push_back(Swap.getValue(1));

                                  Node->getOperand(1), Node->getOperand(2),
                                  cast<AtomicSDNode>(Node)->getMemOperand(),
                                  cast<AtomicSDNode>(Node)->getOrdering(),
+                                 cast<AtomicSDNode>(Node)->getHLEHint(),
                                  cast<AtomicSDNode>(Node)->getSynchScope());
     Results.push_back(Swap.getValue(1));
     break;

lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
                               N->getMemoryVT(), ResVT,
                               N->getChain(), N->getBasePtr(),
-                              N->getMemOperand(), N->getOrdering(),
+                              N->getMemOperand(),
+                              N->getOrdering(), N->getHLEHint(),
                               N->getSynchScope());
   // Legalized the chain result - switch anything that used the old chain to
   // use the new one.

   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
                               N->getMemoryVT(),
                               N->getChain(), N->getBasePtr(),
-                              Op2, N->getMemOperand(), N->getOrdering(),
+                              Op2, N->getMemOperand(),
+                              N->getOrdering(), N->getHLEHint(),
                               N->getSynchScope());
   // Legalized the chain result - switch anything that used the old chain to
   // use the new one.

   SDValue Op3 = GetPromotedInteger(N->getOperand(3));
   SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
                               N->getMemoryVT(), N->getChain(), N->getBasePtr(),
-                              Op2, Op3, N->getMemOperand(), N->getOrdering(),
+                              Op2, Op3, N->getMemOperand(),
+                              N->getOrdering(), N->getHLEHint(),
                               N->getSynchScope());
   // Legalized the chain result - switch anything that used the old chain to
   // use the new one.

   SDValue Op2 = GetPromotedInteger(N->getOperand(2));
   return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(),
                        N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(),
-                       N->getOrdering(), N->getSynchScope());
+                       N->getOrdering(), N->getHLEHint(),
+                       N->getSynchScope());
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {

                                N->getOperand(1), Zero, Zero,
                                cast<AtomicSDNode>(N)->getMemOperand(),
                                cast<AtomicSDNode>(N)->getOrdering(),
+                               cast<AtomicSDNode>(N)->getHLEHint(),
                                cast<AtomicSDNode>(N)->getSynchScope());
   ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
   ReplaceValueWith(SDValue(N, 1), Swap.getValue(1));

                                N->getOperand(1), N->getOperand(2),
                                cast<AtomicSDNode>(N)->getMemOperand(),
                                cast<AtomicSDNode>(N)->getOrdering(),
+                               cast<AtomicSDNode>(N)->getHLEHint(),
                                cast<AtomicSDNode>(N)->getSynchScope());
   return Swap.getValue(1);
 }

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

                                 SDValue Chain, SDValue Ptr, SDValue Cmp,
                                 SDValue Swp, MachinePointerInfo PtrInfo,
                                 unsigned Alignment,
-                                AtomicOrdering Ordering,
+                                AtomicOrdering Ordering, HLEHint Hint,
                                 SynchronizationScope SynchScope) {
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(MemVT);

     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
 
   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
-                   Ordering, SynchScope);
+                   Ordering, Hint, SynchScope);
 }
 
 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 SDValue Chain,
                                 SDValue Ptr, SDValue Cmp,
                                 SDValue Swp, MachineMemOperand *MMO,
-                                AtomicOrdering Ordering,
+                                AtomicOrdering Ordering, HLEHint Hint,
                                 SynchronizationScope SynchScope) {
   assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");

   }
   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
                                                Ptr, Cmp, Swp, MMO, Ordering,
-                                               SynchScope);
+                                               Hint, SynchScope);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDValue(N, 0);

                                 SDValue Ptr, SDValue Val,
                                 const Value* PtrVal,
                                 unsigned Alignment,
-                                AtomicOrdering Ordering,
+                                AtomicOrdering Ordering, HLEHint Hint,
                                 SynchronizationScope SynchScope) {
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(MemVT);

                             MemVT.getStoreSize(), Alignment);
 
   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
-                   Ordering, SynchScope);
+                   Ordering, Hint, SynchScope);
 }
 
 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 SDValue Chain,
                                 SDValue Ptr, SDValue Val,
                                 MachineMemOperand *MMO,
-                                AtomicOrdering Ordering,
+                                AtomicOrdering Ordering, HLEHint Hint,
                                 SynchronizationScope SynchScope) {
   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
           Opcode == ISD::ATOMIC_LOAD_SUB ||

     return SDValue(E, 0);
   }
   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
-                                               Ptr, Val, MMO,
-                                               Ordering, SynchScope);
+                                               Ptr, Val, MMO, Ordering,
+                                               Hint, SynchScope);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDValue(N, 0);

                                 SDValue Ptr,
                                 const Value* PtrVal,
                                 unsigned Alignment,
-                                AtomicOrdering Ordering,
+                                AtomicOrdering Ordering, HLEHint Hint,
                                 SynchronizationScope SynchScope) {
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(MemVT);

                             MemVT.getStoreSize(), Alignment);
 
   return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO,
-                   Ordering, SynchScope);
+                   Ordering, Hint, SynchScope);
 }
 
 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 EVT VT, SDValue Chain,
                                 SDValue Ptr,
                                 MachineMemOperand *MMO,
-                                AtomicOrdering Ordering,
+                                AtomicOrdering Ordering, HLEHint Hint,
                                 SynchronizationScope SynchScope) {
   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
 

     return SDValue(E, 0);
   }
   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
-                                               Ptr, MMO, Ordering, SynchScope);
+                                               Ptr, MMO, Ordering, Hint,
+                                               SynchScope);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDValue(N, 0);

                                   const SDValue *Ops, unsigned NumOps,
                                   EVT MemVT, MachinePointerInfo PtrInfo,
                                   unsigned Align, bool Vol,
-                                  bool ReadMem, bool WriteMem) {
+                                  bool ReadMem, bool WriteMem,
+                                  HLEHint Hint) {
   return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps,
                              MemVT, PtrInfo, Align, Vol,
-                             ReadMem, WriteMem);
+                             ReadMem, WriteMem, Hint);
 }
 
 SDValue

                                   const SDValue *Ops, unsigned NumOps,
                                   EVT MemVT, MachinePointerInfo PtrInfo,
                                   unsigned Align, bool Vol,
-                                  bool ReadMem, bool WriteMem) {
+                                  bool ReadMem, bool WriteMem,
+                                  HLEHint Hint) {
   if (Align == 0)  // Ensure that codegen never sees alignment 0
     Align = getEVTAlignment(MemVT);
 

   MachineMemOperand *MMO =
     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align);
 
-  return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
+  return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO, Hint);
 }
 
 SDValue
 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
                                   const SDValue *Ops, unsigned NumOps,
-                                  EVT MemVT, MachineMemOperand *MMO) {
+                                  EVT MemVT, MachineMemOperand *MMO,
+                                  HLEHint Hint) {
   assert((Opcode == ISD::INTRINSIC_VOID ||
           Opcode == ISD::INTRINSIC_W_CHAIN ||
           Opcode == ISD::PREFETCH ||

     }
 
     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps,
-                                               MemVT, MMO);
+                                               MemVT, MMO, Hint);
     CSEMap.InsertNode(N, IP);
   } else {
     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps,
-                                               MemVT, MMO);
+                                               MemVT, MMO, Hint);
   }
   AllNodes.push_back(N);
   return SDValue(N, 0);

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

   return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3);
 }
 
+static HLEHint GetHLEHint(const Instruction &I) {
+  const MDNode* HLEHintInfo = I.getMetadata("hle.lock");
+
+  if (!HLEHintInfo)
+    return HLENone;
+
+  assert((HLEHintInfo->getNumOperands() > 0) &&
+         "'hle.lock' requires 1 operand!");
+  const MDString *string =
+    dyn_cast<MDString>(HLEHintInfo->getOperand(0));
+  assert(string && "'hle.lock' requires 1 string hint!");
+
+  if (string->getString() == "acquire")
+    return HLEAcquire;
+  else if (string->getString() == "release")
+    return HLERelease;
+
+  llvm_unreachable("Unknown 'hle.lock' hint string!");
+
+  return HLENone;
+}
+
 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
   DebugLoc dl = getCurDebugLoc();
   AtomicOrdering Order = I.getOrdering();
+  HLEHint Hint = GetHLEHint(I);
   SynchronizationScope Scope = I.getSynchScope();
 
   SDValue InChain = getRoot();

                   getValue(I.getNewValOperand()),
                   MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
                   TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Hint,
                   Scope);
 
   SDValue OutChain = L.getValue(1);

   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
   }
   AtomicOrdering Order = I.getOrdering();
+  HLEHint Hint = GetHLEHint(I);
   SynchronizationScope Scope = I.getSynchScope();
 
   SDValue InChain = getRoot();

                   getValue(I.getValOperand()),
                   I.getPointerOperand(), 0 /* Alignment */,
                   TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Hint,
                   Scope);
 
   SDValue OutChain = L.getValue(1);

 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
   DebugLoc dl = getCurDebugLoc();
   AtomicOrdering Order = I.getOrdering();
+  HLEHint Hint = GetHLEHint(I);
   SynchronizationScope Scope = I.getSynchScope();
 
   SDValue InChain = getRoot();

                   getValue(I.getPointerOperand()),
                   I.getPointerOperand(), I.getAlignment(),
                   TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Hint,
                   Scope);
 
   SDValue OutChain = L.getValue(1);

   DebugLoc dl = getCurDebugLoc();
 
   AtomicOrdering Order = I.getOrdering();
+  HLEHint Hint = GetHLEHint(I);
   SynchronizationScope Scope = I.getSynchScope();
 
   SDValue InChain = getRoot();

                   getValue(I.getValueOperand()),
                   I.getPointerOperand(), I.getAlignment(),
                   TLI.getInsertFencesForAtomic() ? Monotonic : Order,
+                  Hint,
                   Scope);
 
   if (TLI.getInsertFencesForAtomic())

lib/Target/AArch64/AArch64ISelLowering.cpp

                              Chain,                  // Chain
                              AtomicOp.getOperand(1), // Pointer
                              AtomicNode->getMemOperand(), Acquire,
+                             AtomicNode->getHLEHint(),
                              FenceScope);
 
   if (AtomicNode->getOpcode() == ISD::ATOMIC_LOAD)

                        AtomicNode->getOperand(1),       // Pointer
                        AtomicNode->getOperand(2),       // Value
                        AtomicNode->getMemOperand(), Release,
+                       AtomicNode->getHLEHint(),
                        FenceScope);
 }
 

lib/Target/X86/X86ISelLowering.cpp

                     DAG.getTargetConstant(size, MVT::i8),
                     cpIn.getValue(1) };
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
-  MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
+  const AtomicSDNode *AT = cast<AtomicSDNode>(Op);
   SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
-                                           Ops, 5, T, MMO);
+                                           Ops, 5, T, AT->getMemOperand(),
+                                           AT->getHLEHint());
   SDValue cpOut =
     DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
   return cpOut;

                        cast<AtomicSDNode>(Node)->getSrcValue(),
                        cast<AtomicSDNode>(Node)->getAlignment(),
                        cast<AtomicSDNode>(Node)->getOrdering(),
+                       cast<AtomicSDNode>(Node)->getHLEHint(),
                        cast<AtomicSDNode>(Node)->getSynchScope());
 }
 

                                  Node->getOperand(1), Node->getOperand(2),
                                  cast<AtomicSDNode>(Node)->getMemOperand(),
                                  cast<AtomicSDNode>(Node)->getOrdering(),
+                                 cast<AtomicSDNode>(Node)->getHLEHint(),
                                  cast<AtomicSDNode>(Node)->getSynchScope());
     return Swap.getValue(1);
   }

                                Node->getOperand(1), Zero, Zero,
                                cast<AtomicSDNode>(Node)->getMemOperand(),
                                cast<AtomicSDNode>(Node)->getOrdering(),
+                               cast<AtomicSDNode>(Node)->getHLEHint(),
                                cast<AtomicSDNode>(Node)->getSynchScope());
   Results.push_back(Swap.getValue(0));
   Results.push_back(Swap.getValue(1));

                              Node->getOperand(2), DAG.getIntPtrConstant(1));
   SDValue Ops[] = { Chain, In1, In2L, In2H };
   SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
-  SDValue Result =
-    DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64,
-                            cast<MemSDNode>(Node)->getMemOperand());
+  const AtomicSDNode *AT = cast<AtomicSDNode>(Node);
+  SDValue Result = DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64,
+                                           AT->getMemOperand(),
+                                           AT->getHLEHint());
   SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)};
   Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
   Results.push_back(Result.getValue(2));

                       N->getOperand(1),
                       swapInH.getValue(1) };
     SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
-    MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
+    const AtomicSDNode *AT = cast<AtomicSDNode>(N);
     unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG :
                                   X86ISD::LCMPXCHG8_DAG;
-    SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys,
-                                             Ops, 3, T, MMO);
+    SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, 3, T,
+                                             AT->getMemOperand(),
+                                             AT->getHLEHint());
     SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
                                         Regs64bit ? X86::RAX : X86::EAX,
                                         HalfT, Result.getValue(1));