Index: llvm/trunk/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
===================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ llvm/trunk/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -49,21 +49,21 @@
 
 class SIMemoryLegalizer final : public MachineFunctionPass {
 private:
-  struct AtomicInfo final {
+  struct MemOpInfo final {
     SyncScope::ID SSID = SyncScope::System;
     AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent;
     AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent;
 
-    AtomicInfo() = default;
+    MemOpInfo() = default;
 
-    AtomicInfo(SyncScope::ID SSID,
-               AtomicOrdering Ordering,
-               AtomicOrdering FailureOrdering)
+    MemOpInfo(SyncScope::ID SSID,
+              AtomicOrdering Ordering,
+              AtomicOrdering FailureOrdering)
         : SSID(SSID),
           Ordering(Ordering),
           FailureOrdering(FailureOrdering) {}
 
-    AtomicInfo(const MachineMemOperand *MMO)
+    MemOpInfo(const MachineMemOperand *MMO)
         : SSID(MMO->getSyncScopeID()),
           Ordering(MMO->getOrdering()),
           FailureOrdering(MMO->getFailureOrdering()) {}
@@ -110,44 +110,38 @@
 
   /// \returns Atomic fence info if \p MI is an atomic fence operation,
   /// "None" otherwise.
-  Optional<AtomicInfo> getAtomicFenceInfo(
-      const MachineBasicBlock::iterator &MI) const;
-  /// \returns Atomic load info if \p MI is an atomic load operation,
-  /// "None" otherwise.
-  Optional<AtomicInfo> getAtomicLoadInfo(
-      const MachineBasicBlock::iterator &MI) const;
-  /// \returns Atomic store info if \p MI is an atomic store operation,
-  /// "None" otherwise.
-  Optional<AtomicInfo> getAtomicStoreInfo(
+  Optional<MemOpInfo> getAtomicFenceInfo(
       const MachineBasicBlock::iterator &MI) const;
+  /// \returns Load info if \p MI is a load operation, "None" otherwise.
+  Optional<MemOpInfo> getLoadInfo(const MachineBasicBlock::iterator &MI) const;
+  /// \returns Store info if \p MI is a store operation, "None" otherwise.
+  Optional<MemOpInfo> getStoreInfo(const MachineBasicBlock::iterator &MI) const;
   /// \returns Atomic cmpxchg info if \p MI is an atomic cmpxchg operation,
   /// "None" otherwise.
-  Optional<AtomicInfo> getAtomicCmpxchgInfo(
+  Optional<MemOpInfo> getAtomicCmpxchgInfo(
       const MachineBasicBlock::iterator &MI) const;
   /// \returns Atomic rmw info if \p MI is an atomic rmw operation,
   /// "None" otherwise.
-  Optional<AtomicInfo> getAtomicRmwInfo(
+  Optional<MemOpInfo> getAtomicRmwInfo(
       const MachineBasicBlock::iterator &MI) const;
 
   /// \brief Expands atomic fence operation \p MI. Returns true if
   /// instructions are added/deleted or \p MI is modified, false otherwise.
-  bool expandAtomicFence(const AtomicInfo &AI,
-                         MachineBasicBlock::iterator &MI);
-  /// \brief Expands atomic load operation \p MI. Returns true if
-  /// instructions are added/deleted or \p MI is modified, false otherwise.
-  bool expandAtomicLoad(const AtomicInfo &AI,
-                        MachineBasicBlock::iterator &MI);
-  /// \brief Expands atomic store operation \p MI. Returns true if
-  /// instructions are added/deleted or \p MI is modified, false otherwise.
-  bool expandAtomicStore(const AtomicInfo &AI,
+  bool expandAtomicFence(const MemOpInfo &MOI,
                          MachineBasicBlock::iterator &MI);
+  /// \brief Expands load operation \p MI. Returns true if instructions are
+  /// added/deleted or \p MI is modified, false otherwise.
+  bool expandLoad(const MemOpInfo &MOI, MachineBasicBlock::iterator &MI);
+  /// \brief Expands store operation \p MI. Returns true if instructions are
+  /// added/deleted or \p MI is modified, false otherwise.
+  bool expandStore(const MemOpInfo &MOI, MachineBasicBlock::iterator &MI);
   /// \brief Expands atomic cmpxchg operation \p MI. Returns true if
   /// instructions are added/deleted or \p MI is modified, false otherwise.
-  bool expandAtomicCmpxchg(const AtomicInfo &AI,
+  bool expandAtomicCmpxchg(const MemOpInfo &MOI,
                            MachineBasicBlock::iterator &MI);
   /// \brief Expands atomic rmw operation \p MI. Returns true if
   /// instructions are added/deleted or \p MI is modified, false otherwise.
-  bool expandAtomicRmw(const AtomicInfo &AI,
+  bool expandAtomicRmw(const MemOpInfo &MOI,
                        MachineBasicBlock::iterator &MI);
 
 public:
@@ -232,7 +226,7 @@
   CTX->diagnose(Diag);
 }
 
-Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicFenceInfo(
+Optional<SIMemoryLegalizer::MemOpInfo> SIMemoryLegalizer::getAtomicFenceInfo(
     const MachineBasicBlock::iterator &MI) const {
   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
 
@@ -243,49 +237,49 @@
       static_cast<SyncScope::ID>(MI->getOperand(1).getImm());
   AtomicOrdering Ordering =
       static_cast<AtomicOrdering>(MI->getOperand(0).getImm());
-  return AtomicInfo(SSID, Ordering, AtomicOrdering::NotAtomic);
+  return MemOpInfo(SSID, Ordering, AtomicOrdering::NotAtomic);
 }
 
-Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicLoadInfo(
+Optional<SIMemoryLegalizer::MemOpInfo> SIMemoryLegalizer::getLoadInfo(
     const MachineBasicBlock::iterator &MI) const {
   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
 
   if (!(MI->mayLoad() && !MI->mayStore()))
     return None;
   if (!MI->hasOneMemOperand())
-    return AtomicInfo();
+    return MemOpInfo();
 
   const MachineMemOperand *MMO = *MI->memoperands_begin();
   if (!MMO->isAtomic())
     return None;
 
-  return AtomicInfo(MMO);
+  return MemOpInfo(MMO);
 }
 
-Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicStoreInfo(
+Optional<SIMemoryLegalizer::MemOpInfo> SIMemoryLegalizer::getStoreInfo(
     const MachineBasicBlock::iterator &MI) const {
   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
 
   if (!(!MI->mayLoad() && MI->mayStore()))
     return None;
   if (!MI->hasOneMemOperand())
-    return AtomicInfo();
+    return MemOpInfo();
 
   const MachineMemOperand *MMO = *MI->memoperands_begin();
   if (!MMO->isAtomic())
     return None;
 
-  return AtomicInfo(MMO);
+  return MemOpInfo(MMO);
 }
 
-Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicCmpxchgInfo(
+Optional<SIMemoryLegalizer::MemOpInfo> SIMemoryLegalizer::getAtomicCmpxchgInfo(
     const MachineBasicBlock::iterator &MI) const {
   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
 
   if (!(MI->mayLoad() && MI->mayStore()))
     return None;
   if (!MI->hasOneMemOperand())
-    return AtomicInfo();
+    return MemOpInfo();
 
   const MachineMemOperand *MMO = *MI->memoperands_begin();
   if (!MMO->isAtomic())
@@ -293,17 +287,17 @@
   if (MMO->getFailureOrdering() == AtomicOrdering::NotAtomic)
     return None;
 
-  return AtomicInfo(MMO);
+  return MemOpInfo(MMO);
 }
 
-Optional<SIMemoryLegalizer::AtomicInfo> SIMemoryLegalizer::getAtomicRmwInfo(
+Optional<SIMemoryLegalizer::MemOpInfo> SIMemoryLegalizer::getAtomicRmwInfo(
     const MachineBasicBlock::iterator &MI) const {
   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
 
   if (!(MI->mayLoad() && MI->mayStore()))
     return None;
   if (!MI->hasOneMemOperand())
-    return AtomicInfo();
+    return MemOpInfo();
 
   const MachineMemOperand *MMO = *MI->memoperands_begin();
   if (!MMO->isAtomic())
@@ -311,32 +305,32 @@
   if (MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
     return None;
 
-  return AtomicInfo(MMO);
+  return MemOpInfo(MMO);
 }
 
-bool SIMemoryLegalizer::expandAtomicFence(const AtomicInfo &AI,
+bool SIMemoryLegalizer::expandAtomicFence(const MemOpInfo &MOI,
                                           MachineBasicBlock::iterator &MI) {
   assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE);
 
   bool Changed = false;
-  if (AI.SSID == SyncScope::System ||
-      AI.SSID == MMI->getAgentSSID()) {
-    if (AI.Ordering == AtomicOrdering::Acquire ||
-        AI.Ordering == AtomicOrdering::Release ||
-        AI.Ordering == AtomicOrdering::AcquireRelease ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent)
+  if (MOI.SSID == SyncScope::System ||
+      MOI.SSID == MMI->getAgentSSID()) {
+    if (MOI.Ordering == AtomicOrdering::Acquire ||
+        MOI.Ordering == AtomicOrdering::Release ||
+        MOI.Ordering == AtomicOrdering::AcquireRelease ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent)
       Changed |= insertWaitcntVmcnt0(MI);
 
-    if (AI.Ordering == AtomicOrdering::Acquire ||
-        AI.Ordering == AtomicOrdering::AcquireRelease ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent)
+    if (MOI.Ordering == AtomicOrdering::Acquire ||
+        MOI.Ordering == AtomicOrdering::AcquireRelease ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent)
       Changed |= insertBufferWbinvl1Vol(MI);
 
     AtomicPseudoMIs.push_back(MI);
     return Changed;
-  } else if (AI.SSID == SyncScope::SingleThread ||
-             AI.SSID == MMI->getWorkgroupSSID() ||
-             AI.SSID == MMI->getWavefrontSSID()) {
+  } else if (MOI.SSID == SyncScope::SingleThread ||
+             MOI.SSID == MMI->getWorkgroupSSID() ||
+             MOI.SSID == MMI->getWavefrontSSID()) {
     AtomicPseudoMIs.push_back(MI);
     return Changed;
   } else {
@@ -345,30 +339,30 @@
   }
 }
 
-bool SIMemoryLegalizer::expandAtomicLoad(const AtomicInfo &AI,
-                                         MachineBasicBlock::iterator &MI) {
+bool SIMemoryLegalizer::expandLoad(const MemOpInfo &MOI,
+                                   MachineBasicBlock::iterator &MI) {
   assert(MI->mayLoad() && !MI->mayStore());
 
   bool Changed = false;
-  if (AI.SSID == SyncScope::System ||
-      AI.SSID == MMI->getAgentSSID()) {
-    if (AI.Ordering == AtomicOrdering::Acquire ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent)
+  if (MOI.SSID == SyncScope::System ||
+      MOI.SSID == MMI->getAgentSSID()) {
+    if (MOI.Ordering == AtomicOrdering::Acquire ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent)
       Changed |= setGLC(MI);
 
-    if (AI.Ordering == AtomicOrdering::SequentiallyConsistent)
+    if (MOI.Ordering == AtomicOrdering::SequentiallyConsistent)
       Changed |= insertWaitcntVmcnt0(MI);
 
-    if (AI.Ordering == AtomicOrdering::Acquire ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent) {
+    if (MOI.Ordering == AtomicOrdering::Acquire ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent) {
       Changed |= insertWaitcntVmcnt0(MI, false);
       Changed |= insertBufferWbinvl1Vol(MI, false);
     }
 
     return Changed;
-  } else if (AI.SSID == SyncScope::SingleThread ||
-             AI.SSID == MMI->getWorkgroupSSID() ||
-             AI.SSID == MMI->getWavefrontSSID()) {
+  } else if (MOI.SSID == SyncScope::SingleThread ||
+             MOI.SSID == MMI->getWorkgroupSSID() ||
+             MOI.SSID == MMI->getWavefrontSSID()) {
     return Changed;
   } else {
     reportUnknownSynchScope(MI);
@@ -376,21 +370,21 @@
   }
 }
 
-bool SIMemoryLegalizer::expandAtomicStore(const AtomicInfo &AI,
-                                          MachineBasicBlock::iterator &MI) {
+bool SIMemoryLegalizer::expandStore(const MemOpInfo &MOI,
+                                    MachineBasicBlock::iterator &MI) {
   assert(!MI->mayLoad() && MI->mayStore());
 
   bool Changed = false;
-  if (AI.SSID == SyncScope::System ||
-      AI.SSID == MMI->getAgentSSID()) {
-    if (AI.Ordering == AtomicOrdering::Release ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent)
+  if (MOI.SSID == SyncScope::System ||
+      MOI.SSID == MMI->getAgentSSID()) {
+    if (MOI.Ordering == AtomicOrdering::Release ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent)
       Changed |= insertWaitcntVmcnt0(MI);
 
     return Changed;
-  } else if (AI.SSID == SyncScope::SingleThread ||
-             AI.SSID == MMI->getWorkgroupSSID() ||
-             AI.SSID == MMI->getWavefrontSSID()) {
+  } else if (MOI.SSID == SyncScope::SingleThread ||
+             MOI.SSID == MMI->getWorkgroupSSID() ||
+             MOI.SSID == MMI->getWavefrontSSID()) {
     return Changed;
   } else {
     reportUnknownSynchScope(MI);
@@ -398,32 +392,32 @@
   }
 }
 
-bool SIMemoryLegalizer::expandAtomicCmpxchg(const AtomicInfo &AI,
+bool SIMemoryLegalizer::expandAtomicCmpxchg(const MemOpInfo &MOI,
                                             MachineBasicBlock::iterator &MI) {
   assert(MI->mayLoad() && MI->mayStore());
 
   bool Changed = false;
-  if (AI.SSID == SyncScope::System ||
-      AI.SSID == MMI->getAgentSSID()) {
-    if (AI.Ordering == AtomicOrdering::Release ||
-        AI.Ordering == AtomicOrdering::AcquireRelease ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent ||
-        AI.FailureOrdering == AtomicOrdering::SequentiallyConsistent)
+  if (MOI.SSID == SyncScope::System ||
+      MOI.SSID == MMI->getAgentSSID()) {
+    if (MOI.Ordering == AtomicOrdering::Release ||
+        MOI.Ordering == AtomicOrdering::AcquireRelease ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent ||
+        MOI.FailureOrdering == AtomicOrdering::SequentiallyConsistent)
       Changed |= insertWaitcntVmcnt0(MI);
 
-    if (AI.Ordering == AtomicOrdering::Acquire ||
-        AI.Ordering == AtomicOrdering::AcquireRelease ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent ||
-        AI.FailureOrdering == AtomicOrdering::Acquire ||
-        AI.FailureOrdering == AtomicOrdering::SequentiallyConsistent) {
+    if (MOI.Ordering == AtomicOrdering::Acquire ||
+        MOI.Ordering == AtomicOrdering::AcquireRelease ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent ||
+        MOI.FailureOrdering == AtomicOrdering::Acquire ||
+        MOI.FailureOrdering == AtomicOrdering::SequentiallyConsistent) {
       Changed |= insertWaitcntVmcnt0(MI, false);
       Changed |= insertBufferWbinvl1Vol(MI, false);
     }
 
     return Changed;
-  } else if (AI.SSID == SyncScope::SingleThread ||
-             AI.SSID == MMI->getWorkgroupSSID() ||
-             AI.SSID == MMI->getWavefrontSSID()) {
+  } else if (MOI.SSID == SyncScope::SingleThread ||
+             MOI.SSID == MMI->getWorkgroupSSID() ||
+             MOI.SSID == MMI->getWavefrontSSID()) {
     Changed |= setGLC(MI);
     return Changed;
   } else {
@@ -432,29 +426,29 @@
   }
 }
 
-bool SIMemoryLegalizer::expandAtomicRmw(const AtomicInfo &AI,
+bool SIMemoryLegalizer::expandAtomicRmw(const MemOpInfo &MOI,
                                         MachineBasicBlock::iterator &MI) {
   assert(MI->mayLoad() && MI->mayStore());
 
   bool Changed = false;
-  if (AI.SSID == SyncScope::System ||
-      AI.SSID == MMI->getAgentSSID()) {
-    if (AI.Ordering == AtomicOrdering::Release ||
-        AI.Ordering == AtomicOrdering::AcquireRelease ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent)
+  if (MOI.SSID == SyncScope::System ||
+      MOI.SSID == MMI->getAgentSSID()) {
+    if (MOI.Ordering == AtomicOrdering::Release ||
+        MOI.Ordering == AtomicOrdering::AcquireRelease ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent)
       Changed |= insertWaitcntVmcnt0(MI);
 
-    if (AI.Ordering == AtomicOrdering::Acquire ||
-        AI.Ordering == AtomicOrdering::AcquireRelease ||
-        AI.Ordering == AtomicOrdering::SequentiallyConsistent) {
+    if (MOI.Ordering == AtomicOrdering::Acquire ||
+        MOI.Ordering == AtomicOrdering::AcquireRelease ||
+        MOI.Ordering == AtomicOrdering::SequentiallyConsistent) {
       Changed |= insertWaitcntVmcnt0(MI, false);
       Changed |= insertBufferWbinvl1Vol(MI, false);
     }
 
     return Changed;
-  } else if (AI.SSID == SyncScope::SingleThread ||
-             AI.SSID == MMI->getWorkgroupSSID() ||
-             AI.SSID == MMI->getWavefrontSSID()) {
+  } else if (MOI.SSID == SyncScope::SingleThread ||
+             MOI.SSID == MMI->getWorkgroupSSID() ||
+             MOI.SSID == MMI->getWavefrontSSID()) {
     Changed |= setGLC(MI);
     return Changed;
   } else {
@@ -482,16 +476,16 @@
       if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic))
         continue;
 
-      if (const auto &AI = getAtomicFenceInfo(MI))
-        Changed |= expandAtomicFence(AI.getValue(), MI);
-      else if (const auto &AI = getAtomicLoadInfo(MI))
-        Changed |= expandAtomicLoad(AI.getValue(), MI);
-      else if (const auto &AI = getAtomicStoreInfo(MI))
-        Changed |= expandAtomicStore(AI.getValue(), MI);
-      else if (const auto &AI = getAtomicCmpxchgInfo(MI))
-        Changed |= expandAtomicCmpxchg(AI.getValue(), MI);
-      else if (const auto &AI = getAtomicRmwInfo(MI))
-        Changed |= expandAtomicRmw(AI.getValue(), MI);
+      if (const auto &MOI = getAtomicFenceInfo(MI))
+        Changed |= expandAtomicFence(MOI.getValue(), MI);
+      else if (const auto &MOI = getLoadInfo(MI))
+        Changed |= expandLoad(MOI.getValue(), MI);
+      else if (const auto &MOI = getStoreInfo(MI))
+        Changed |= expandStore(MOI.getValue(), MI);
+      else if (const auto &MOI = getAtomicCmpxchgInfo(MI))
+        Changed |= expandAtomicCmpxchg(MOI.getValue(), MI);
+      else if (const auto &MOI = getAtomicRmwInfo(MI))
+        Changed |= expandAtomicRmw(MOI.getValue(), MI);
     }
   }