Index: lib/Target/AMDGPU/AMDGPUInstructions.td
===================================================================
--- lib/Target/AMDGPU/AMDGPUInstructions.td
+++ lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -222,6 +222,10 @@
 // Load/Store Pattern Fragments
 //===----------------------------------------------------------------------===//
 
+class Aligned8Bytes <dag ops, dag frag> : PatFrag <ops, frag, [{
+  return cast<MemSDNode>(N)->getAlignment() % 8 == 0;
+}]>;
+
 class LoadFrag <SDPatternOperator op> : PatFrag<(ops node:$ptr), (op node:$ptr)>;
 
 class StoreFrag<SDPatternOperator op> : PatFrag <
@@ -236,6 +240,10 @@
   return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.PRIVATE_ADDRESS;
 }]>;
 
+class ConstantAddress : CodePatPred<[{
+  return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS;
+}]>;
+
 class LocalAddress : CodePatPred<[{
   return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS;
 }]>;
@@ -247,72 +255,16 @@
 class FlatLoadAddress : CodePatPred<[{
   const auto AS = cast<MemSDNode>(N)->getAddressSpace();
   return AS == AMDGPUASI.FLAT_ADDRESS ||
-         AS == AMDGPUASI.GLOBAL_ADDRESS;
-}]>;
-
-
-def load_private : LoadFrag <load>, PrivateAddress;
-def truncstorei8_private : StoreFrag<truncstorei8>, PrivateAddress;
-def truncstorei16_private : StoreFrag <truncstorei16>, PrivateAddress;
-def store_private : StoreFrag <store>, PrivateAddress;
-
-def store_private_hi16 : StoreHi16 <truncstorei16>, PrivateAddress;
-def truncstorei8_private_hi16 : StoreHi16<truncstorei8>, PrivateAddress;
-
-
-class GlobalMemOp <dag ops, dag frag> : PatFrag <ops, frag, [{
-  return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS;
-}]>;
-
-// Global address space loads
-class GlobalLoad <SDPatternOperator op> : GlobalMemOp <
-  (ops node:$ptr), (op node:$ptr)
->;
-
-def global_load : GlobalLoad <load>;
-def global_atomic_load : GlobalLoad<atomic_load>;
-
-// Global address space stores
-class GlobalStore <SDPatternOperator op> : GlobalMemOp <
-  (ops node:$value, node:$ptr), (op node:$value, node:$ptr)
->;
-
-def global_store : GlobalStore <store>;
-def global_store_atomic : GlobalStore<atomic_store>;
-
-
-class ConstantMemOp <dag ops, dag frag> : PatFrag <ops, frag, [{
-  return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS;
+         AS == AMDGPUASI.GLOBAL_ADDRESS ||
+         AS == AMDGPUASI.CONSTANT_ADDRESS;
 }]>;
 
-// Constant address space loads
-class ConstantLoad <SDPatternOperator op> : ConstantMemOp <
-  (ops node:$ptr), (op node:$ptr)
->;
-
-def constant_load : ConstantLoad<load>;
-
-class LocalMemOp <dag ops, dag frag> : PatFrag <ops, frag, [{
-  return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS;
-}]>;
-
-// Local address space loads
-class LocalLoad <SDPatternOperator op> : LocalMemOp <
-  (ops node:$ptr), (op node:$ptr)
->;
-
-class LocalStore <SDPatternOperator op> : LocalMemOp <
-  (ops node:$value, node:$ptr), (op node:$value, node:$ptr)
->;
-
-class FlatMemOp <dag ops, dag frag> : PatFrag <ops, frag, [{
-  return cast<MemSDNode>(N)->getAddressSPace() == AMDGPUASI.FLAT_ADDRESS;
+class FlatStoreAddress : CodePatPred<[{
+  const auto AS = cast<MemSDNode>(N)->getAddressSpace();
+  return AS == AMDGPUASI.FLAT_ADDRESS ||
+         AS == AMDGPUASI.GLOBAL_ADDRESS;
 }]>;
 
-class FlatLoad <SDPatternOperator op> : FlatMemOp <
-  (ops node:$ptr), (op node:$ptr)
->;
-
 class AZExtLoadBase <SDPatternOperator ld_node>: PatFrag<(ops node:$ptr),
                                               (ld_node node:$ptr), [{
   LoadSDNode *L = cast<LoadSDNode>(N);
@@ -326,71 +278,98 @@
   return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8;
 }]>;
 
-def az_extloadi8_global : GlobalLoad <az_extloadi8>;
-def sextloadi8_global : GlobalLoad <sextloadi8>;
+def az_extloadi16 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{
+  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
 
-def az_extloadi8_constant : ConstantLoad <az_extloadi8>;
-def sextloadi8_constant : ConstantLoad <sextloadi8>;
+def az_extloadi32 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{
+  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i32;
+}]>;
 
-def az_extloadi8_local : LocalLoad <az_extloadi8>;
-def sextloadi8_local : LocalLoad <sextloadi8>;
+class PrivateLoad <SDPatternOperator op> : LoadFrag <op>, PrivateAddress;
+class PrivateStore <SDPatternOperator op> : StoreFrag <op>, PrivateAddress;
 
-def store_local_hi16 : StoreHi16 <truncstorei16>, LocalAddress;
-def truncstorei8_local_hi16 : StoreHi16<truncstorei8>, LocalAddress;
+class LocalLoad <SDPatternOperator op> : LoadFrag <op>, LocalAddress;
+class LocalStore <SDPatternOperator op> : StoreFrag <op>, LocalAddress;
 
+class GlobalLoad <SDPatternOperator op> : LoadFrag<op>, GlobalAddress;
+class GlobalStore <SDPatternOperator op> : StoreFrag<op>, GlobalAddress;
 
-def extloadi8_private : LoadFrag <az_extloadi8>, PrivateAddress;
-def sextloadi8_private : LoadFrag <sextloadi8>, PrivateAddress;
+class FlatLoad <SDPatternOperator op> : LoadFrag <op>, FlatLoadAddress;
+class FlatStore <SDPatternOperator op> : StoreFrag <op>, FlatStoreAddress;
 
-def az_extloadi16 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{
-  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16;
-}]>;
+class ConstantLoad <SDPatternOperator op> : LoadFrag <op>, ConstantAddress;
 
-def az_extloadi16_global : GlobalLoad <az_extloadi16>;
-def sextloadi16_global : GlobalLoad <sextloadi16>;
 
-def az_extloadi16_constant : ConstantLoad <az_extloadi16>;
-def sextloadi16_constant : ConstantLoad <sextloadi16>;
+def load_private : PrivateLoad <load>;
+def extloadi8_private : PrivateLoad <az_extloadi8>;
+def sextloadi8_private : PrivateLoad <sextloadi8>;
+def extloadi16_private : PrivateLoad <az_extloadi16>;
+def sextloadi16_private : PrivateLoad <sextloadi16>;
 
-def az_extloadi16_local : LocalLoad <az_extloadi16>;
-def sextloadi16_local : LocalLoad <sextloadi16>;
+def store_private : PrivateStore <store>;
+def truncstorei8_private : PrivateStore<truncstorei8>;
+def truncstorei16_private : PrivateStore <truncstorei16>;
+def store_hi16_private : StoreHi16 <truncstorei16>, PrivateAddress;
+def truncstorei8_hi16_private : StoreHi16<truncstorei8>, PrivateAddress;
 
-def extloadi16_private : LoadFrag <az_extloadi16>, PrivateAddress;
-def sextloadi16_private : LoadFrag <sextloadi16>, PrivateAddress;
 
-def az_extloadi32 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{
-  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i32;
-}]>;
+def load_global : GlobalLoad <load>;
+def sextloadi8_global : GlobalLoad <sextloadi8>;
+def az_extloadi8_global : GlobalLoad <az_extloadi8>;
+def sextloadi16_global : GlobalLoad <sextloadi16>;
+def az_extloadi16_global : GlobalLoad <az_extloadi16>;
+def atomic_load_global : GlobalLoad<atomic_load>;
 
-def az_extloadi32_global : GlobalLoad <az_extloadi32>;
+def store_global : GlobalStore <store>;
+def truncstorei8_global : GlobalStore <truncstorei8>;
+def truncstorei16_global : GlobalStore <truncstorei16>;
+def store_atomic_global : GlobalStore<atomic_store>;
+def truncstorei8_hi16_global : StoreHi16 <truncstorei8>, GlobalAddress;
+def truncstorei16_hi16_global : StoreHi16 <truncstorei16>, GlobalAddress;
 
-def az_extloadi32_flat : FlatLoad <az_extloadi32>;
+def load_local : LocalLoad <load>;
+def az_extloadi8_local : LocalLoad <az_extloadi8>;
+def sextloadi8_local : LocalLoad <sextloadi8>;
+def az_extloadi16_local : LocalLoad <az_extloadi16>;
+def sextloadi16_local : LocalLoad <sextloadi16>;
 
-def az_extloadi32_constant : ConstantLoad <az_extloadi32>;
+def store_local : LocalStore <store>;
+def truncstorei8_local : LocalStore <truncstorei8>;
+def truncstorei16_local : LocalStore <truncstorei16>;
+def store_local_hi16 : StoreHi16 <truncstorei16>, LocalAddress;
+def truncstorei8_local_hi16 : StoreHi16<truncstorei8>, LocalAddress;
 
-def truncstorei8_global : GlobalStore <truncstorei8>;
-def truncstorei16_global : GlobalStore <truncstorei16>;
+def load_align8_local : Aligned8Bytes <
+  (ops node:$ptr), (load_local node:$ptr)
+>;
 
-def truncstorei8_global_hi16 : StoreHi16 <truncstorei8>, GlobalAddress;
-def truncstorei16_global_hi16 : StoreHi16 <truncstorei16>, GlobalAddress;
+def store_align8_local : Aligned8Bytes <
+  (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr)
+>;
 
-def local_store : StoreFrag <store>, LocalAddress;
-def truncstorei8_local : StoreFrag <truncstorei8>, LocalAddress;
-def truncstorei16_local : StoreFrag <truncstorei16>, LocalAddress;
 
-def local_load : LocalLoad <load>;
+def load_flat          : FlatLoad <load>;
+def az_extloadi8_flat  : FlatLoad <az_extloadi8>;
+def sextloadi8_flat    : FlatLoad <sextloadi8>;
+def az_extloadi16_flat : FlatLoad <az_extloadi16>;
+def sextloadi16_flat   : FlatLoad <sextloadi16>;
+def atomic_load_flat   : FlatLoad<atomic_load>;
 
-class Aligned8Bytes <dag ops, dag frag> : PatFrag <ops, frag, [{
-    return cast<MemSDNode>(N)->getAlignment() % 8 == 0;
-}]>;
+def store_flat         : FlatStore <store>;
+def truncstorei8_flat  : FlatStore <truncstorei8>;
+def truncstorei16_flat : FlatStore <truncstorei16>;
+def atomic_store_flat  : FlatStore <atomic_store>;
+def truncstorei8_hi16_flat  : StoreHi16<truncstorei8>, FlatStoreAddress;
+def truncstorei16_hi16_flat : StoreHi16<truncstorei16>, FlatStoreAddress;
 
-def local_load_aligned8bytes : Aligned8Bytes <
-  (ops node:$ptr), (local_load node:$ptr)
->;
 
-def local_store_aligned8bytes : Aligned8Bytes <
-  (ops node:$val, node:$ptr), (local_store node:$val, node:$ptr)
->;
+def constant_load : ConstantLoad<load>;
+def sextloadi8_constant : ConstantLoad <sextloadi8>;
+def az_extloadi8_constant : ConstantLoad <az_extloadi8>;
+def sextloadi16_constant : ConstantLoad <sextloadi16>;
+def az_extloadi16_constant : ConstantLoad <az_extloadi16>;
+
 
 class local_binary_atomic_op<SDNode atomic_op> :
   PatFrag<(ops node:$ptr, node:$value),
@@ -398,7 +377,6 @@
   return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS;
 }]>;
 
-
 def atomic_swap_local : local_binary_atomic_op<atomic_swap>;
 def atomic_load_add_local : local_binary_atomic_op<atomic_load_add>;
 def atomic_load_sub_local : local_binary_atomic_op<atomic_load_sub>;
@@ -465,26 +443,25 @@
 defm atomic_umin_global : global_binary_atomic_op<atomic_load_umin>;
 defm atomic_xor_global : global_binary_atomic_op<atomic_load_xor>;
 
-//legacy
+// Legacy.
 def AMDGPUatomic_cmp_swap_global : PatFrag<
-        (ops node:$ptr, node:$value),
-        (AMDGPUatomic_cmp_swap node:$ptr, node:$value),
-        [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS;}]>;
+  (ops node:$ptr, node:$value),
+  (AMDGPUatomic_cmp_swap node:$ptr, node:$value)>, GlobalAddress;
 
 def atomic_cmp_swap_global : PatFrag<
-      (ops node:$ptr, node:$cmp, node:$value),
-      (atomic_cmp_swap node:$ptr, node:$cmp, node:$value),
-      [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS;}]>;
+  (ops node:$ptr, node:$cmp, node:$value),
+  (atomic_cmp_swap node:$ptr, node:$cmp, node:$value)>, GlobalAddress;
+
 
 def atomic_cmp_swap_global_noret : PatFrag<
-      (ops node:$ptr, node:$cmp, node:$value),
-      (atomic_cmp_swap node:$ptr, node:$cmp, node:$value),
-      [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>;
+  (ops node:$ptr, node:$cmp, node:$value),
+  (atomic_cmp_swap node:$ptr, node:$cmp, node:$value),
+  [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>;
 
 def atomic_cmp_swap_global_ret : PatFrag<
-      (ops node:$ptr, node:$cmp, node:$value),
-      (atomic_cmp_swap node:$ptr, node:$cmp, node:$value),
-      [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>;
+  (ops node:$ptr, node:$cmp, node:$value),
+  (atomic_cmp_swap node:$ptr, node:$cmp, node:$value),
+  [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>;
 
 //===----------------------------------------------------------------------===//
 // Misc Pattern Fragments
Index: lib/Target/AMDGPU/BUFInstructions.td
===================================================================
--- lib/Target/AMDGPU/BUFInstructions.td
+++ lib/Target/AMDGPU/BUFInstructions.td
@@ -699,16 +699,16 @@
   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 >;
 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
-  "buffer_store_dword", VGPR_32, i32, global_store
+  "buffer_store_dword", VGPR_32, i32, store_global
 >;
 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
-  "buffer_store_dwordx2", VReg_64, v2i32, global_store
+  "buffer_store_dwordx2", VReg_64, v2i32, store_global
 >;
 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
-  "buffer_store_dwordx3", VReg_96, untyped, global_store
+  "buffer_store_dwordx3", VReg_96, untyped, store_global
 >;
 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
-  "buffer_store_dwordx4", VReg_128, v4i32, global_store
+  "buffer_store_dwordx4", VReg_128, v4i32, store_global
 >;
 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global
@@ -1219,8 +1219,8 @@
   >;
 }
 let Predicates = [isSICI] in {
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, global_store_atomic>;
-defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, global_store_atomic>;
+defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>;
+defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>;
 } // End Predicates = [isSICI]
 
 
@@ -1235,7 +1235,7 @@
 }
 
 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
-defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, global_store>;
+defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
 
 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
                                  MUBUF_Pseudo InstrOffset,
@@ -1266,8 +1266,8 @@
  // Hiding the extract high pattern in the PatFrag seems to not
  // automatically increase the complexity.
 let AddedComplexity = 1 in {
-defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_private_hi16>;
-defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_private_hi16>;
+defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
+defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
 }
 }
 
Index: lib/Target/AMDGPU/CaymanInstructions.td
===================================================================
--- lib/Target/AMDGPU/CaymanInstructions.td
+++ lib/Target/AMDGPU/CaymanInstructions.td
@@ -76,7 +76,7 @@
   CF_MEM_RAT_CACHELESS <0x14, 0, mask,
                         (ins rc:$rw_gpr, R600_TReg32_X:$index_gpr),
                         "STORE_DWORD $rw_gpr, $index_gpr",
-                        [(global_store vt:$rw_gpr, i32:$index_gpr)]> {
+                        [(store_global vt:$rw_gpr, i32:$index_gpr)]> {
   let eop = 0; // This bit is not used on Cayman.
 }
 
Index: lib/Target/AMDGPU/DSInstructions.td
===================================================================
--- lib/Target/AMDGPU/DSInstructions.td
+++ lib/Target/AMDGPU/DSInstructions.td
@@ -543,25 +543,25 @@
   (inst $ptr, (as_i16imm $offset), (i1 0))
 >;
 
-def : DSReadPat <DS_READ_I8,  i32, si_sextload_local_i8>;
-def : DSReadPat <DS_READ_U8,  i32, si_az_extload_local_i8>;
-def : DSReadPat <DS_READ_I8,  i16, si_sextload_local_i8>;
-def : DSReadPat <DS_READ_U8,  i16, si_az_extload_local_i8>;
-def : DSReadPat <DS_READ_I16, i32, si_sextload_local_i16>;
-def : DSReadPat <DS_READ_I16, i32, si_sextload_local_i16>;
-def : DSReadPat <DS_READ_U16, i32, si_az_extload_local_i16>;
-def : DSReadPat <DS_READ_U16, i16, si_load_local>;
-def : DSReadPat <DS_READ_B32, i32, si_load_local>;
+def : DSReadPat <DS_READ_I8,  i32, sextloadi8_local_m0>;
+def : DSReadPat <DS_READ_U8,  i32, az_extloadi8_local_m0>;
+def : DSReadPat <DS_READ_I8,  i16, sextloadi8_local_m0>;
+def : DSReadPat <DS_READ_U8,  i16, az_extloadi8_local_m0>;
+def : DSReadPat <DS_READ_I16, i32, sextloadi16_local_m0>;
+def : DSReadPat <DS_READ_I16, i32, sextloadi16_local_m0>;
+def : DSReadPat <DS_READ_U16, i32, az_extloadi16_local_m0>;
+def : DSReadPat <DS_READ_U16, i16, load_local_m0>;
+def : DSReadPat <DS_READ_B32, i32, load_local_m0>;
 
 let AddedComplexity = 100 in {
 
-def : DSReadPat <DS_READ_B64, v2i32, si_load_local_align8>;
+def : DSReadPat <DS_READ_B64, v2i32, load_align8_local_m0>;
 
 } // End AddedComplexity = 100
 
 def : Pat <
-  (v2i32 (si_load_local (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
-                                                    i8:$offset1))),
+  (v2i32 (load_local_m0 (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
+                                                       i8:$offset1))),
   (DS_READ2_B32 $ptr, $offset0, $offset1, (i1 0))
 >;
 
@@ -570,11 +570,11 @@
   (inst $ptr, $value, (as_i16imm $offset), (i1 0))
 >;
 
-def : DSWritePat <DS_WRITE_B8, i32, si_truncstore_local_i8>;
-def : DSWritePat <DS_WRITE_B16, i32, si_truncstore_local_i16>;
-def : DSWritePat <DS_WRITE_B8, i16, si_truncstore_local_i8>;
-def : DSWritePat <DS_WRITE_B16, i16, si_store_local>;
-def : DSWritePat <DS_WRITE_B32, i32, si_store_local>;
+def : DSWritePat <DS_WRITE_B8, i32, truncstorei8_local_m0>;
+def : DSWritePat <DS_WRITE_B16, i32, truncstorei16_local_m0>;
+def : DSWritePat <DS_WRITE_B8, i16, truncstorei8_local_m0>;
+def : DSWritePat <DS_WRITE_B16, i16, store_local_m0>;
+def : DSWritePat <DS_WRITE_B32, i32, store_local_m0>;
 
 let Predicates = [HasD16LoadStore] in {
 def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_local_hi16>;
@@ -583,11 +583,11 @@
 
 let AddedComplexity = 100 in {
 
-def : DSWritePat <DS_WRITE_B64, v2i32, si_store_local_align8>;
+def : DSWritePat <DS_WRITE_B64, v2i32, store_align8_local_m0>;
 } // End AddedComplexity = 100
 
 def : Pat <
-  (si_store_local v2i32:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
+  (store_local_m0 v2i32:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0,
                                                                i8:$offset1)),
   (DS_WRITE2_B32 $ptr, (i32 (EXTRACT_SUBREG $value, sub0)),
                        (i32 (EXTRACT_SUBREG $value, sub1)), $offset0, $offset1,
Index: lib/Target/AMDGPU/EvergreenInstructions.td
===================================================================
--- lib/Target/AMDGPU/EvergreenInstructions.td
+++ lib/Target/AMDGPU/EvergreenInstructions.td
@@ -128,21 +128,21 @@
 def RAT_WRITE_CACHELESS_32_eg : CF_MEM_RAT_CACHELESS <0x2, 0, 0x1,
   (ins R600_TReg32_X:$rw_gpr, R600_TReg32_X:$index_gpr, InstFlag:$eop),
   "STORE_RAW $rw_gpr, $index_gpr, $eop",
-  [(global_store i32:$rw_gpr, i32:$index_gpr)]
+  [(store_global i32:$rw_gpr, i32:$index_gpr)]
 >;
 
 // 64-bit store
 def RAT_WRITE_CACHELESS_64_eg : CF_MEM_RAT_CACHELESS <0x2, 0, 0x3,
   (ins R600_Reg64:$rw_gpr, R600_TReg32_X:$index_gpr, InstFlag:$eop),
   "STORE_RAW $rw_gpr.XY, $index_gpr, $eop",
-  [(global_store v2i32:$rw_gpr, i32:$index_gpr)]
+  [(store_global v2i32:$rw_gpr, i32:$index_gpr)]
 >;
 
 //128-bit store
 def RAT_WRITE_CACHELESS_128_eg : CF_MEM_RAT_CACHELESS <0x2, 0, 0xf,
   (ins R600_Reg128:$rw_gpr, R600_TReg32_X:$index_gpr, InstFlag:$eop),
   "STORE_RAW $rw_gpr.XYZW, $index_gpr, $eop",
-  [(global_store v4i32:$rw_gpr, i32:$index_gpr)]
+  [(store_global v4i32:$rw_gpr, i32:$index_gpr)]
 >;
 
 def RAT_STORE_TYPED_eg: CF_MEM_RAT_STORE_TYPED<1>;
@@ -614,7 +614,7 @@
 def LDS_MIN_UINT : R600_LDS_1A1D_NORET <0x7, "LDS_MIN_UINT", [] >;
 def LDS_MAX_UINT : R600_LDS_1A1D_NORET <0x8, "LDS_MAX_UINT", [] >;
 def LDS_WRITE : R600_LDS_1A1D_NORET <0xD, "LDS_WRITE",
-  [(local_store (i32 R600_Reg32:$src1), R600_Reg32:$src0)]
+  [(store_local (i32 R600_Reg32:$src1), R600_Reg32:$src0)]
 >;
 def LDS_BYTE_WRITE : R600_LDS_1A1D_NORET<0x12, "LDS_BYTE_WRITE",
   [(truncstorei8_local i32:$src1, i32:$src0)]
@@ -656,7 +656,7 @@
   [(set i32:$dst, (atomic_cmp_swap_32_local i32:$src0, i32:$src1, i32:$src2))]
 >;
 def LDS_READ_RET : R600_LDS_1A <0x32, "LDS_READ_RET",
-  [(set (i32 R600_Reg32:$dst), (local_load R600_Reg32:$src0))]
+  [(set (i32 R600_Reg32:$dst), (load_local R600_Reg32:$src0))]
 >;
 def LDS_BYTE_READ_RET : R600_LDS_1A <0x36, "LDS_BYTE_READ_RET",
   [(set i32:$dst, (sextloadi8_local i32:$src0))]
Index: lib/Target/AMDGPU/FLATInstructions.td
===================================================================
--- lib/Target/AMDGPU/FLATInstructions.td
+++ lib/Target/AMDGPU/FLATInstructions.td
@@ -620,37 +620,6 @@
 // Flat Patterns
 //===----------------------------------------------------------------------===//
 
-class flat_ld <SDPatternOperator ld> : PatFrag<(ops node:$ptr),
-                                               (ld node:$ptr), [{
-  auto const AS = cast<MemSDNode>(N)->getAddressSpace();
-  return AS == AMDGPUASI.FLAT_ADDRESS ||
-         AS == AMDGPUASI.GLOBAL_ADDRESS ||
-         AS == AMDGPUASI.CONSTANT_ADDRESS;
-}]>;
-
-class flat_st <SDPatternOperator st> : PatFrag<(ops node:$val, node:$ptr),
-                                               (st node:$val, node:$ptr), [{
-  auto const AS = cast<MemSDNode>(N)->getAddressSpace();
-  return AS == AMDGPUASI.FLAT_ADDRESS ||
-         AS == AMDGPUASI.GLOBAL_ADDRESS;
-}]>;
-
-def atomic_flat_load   : flat_ld <atomic_load>;
-def flat_load          : flat_ld <load>;
-def flat_az_extloadi8  : flat_ld <az_extloadi8>;
-def flat_sextloadi8    : flat_ld <sextloadi8>;
-def flat_az_extloadi16 : flat_ld <az_extloadi16>;
-def flat_sextloadi16   : flat_ld <sextloadi16>;
-
-def atomic_flat_store  : flat_st <atomic_store>;
-def flat_store         : flat_st <store>;
-def flat_truncstorei8  : flat_st <truncstorei8>;
-def flat_truncstorei16 : flat_st <truncstorei16>;
-
-def flat_truncstorei8_hi16  : StoreHi16<truncstorei8>, FlatLoadAddress;
-def flat_truncstorei16_hi16 : StoreHi16<truncstorei16>, FlatLoadAddress;
-
-
 // Patterns for global loads with no offset.
 class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : Pat <
   (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))),
@@ -705,27 +674,27 @@
 
 let Predicates = [HasFlatAddressSpace] in {
 
-def : FlatLoadPat <FLAT_LOAD_UBYTE, flat_az_extloadi8, i32>;
-def : FlatLoadPat <FLAT_LOAD_SBYTE, flat_sextloadi8, i32>;
-def : FlatLoadPat <FLAT_LOAD_UBYTE, flat_az_extloadi8, i16>;
-def : FlatLoadPat <FLAT_LOAD_SBYTE, flat_sextloadi8, i16>;
-def : FlatLoadPat <FLAT_LOAD_USHORT, flat_az_extloadi16, i32>;
-def : FlatLoadPat <FLAT_LOAD_SSHORT, flat_sextloadi16, i32>;
-def : FlatLoadPat <FLAT_LOAD_DWORD, flat_load, i32>;
-def : FlatLoadPat <FLAT_LOAD_DWORDX2, flat_load, v2i32>;
-def : FlatLoadPat <FLAT_LOAD_DWORDX4, flat_load, v4i32>;
+def : FlatLoadPat <FLAT_LOAD_UBYTE, az_extloadi8_flat, i32>;
+def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>;
+def : FlatLoadPat <FLAT_LOAD_UBYTE, az_extloadi8_flat, i16>;
+def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>;
+def : FlatLoadPat <FLAT_LOAD_USHORT, az_extloadi16_flat, i32>;
+def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>;
+def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, i32>;
+def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, v2i32>;
+def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, v4i32>;
 
-def : FlatLoadAtomicPat <FLAT_LOAD_DWORD, atomic_flat_load, i32>;
-def : FlatLoadAtomicPat <FLAT_LOAD_DWORDX2, atomic_flat_load, i64>;
+def : FlatLoadAtomicPat <FLAT_LOAD_DWORD, atomic_load_flat, i32>;
+def : FlatLoadAtomicPat <FLAT_LOAD_DWORDX2, atomic_load_flat, i64>;
 
-def : FlatStorePat <FLAT_STORE_BYTE, flat_truncstorei8, i32>;
-def : FlatStorePat <FLAT_STORE_SHORT, flat_truncstorei16, i32>;
-def : FlatStorePat <FLAT_STORE_DWORD, flat_store, i32>;
-def : FlatStorePat <FLAT_STORE_DWORDX2, flat_store, v2i32>;
-def : FlatStorePat <FLAT_STORE_DWORDX4, flat_store, v4i32>;
+def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
+def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>;
+def : FlatStorePat <FLAT_STORE_DWORD, store_flat, i32>;
+def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, v2i32>;
+def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, v4i32>;
 
-def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_flat_store, i32>;
-def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_flat_store, i64>;
+def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat, i32>;
+def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat, i64>;
 
 def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_add_global, i32>;
 def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_sub_global, i32>;
@@ -755,12 +724,12 @@
 def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>;
 def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_xor_global, i64>;
 
-def : FlatStorePat <FLAT_STORE_BYTE, flat_truncstorei8, i16>;
-def : FlatStorePat <FLAT_STORE_SHORT, flat_store, i16>;
+def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>;
+def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>;
 
  let Predicates = [HasD16LoadStore] in {
-def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, flat_truncstorei16_hi16, i32>;
-def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, flat_truncstorei8_hi16, i32>;
+def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>;
+def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>;
 }
 
 } // End Predicates = [HasFlatAddressSpace]
@@ -775,30 +744,28 @@
 def : FlatLoadSignedPat <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>;
 
 
-def : FlatLoadSignedPat <GLOBAL_LOAD_DWORD, global_load, i32>;
-def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX2, global_load, v2i32>;
-def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX4, global_load, v4i32>;
+def : FlatLoadSignedPat <GLOBAL_LOAD_DWORD, load_global, i32>;
+def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX2, load_global, v2i32>;
+def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX4, load_global, v4i32>;
 
-def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORD, global_atomic_load, i32>;
-def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORDX2, global_atomic_load, i64>;
+def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORD, atomic_load_global, i32>;
+def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORDX2, atomic_load_global, i64>;
 
 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i16>;
 def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
-def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, global_store, i16>;
-def : FlatStoreSignedPat <GLOBAL_STORE_DWORD, global_store, i32>;
-def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX2, global_store, v2i32>;
-def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX4, global_store, v4i32>;
-
+def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, store_global, i16>;
+def : FlatStoreSignedPat <GLOBAL_STORE_DWORD, store_global, i32>;
+def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX2, store_global, v2i32>;
+def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX4, store_global, v4i32>;
 
  let Predicates = [HasD16LoadStore] in {
-def : FlatStoreSignedPat <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_global_hi16, i32>;
-def : FlatStoreSignedPat <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_global_hi16, i32>;
+def : FlatStoreSignedPat <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>;
+def : FlatStoreSignedPat <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>;
 }
 
-
-def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORD, global_store_atomic, i32>;
-def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORDX2, global_store_atomic, i64>;
+def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORD, store_atomic_global, i32>;
+def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORDX2, store_atomic_global, i64>;
 
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_RTN, atomic_add_global, i32>;
 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SUB_RTN, atomic_sub_global, i32>;
Index: lib/Target/AMDGPU/SIInstrInfo.td
===================================================================
--- lib/Target/AMDGPU/SIInstrInfo.td
+++ lib/Target/AMDGPU/SIInstrInfo.td
@@ -115,80 +115,109 @@
 defm atomic_dec_global : global_binary_atomic_op<SIatomic_dec>;
 
 //===----------------------------------------------------------------------===//
-// SDNodes and PatFrag for local loads and stores to enable s_mov_b32 m0, -1
-// to be glued to the memory instructions.
+// SDNodes PatFrags for loads/stores with a glue input.
+// This is for SDNodes and PatFrag for local loads and stores to
+// enable s_mov_b32 m0, -1 to be glued to the memory instructions.
+//
+// These mirror the regular load/store PatFrags and rely on special
+// processing during Select() to add the glued copy.
+//
 //===----------------------------------------------------------------------===//
 
-def SIld_local : SDNode <"ISD::LOAD", SDTLoad,
+def AMDGPUld_glue : SDNode <"ISD::LOAD", SDTLoad,
   [SDNPHasChain, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
 >;
 
-def si_ld_local : PatFrag <(ops node:$ptr), (SIld_local node:$ptr), [{
-  return cast<LoadSDNode>(N)->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS;
+def unindexedload_glue : PatFrag <(ops node:$ptr), (AMDGPUld_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
 }]>;
 
-def si_load_local : PatFrag <(ops node:$ptr), (si_ld_local node:$ptr), [{
-  return cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED &&
-         cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
+def load_glue : PatFrag <(ops node:$ptr), (unindexedload_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
 }]>;
 
-def si_load_local_align8 : Aligned8Bytes <
-  (ops node:$ptr), (si_load_local node:$ptr)
->;
+def extload_glue : PatFrag<(ops node:$ptr), (load_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
+}]>;
 
-def si_sextload_local : PatFrag <(ops node:$ptr), (si_ld_local node:$ptr), [{
+def sextload_glue : PatFrag<(ops node:$ptr), (unindexedload_glue node:$ptr), [{
   return cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
 }]>;
-def si_az_extload_local : AZExtLoadBase <si_ld_local>;
 
-multiclass SIExtLoadLocal <PatFrag ld_node> {
+def zextload_glue : PatFrag<(ops node:$ptr), (unindexedload_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
+}]>;
 
-  def _i8 : PatFrag <(ops node:$ptr), (ld_node node:$ptr),
-                     [{return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8;}]
-  >;
+def az_extload_glue : AZExtLoadBase <unindexedload_glue>;
 
-  def _i16 : PatFrag <(ops node:$ptr), (ld_node node:$ptr),
-                     [{return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16;}]
-  >;
-}
+def az_extloadi8_glue : PatFrag<(ops node:$ptr), (az_extload_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8;
+}]>;
+
+def az_extloadi16_glue : PatFrag<(ops node:$ptr), (az_extload_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
 
-defm si_sextload_local : SIExtLoadLocal <si_sextload_local>;
-defm si_az_extload_local : SIExtLoadLocal <si_az_extload_local>;
+def sextloadi8_glue : PatFrag<(ops node:$ptr), (sextload_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i8;
+}]>;
 
-def SIst_local : SDNode <"ISD::STORE", SDTStore,
+def sextloadi16_glue : PatFrag<(ops node:$ptr), (sextload_glue node:$ptr), [{
+  return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
+
+def load_glue_align8 : Aligned8Bytes <
+  (ops node:$ptr), (load_glue node:$ptr)
+>;
+
+
+def load_local_m0 : LoadFrag<load_glue>, LocalAddress;
+def sextloadi8_local_m0 : LoadFrag<sextloadi8_glue>, LocalAddress;
+def sextloadi16_local_m0 : LoadFrag<sextloadi16_glue>, LocalAddress;
+def az_extloadi8_local_m0 : LoadFrag<az_extloadi8_glue>, LocalAddress;
+def az_extloadi16_local_m0 : LoadFrag<az_extloadi16_glue>, LocalAddress;
+def load_align8_local_m0 : LoadFrag <load_glue_align8>, LocalAddress;
+
+
+def AMDGPUst_glue : SDNode <"ISD::STORE", SDTStore,
   [SDNPHasChain, SDNPMayStore, SDNPMemOperand, SDNPInGlue]
 >;
 
-def si_st_local : PatFrag <
-  (ops node:$val, node:$ptr), (SIst_local node:$val, node:$ptr), [{
-  return cast<StoreSDNode>(N)->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS;
+def unindexedstore_glue : PatFrag<(ops node:$val, node:$ptr),
+                                   (AMDGPUst_glue node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
 }]>;
 
-def si_store_local : PatFrag <
-  (ops node:$val, node:$ptr), (si_st_local node:$val, node:$ptr), [{
-  return cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED &&
-         !cast<StoreSDNode>(N)->isTruncatingStore();
+def store_glue : PatFrag<(ops node:$val, node:$ptr),
+                      (unindexedstore_glue node:$val, node:$ptr), [{
+  return !cast<StoreSDNode>(N)->isTruncatingStore();
 }]>;
 
-def si_store_local_align8 : Aligned8Bytes <
-  (ops node:$val, node:$ptr), (si_store_local node:$val, node:$ptr)
->;
-
-def si_truncstore_local : PatFrag <
-  (ops node:$val, node:$ptr), (si_st_local node:$val, node:$ptr), [{
+def truncstore_glue : PatFrag<(ops node:$val, node:$ptr),
+  (unindexedstore_glue node:$val, node:$ptr), [{
   return cast<StoreSDNode>(N)->isTruncatingStore();
 }]>;
 
-def si_truncstore_local_i8 : PatFrag <
-  (ops node:$val, node:$ptr), (si_truncstore_local node:$val, node:$ptr), [{
+def truncstorei8_glue : PatFrag<(ops node:$val, node:$ptr),
+                           (truncstore_glue node:$val, node:$ptr), [{
   return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i8;
 }]>;
 
-def si_truncstore_local_i16 : PatFrag <
-  (ops node:$val, node:$ptr), (si_truncstore_local node:$val, node:$ptr), [{
+def truncstorei16_glue : PatFrag<(ops node:$val, node:$ptr),
+                           (truncstore_glue node:$val, node:$ptr), [{
   return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i16;
 }]>;
 
+def store_glue_align8 : Aligned8Bytes <
+  (ops node:$value, node:$ptr), (store_glue node:$value, node:$ptr)
+>;
+
+def store_local_m0 : StoreFrag<store_glue>, LocalAddress;
+def truncstorei8_local_m0 : StoreFrag<truncstorei8_glue>, LocalAddress;
+def truncstorei16_local_m0 : StoreFrag<truncstorei16_glue>, LocalAddress;
+
+def store_align8_local_m0 : StoreFrag<store_glue_align8>, LocalAddress;
+
 def si_setcc_uniform : PatFrag <
   (ops node:$lhs, node:$rhs, node:$cond),
   (setcc node:$lhs, node:$rhs, node:$cond), [{