diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PriorityQueue.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AliasAnalysis.h"
@@ -1525,7 +1526,9 @@
   void apply(ScheduleDAGInstrs *DAGInstrs) override;
 
 protected:
-  void clusterNeighboringMemOps(ArrayRef<SUnit *> MemOps, ScheduleDAGInstrs *DAG);
+  void clusterNeighboringMemOps(ArrayRef<SUnit *> MemOps,
+                                ScheduleDAGInstrs *DAG,
+                                SmallBitVector &ClusteredUnits);
 };
 
 class StoreClusterMutation : public BaseMemOpClusterMutation {
@@ -1562,9 +1565,13 @@
 } // end namespace llvm
 
 void BaseMemOpClusterMutation::clusterNeighboringMemOps(
-    ArrayRef<SUnit *> MemOps, ScheduleDAGInstrs *DAG) {
+    ArrayRef<SUnit *> MemOps, ScheduleDAGInstrs *DAG,
+    SmallBitVector &ClusteredUnits) {
   SmallVector<MemOpInfo, 32> MemOpRecords;
   for (SUnit *SU : MemOps) {
+    // Skip those already clustered
+    if (ClusteredUnits.test(SU->NodeNum))
+      continue;
     SmallVector<const MachineOperand *, 4> BaseOps;
     int64_t Offset;
     bool OffsetIsScalable;
@@ -1604,6 +1611,8 @@
           DAG->addEdge(Succ.getSUnit(), SDep(SUb, SDep::Artificial));
         }
         ++ClusterLength;
+        ClusteredUnits.set(SUa->NodeNum);
+        ClusteredUnits.set(SUb->NodeNum);
       } else
         ClusterLength = 1;
     } else
@@ -1632,9 +1641,16 @@
     Chain.push_back(&SU);
   }
 
-  // Iterate over the store chains.
-  for (auto &SCD : StoreChains)
-    clusterNeighboringMemOps(SCD.second, DAG);
+  // Iterate over the store chains. Each time, insert units without any ctrl
+  // preds into other groups.
+  SmallBitVector ClusteredUnits(DAG->SUnits.size());
+  const auto &Free = StoreChains.FindAndConstruct(DAG->SUnits.size()).second;
+  for (auto &SCD : StoreChains) {
+    if (SCD.first != DAG->SUnits.size())
+      for (SUnit *S : Free)
+        SCD.second.push_back(S);
+    clusterNeighboringMemOps(SCD.second, DAG, ClusteredUnits);
+  }
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll b/llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll
--- a/llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll
+++ b/llvm/test/CodeGen/AArch64/overeager_mla_fusing.ll
@@ -3,15 +3,17 @@
 
 define dso_local void @jsimd_idct_ifast_neon_intrinsic(i8* nocapture readonly %dct_table, i16* nocapture readonly %coef_block, i8** nocapture readonly %output_buf, i32 %output_col) local_unnamed_addr #0 {
 ; CHECK-LABEL: jsimd_idct_ifast_neon_intrinsic:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       .Ljsimd_idct_ifast_neon_intrinsic$local:
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ldr q0, [x1, #32]
-; CHECK-NEXT:    ldr q1, [x1, #96]
-; CHECK-NEXT:    ldr q2, [x0, #32]
+; CHECK-NEXT:    ldr q1, [x0, #32]
+; CHECK-NEXT:    ldr q2, [x1, #96]
 ; CHECK-NEXT:    ldr q3, [x0, #96]
 ; CHECK-NEXT:    ldr x8, [x2, #48]
+; CHECK-NEXT:    mul v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    mov w9, w3
-; CHECK-NEXT:    mul v0.8h, v2.8h, v0.8h
-; CHECK-NEXT:    mul v1.8h, v3.8h, v1.8h
+; CHECK-NEXT:    mul v1.8h, v3.8h, v2.8h
 ; CHECK-NEXT:    add v2.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    str q2, [x8, x9]
 ; CHECK-NEXT:    ldr x8, [x2, #56]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
@@ -1595,19 +1595,19 @@
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX9-NEXT:    s_load_dword s4, s[4:5], 0x10
-; GFX9-NEXT:    v_mov_b32_e32 v0, 42
+; GFX9-NEXT:    v_mov_b32_e32 v4, 42
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-NEXT:    ds_inc_rtn_u32 v4, v1, v0
-; GFX9-NEXT:    ds_inc_rtn_u32 v5, v1, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-NEXT:    ds_inc_rtn_u32 v6, v5, v4
+; GFX9-NEXT:    ds_inc_rtn_u32 v4, v5, v4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX9-NEXT:    global_store_dword v[0:1], v4, off
+; GFX9-NEXT:    global_store_dword v[0:1], v6, off
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_store_dword v[2:3], v5, off
+; GFX9-NEXT:    global_store_dword v[2:3], v4, off
 ; GFX9-NEXT:    s_endpgm
   %result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
   %result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll
@@ -14,9 +14,9 @@
 ; GFX9-NEXT:    s_cbranch_scc0 BB0_2
 ; GFX9-NEXT:  ; %bb.1: ; %bb0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7b
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x1c8
 ; GFX9-NEXT:    global_store_dword v[0:1], v0, off
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x1c8
-; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3e7
 ; GFX9-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3e8
@@ -28,9 +28,9 @@
 ; GFX9-NEXT:    s_endpgm
 ; GFX9-NEXT:  BB0_2: ; %bb1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x5be6
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x1c7
 ; GFX9-NEXT:    global_store_dword v[0:1], v0, off
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x1c7
-; GFX9-NEXT:    global_store_dword v[0:1], v0, off
+; GFX9-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3e8
 ; GFX9-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x1c8
diff --git a/llvm/test/CodeGen/AMDGPU/bitcast-vector-extract.ll b/llvm/test/CodeGen/AMDGPU/bitcast-vector-extract.ll
--- a/llvm/test/CodeGen/AMDGPU/bitcast-vector-extract.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitcast-vector-extract.ll
@@ -7,7 +7,6 @@
 ; GCN-LABEL: {{^}}store_bitcast_constant_v8i32_to_v8f32:
 ; GCN: buffer_store_dwordx4
 ; GCN: buffer_store_dwordx4
-; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
 ; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
@@ -23,7 +22,6 @@
 ; GCN-LABEL: {{^}}store_bitcast_constant_v4i64_to_v8f32:
 ; GCN: buffer_store_dwordx4
 ; GCN: buffer_store_dwordx4
-; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
 ; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
@@ -39,7 +37,6 @@
 ; GCN-LABEL: {{^}}store_bitcast_constant_v4i64_to_v4f64:
 ; GCN: buffer_store_dwordx4
 ; GCN: buffer_store_dwordx4
-; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
 ; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
@@ -55,7 +52,6 @@
 ; GCN-LABEL: {{^}}store_bitcast_constant_v8i32_to_v16i16:
 ; GCN: buffer_store_dwordx4
 ; GCN: buffer_store_dwordx4
-; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
 ; GCN-NOT: v_mov_b32
 ; GCN: buffer_store_dwordx4
diff --git a/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll b/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
--- a/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
@@ -491,12 +491,12 @@
 ; GCN: enable_vgpr_workitem_id = 0
 ; GCN-DAG: s_mov_b32 s33, s7
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
+; GCN: s_add_u32 s32, s33, 0x400{{$}}
 ; GCN: buffer_store_dword [[K]], off, s[0:3], s33 offset:4
+; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s33 offset:4
-; GCN: s_add_u32 s32, s33, 0x400{{$}}
 
 ; GCN-NOT: s32
-; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
 
 ; GCN: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32{{$}}
 ; GCN: v_mov_b32_e32 [[RELOAD_BYVAL]],
@@ -520,8 +520,8 @@
 ; GCN-LABEL: {{^}}func_call_too_many_args_use_workitem_id_x_byval:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
 ; GCN: buffer_store_dword [[K]], off, s[0:3], s34{{$}}
-; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s34{{$}}
 ; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
+; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s34{{$}}
 ; GCN: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32{{$}}
 ; GCN: v_mov_b32_e32 [[RELOAD_BYVAL]],
 ; GCN: s_swappc_b64
diff --git a/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll b/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll
--- a/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll
+++ b/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll
@@ -51,8 +51,8 @@
 ; Same frame index is used multiple times in the store
 ; GCN-LABEL: {{^}}stored_fi_to_self:
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x4d2{{$}}
-; GCN: buffer_store_dword [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}}
 ; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 4{{$}}
+; GCN: buffer_store_dword [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}}
 ; GCN: buffer_store_dword [[ZERO]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}}
 define amdgpu_kernel void @stored_fi_to_self() #0 {
   %tmp = alloca i32 addrspace(5)*, addrspace(5)
@@ -66,9 +66,9 @@
 
 ; GCN-LABEL: {{^}}stored_fi_to_self_offset:
 ; GCN-DAG: v_mov_b32_e32 [[K0:v[0-9]+]], 32{{$}}
-; GCN: buffer_store_dword [[K0]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}}
-
 ; GCN-DAG: v_mov_b32_e32 [[K1:v[0-9]+]], 0x4d2{{$}}
+
+; GCN: buffer_store_dword [[K0]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}}
 ; GCN: buffer_store_dword [[K1]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:2052{{$}}
 
 ; GCN: v_mov_b32_e32 [[OFFSETK:v[0-9]+]], 0x804{{$}}
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -364,13 +364,13 @@
 ; GCN-LABEL: chain_hi_to_lo_group_may_alias_store:
 ; GCN:       ; %bb.0: ; %bb
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v3, 0x7b
-; GCN-NEXT:    ds_read_u16 v2, v0
-; GCN-NEXT:    ds_write_b16 v1, v3
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x7b
+; GCN-NEXT:    ds_read_u16 v3, v0
+; GCN-NEXT:    ds_write_b16 v1, v2
 ; GCN-NEXT:    ds_read_u16 v0, v0 offset:2
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GCN-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
+; GCN-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
@@ -98,8 +98,8 @@
 ; GCN: s_load_dword [[VAL:s[0-9]+]]
 ; GCN-NOT: {{s|flat|buffer|global}}_load
 ; GCN: s_lshr_b32 [[ELT2:s[0-9]+]], [[VAL]], 16
-; GCN-DAG: v_mov_b32_e32 [[V_LOAD0:v[0-9]+]], s{{[0-9]+}}
 ; GCN-DAG: v_mov_b32_e32 [[V_ELT2:v[0-9]+]], [[ELT2]]
+; GCN-DAG: v_mov_b32_e32 [[V_LOAD0:v[0-9]+]], [[VAL]]
 ; GCN: buffer_store_byte [[V_ELT2]]
 ; GCN: buffer_store_byte [[V_LOAD0]]
 define amdgpu_kernel void @extract_vector_elt_v32i8(<32 x i8> %foo) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
--- a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
+++ b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
@@ -52,33 +52,33 @@
 ; GFX7-ALIGNED-LABEL: global_store_2xi16_align2:
 ; GFX7-ALIGNED:       ; %bb.0:
 ; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v4, 1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v5, 2
 ; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-ALIGNED-NEXT:    s_add_u32 s2, s0, 2
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-ALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-ALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 2
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-ALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-ALIGNED-NEXT:    flat_store_short v[0:1], v4
+; GFX7-ALIGNED-NEXT:    flat_store_short v[2:3], v5
 ; GFX7-ALIGNED-NEXT:    s_endpgm
 ;
 ; GFX7-UNALIGNED-LABEL: global_store_2xi16_align2:
 ; GFX7-UNALIGNED:       ; %bb.0:
 ; GFX7-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
-; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 1
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v4, 1
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v5, 2
 ; GFX7-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-UNALIGNED-NEXT:    s_add_u32 s2, s0, 2
-; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-UNALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-UNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
-; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 2
-; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-UNALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-UNALIGNED-NEXT:    flat_store_short v[0:1], v4
+; GFX7-UNALIGNED-NEXT:    flat_store_short v[2:3], v5
 ; GFX7-UNALIGNED-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_store_2xi16_align2:
@@ -155,8 +155,9 @@
 ; GFX7-ALIGNED-LABEL: global_store_2xi16_align1:
 ; GFX7-ALIGNED:       ; %bb.0:
 ; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v4, 1
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v5, 0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v8, 1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v9, 0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v10, 2
 ; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-ALIGNED-NEXT:    s_add_u32 s2, s0, 2
 ; GFX7-ALIGNED-NEXT:    s_addc_u32 s3, s1, 0
@@ -165,18 +166,17 @@
 ; GFX7-ALIGNED-NEXT:    s_addc_u32 s5, s1, 0
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX7-ALIGNED-NEXT:    s_add_u32 s0, s0, 3
+; GFX7-ALIGNED-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v7, s3
 ; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-ALIGNED-NEXT:    flat_store_byte v[0:1], v4
-; GFX7-ALIGNED-NEXT:    flat_store_byte v[2:3], v5
-; GFX7-ALIGNED-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v4, 2
-; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-ALIGNED-NEXT:    flat_store_byte v[0:1], v5
-; GFX7-ALIGNED-NEXT:    flat_store_byte v[2:3], v4
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v6, s2
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[0:1], v8
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[2:3], v9
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[4:5], v9
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[6:7], v10
 ; GFX7-ALIGNED-NEXT:    s_endpgm
 ;
 ; GFX7-UNALIGNED-LABEL: global_store_2xi16_align1:
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr.ll b/llvm/test/CodeGen/AMDGPU/global-saddr.ll
--- a/llvm/test/CodeGen/AMDGPU/global-saddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr.ll
@@ -45,8 +45,8 @@
   store volatile i64 %add7, i64 addrspace(1)* %ptr9
 
 ; Test various offset boundaries.
-; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:4088{{$}}
 ; GFX9: global_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+}}:{{[0-9]+}}] offset:2040{{$}}
+; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:4088{{$}}
 ; GFX9: global_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+}}:{{[0-9]+}}] offset:4088{{$}}
   %gep11 = getelementptr inbounds i64, i64 addrspace(1)* %gep, i64 511
   %load11 = load i64, i64 addrspace(1)* %gep11
diff --git a/llvm/test/CodeGen/AMDGPU/sched-assert-onlydbg-value-empty-region.mir b/llvm/test/CodeGen/AMDGPU/sched-assert-onlydbg-value-empty-region.mir
--- a/llvm/test/CodeGen/AMDGPU/sched-assert-onlydbg-value-empty-region.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-assert-onlydbg-value-empty-region.mir
@@ -35,12 +35,12 @@
   ; CHECK:   [[DEF5:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
   ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK:   [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-  ; CHECK:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
   ; CHECK:   [[COPY1:%[0-9]+]]:vreg_64 = COPY [[GLOBAL_LOAD_DWORDX2_]]
   ; CHECK:   undef %6.sub0:vreg_64 = V_ADD_F32_e32 [[DEF]].sub0, [[COPY1]].sub0, implicit $exec
   ; CHECK:   dead undef %6.sub1:vreg_64 = V_ADD_F32_e32 [[DEF]].sub1, [[COPY1]].sub0, implicit $exec
   ; CHECK:   [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, 0, 0, implicit $exec
+  ; CHECK:   [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+  ; CHECK:   [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
   ; CHECK:   [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
   ; CHECK:   undef %19.sub0:vreg_64 = V_ADD_F32_e32 [[GLOBAL_LOAD_DWORD1]], [[GLOBAL_LOAD_DWORDX2_]].sub0, implicit $exec
   ; CHECK:   %19.sub1:vreg_64 = V_ADD_F32_e32 [[GLOBAL_LOAD_DWORD]], [[GLOBAL_LOAD_DWORD]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/sign_extend.ll b/llvm/test/CodeGen/AMDGPU/sign_extend.ll
--- a/llvm/test/CodeGen/AMDGPU/sign_extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/sign_extend.ll
@@ -341,16 +341,15 @@
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_bfe_i32 s3, s0, 0x80008
 ; SI-NEXT:    s_ashr_i32 s1, s0, 24
 ; SI-NEXT:    s_bfe_i32 s2, s0, 0x80010
-; SI-NEXT:    s_bfe_i32 s3, s0, 0x80008
 ; SI-NEXT:    s_sext_i32_i8 s0, s0
 ; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s3
-; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_store_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    v_mov_b32_e32 v0, s2
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -469,13 +468,12 @@
 ; SI-NEXT:    s_ashr_i64 s[4:5], s[6:7], 48
 ; SI-NEXT:    s_ashr_i32 s5, s6, 16
 ; SI-NEXT:    s_sext_i32_i16 s6, s6
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s5
 ; SI-NEXT:    s_sext_i32_i16 s7, s7
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    v_mov_b32_e32 v1, s5
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_store_dword v1, off, s[0:3], 0
+; SI-NEXT:    s_waitcnt expcnt(1)
 ; SI-NEXT:    v_mov_b32_e32 v0, s7
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -493,12 +491,12 @@
 ; VI-NEXT:    s_ashr_i32 s5, s6, 16
 ; VI-NEXT:    s_sext_i32_i16 s6, s6
 ; VI-NEXT:    s_mov_b32 s0, s4
-; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    s_ashr_i32 s4, s7, 16
-; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; VI-NEXT:    v_mov_b32_e32 v0, s5
 ; VI-NEXT:    s_sext_i32_i16 s7, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s6
+; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT:    buffer_store_dword v1, off, s[0:3], 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s7
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s4