[AMDGPU] Scheduler: Don't revert the schedule if the register pressure isn't changed for a region

[vpykhtin]

Summary

This one-linear fix improves compilation time for about ~40% on ASAN enabled code.

[arsenm]

llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp

995

Maybe adjust the debug statement to say it's unchanged

[arsenm]

llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp

995

Plus this is mayCauseSpilling and falsely claiming true, should this check be in the caller?

[kerbowa]

Interesting. Why exactly does this improve compile time so much? I thought reverting scheduling wasn't exactly expensive and the RP tracking was the problem.

llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp

995

Agreed. Can we put it in the base class shouldRevertScheduling?

[vpykhtin]

In D136069#3862399, @kerbowa wrote:

Interesting. Why exactly does this improve compile time so much? I thought reverting scheduling wasn't exactly expensive and the RP tracking was the problem.

The number of reverted regions dropped from 34813 to 1289 in my testcase but they're RP measured anyway and this is strange.

[vpykhtin]

I was a bit inaccurate in wording - ~40% improvement is only for scheduling, not the overall compile time.

[rampitec]

The check seems reasonable modulo infrastructure comments.

[vpykhtin]

Rebased, moved the check into shouldRevertScheduling of two stages:

OccInitialScheduleStage
ClusteredLowOccStage

I don't add it to the unclustered stage as this schedule should only be kept if some improvement in RP is achieved.

[vpykhtin]

ping.

[arsenm]

LGTM

llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp

 bool GCNSchedStage::shouldRevertScheduling(unsigned WavesAfter) {
   if (WavesAfter < DAG.MinOccupancy)
     return true;
 
   return false;
 }
 
 bool OccInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
+  if (PressureAfter == PressureBefore)
+    return false;
+
   if (GCNSchedStage::shouldRevertScheduling(WavesAfter))
     return true;
 
   if (mayCauseSpilling(WavesAfter))
     return true;
 
   return false;
 }
 
 bool UnclusteredHighRPStage::shouldRevertScheduling(unsigned WavesAfter) {
   // If RP is not reduced in the unclustred reschedule stage, revert to the
   // old schedule.
   if ((WavesAfter <= PressureBefore.getOccupancy(ST) &&
        mayCauseSpilling(WavesAfter)) ||
       GCNSchedStage::shouldRevertScheduling(WavesAfter)) {
     LLVM_DEBUG(dbgs() << "Unclustered reschedule did not help.\n");
     return true;
   }
 
   return false;
 }
 
 bool ClusteredLowOccStage::shouldRevertScheduling(unsigned WavesAfter) {
+  if (PressureAfter == PressureBefore)
+    return false;
+
   if (GCNSchedStage::shouldRevertScheduling(WavesAfter))
     return true;
 
   if (mayCauseSpilling(WavesAfter))
     return true;
 
   return false;
 }
bool ILPInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
 
   return false;
 }
 
 bool GCNSchedStage::mayCauseSpilling(unsigned WavesAfter) {
   if (WavesAfter <= MFI.getMinWavesPerEU() &&
       !PressureAfter.less(ST, PressureBefore) &&
       DAG.RegionsWithExcessRP[RegionIdx]) {
     LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n");
     return true;
   }
 
   return false;
 }
 
 void GCNSchedStage::revertScheduling() {
   DAG.RegionsWithMinOcc[RegionIdx] =

llvm/test/CodeGen/AMDGPU/load-global-i16.ll

 ; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
-; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[8:11], 0
-; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:16
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v15
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v14
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v12
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v11, 16, v19
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v18
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v2, 0xffff, v15
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xffff, v14
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v1, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v2, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v3, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v5, 0xffff, v13
-; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v3, 0xffff, v12
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v3, off, s[12:15], 0 offset:20 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v4, off, s[12:15], 0 offset:24 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v5, off, s[12:15], 0 offset:28 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    buffer_store_dword v6, off, s[12:15], 0 offset:32 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v17
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v16
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v10, 0xffff, v19
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xffff, v18
-; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[18:21], off, s[8:11], 0 offset:32
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v14, 0xffff, v17
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v12, 0xffff, v16
-; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[22:25], off, s[8:11], 0 offset:48
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v29, 16, v21
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v27, 16, v20
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v33, 16, v19
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v18
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v28, 0xffff, v21
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v26, 0xffff, v20
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v32, 0xffff, v19
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v30, 0xffff, v18
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v25
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v24
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v22
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v18, 0xffff, v25
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v16, 0xffff, v24
-; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[38:41], off, s[8:11], 0 offset:64
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v36, 0xffff, v23
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v34, 0xffff, v22
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
 ; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v44, 16, v41
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v42, 16, v40
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v48, 16, v39
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v46, 16, v38
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v43, 0xffff, v41
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v41, 0xffff, v40
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v47, 0xffff, v39
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v45, 0xffff, v38
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v52, 16, v23
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v50, 16, v22
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v56, 16, v21
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v54, 16, v20
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v51, 0xffff, v23
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v49, 0xffff, v22
-; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[22:25], off, s[8:11], 0 offset:96
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v55, 0xffff, v21
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v53, 0xffff, v20
-; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[57:60], off, s[8:11], 0 offset:112
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v25
-; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(1)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v24
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v23
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v22
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v39, 16, v1
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v43, 16, v7
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v34, 0xffff, v3
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v32, 0xffff, v2
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v32, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v33, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v34, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v35, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v38, 0xffff, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v36, 0xffff, v0
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v6, 0xffff, v25
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xffff, v24
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v2, 0xffff, v23
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xffff, v22
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v23, 16, v60
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v21, 16, v59
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v63, 16, v58
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v61, 16, v57
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v22, 0xffff, v60
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v20, 0xffff, v59
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v62, 0xffff, v58
-; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v60, 0xffff, v57
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v42, 0xffff, v7
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v40, 0xffff, v6
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v34, 0xffff, v5
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v32, 0xffff, v4
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v11
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v10
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v47, 16, v9
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v45, 16, v8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v6, 0xffff, v11
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xffff, v10
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v46, 0xffff, v9
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v44, 0xffff, v8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v14
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v10, 0xffff, v15
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xffff, v14
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v50, 0xffff, v13
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v48, 0xffff, v12
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v19
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v18
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v14, 0xffff, v19
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v12, 0xffff, v18
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v54, 0xffff, v17
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v52, 0xffff, v16
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v23
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v22
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v59, 16, v21
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v57, 16, v20
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v18, 0xffff, v23
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v16, 0xffff, v22
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v58, 0xffff, v21
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v56, 0xffff, v20
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v23, 16, v27
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v21, 16, v26
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v63, 16, v25
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v61, 16, v24
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v22, 0xffff, v27
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v20, 0xffff, v26
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v62, 0xffff, v25
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v60, 0xffff, v24
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v27, 16, v31
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v25, 16, v30
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v29
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v28
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v26, 0xffff, v31
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v24, 0xffff, v30
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v2, 0xffff, v29
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xffff, v28
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:224
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:240
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:192
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:208
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[53:56], off, s[0:3], 0 offset:160
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[49:52], off, s[0:3], 0 offset:176
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[45:48], off, s[0:3], 0 offset:128
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[41:44], off, s[0:3], 0 offset:144
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[34:37], off, s[0:3], 0 offset:96
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:112
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[30:33], off, s[0:3], 0 offset:64
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[26:29], off, s[0:3], 0 offset:80
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
-; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0 offset:20 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:24 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:28 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:32 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:240
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:192
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:208
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:160
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:128
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:144
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0
 ; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
 ; GCN-NOHSA-SI-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
 ; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
 ; GCN-NOHSA-SI-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:16 ; 4-byte Folded Reload
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
 ; GCN-NOHSA-SI-NEXT:    s_endpgm
 ;

 ; GCN-NOHSA-VI-NEXT:    s_add_u32 s88, s88, s3
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:112
 ; GCN-NOHSA-VI-NEXT:    s_addc_u32 s89, s89, 0
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v15
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v14
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v2, 0xffff, v15
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xffff, v14
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 16, v13
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v12
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v11, 16, v19
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v9, 16, v18
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v6, 0xffff, v13
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xffff, v12
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 16, v17
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v13, 16, v16
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v10, 0xffff, v19
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v8, 0xffff, v18
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v14, 0xffff, v17
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xffff, v16
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:32
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:48
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v19
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v18
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v17
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v29, 16, v16
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v26, 0xffff, v19
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v24, 0xffff, v18
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v30, 0xffff, v17
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v28, 0xffff, v16
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v3
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v2
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v34, 0xffff, v3
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v32, 0xffff, v2
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v32, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v33, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v34, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v35, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v1
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v38, 0xffff, v1
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v36, 0xffff, v0
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v29
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v28
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v2, 0xffff, v29
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xffff, v28
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v7
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v5
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v4
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v42, 0xffff, v7
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v40, 0xffff, v6
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v34, 0xffff, v5
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v32, 0xffff, v4
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 16, v11
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v10
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v9
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v6, 0xffff, v11
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xffff, v10
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v46, 0xffff, v9
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v44, 0xffff, v8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v11, 16, v15
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v9, 16, v14
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v10, 0xffff, v15
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v8, 0xffff, v14
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v50, 0xffff, v13
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v48, 0xffff, v12
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 16, v19
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v13, 16, v18
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v14, 0xffff, v19
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xffff, v18
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v54, 0xffff, v17
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v52, 0xffff, v16
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v23
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v17, 16, v22
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v35, 16, v21
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v21
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v20
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v18, 0xffff, v23
 ; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v16, 0xffff, v22
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v34, 0xffff, v21
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v32, 0xffff, v20
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:64
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[36:39], off, s[8:11], 0 offset:80
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v23
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v22
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v21
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v20
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v42, 0xffff, v23
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v40, 0xffff, v22
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v46, 0xffff, v21
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v44, 0xffff, v20
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v39
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v38
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v37
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v36
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v22, 0xffff, v39
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v20, 0xffff, v38
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v50, 0xffff, v37
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v48, 0xffff, v36
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[36:39], off, s[8:11], 0 offset:96
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[52:55], off, s[8:11], 0 offset:112
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v39
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v53
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v52
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v2, 0xffff, v53
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xffff, v52
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v38
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v63, 16, v37
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v61, 16, v36
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v58, 0xffff, v39
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v56, 0xffff, v38
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v62, 0xffff, v37
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v60, 0xffff, v36
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v55
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v54
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v38, 0xffff, v55
-; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v36, 0xffff, v54
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v58, 0xffff, v21
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v56, 0xffff, v20
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v23, 16, v27
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v21, 16, v26
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v63, 16, v25
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v61, 16, v24
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v22, 0xffff, v27
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v20, 0xffff, v26
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v62, 0xffff, v25
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v60, 0xffff, v24
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v27, 16, v31
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v25, 16, v30
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v26, 0xffff, v31
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v24, 0xffff, v30
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:240
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:192
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:208
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:160
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:176
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:128
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:144
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
 ; GCN-NOHSA-VI-NEXT:    s_endpgm
 ;

 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
 ; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s90, -1
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s91, 0xe80000
 ; GCN-NOHSA-VI-NEXT:    s_add_u32 s88, s88, s3
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    s_addc_u32 s89, s89, 0
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s6
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s7
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, s2
-; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, s3
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[8:11], off, s[4:7], 0 offset:112
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[4:7], 0 offset:96
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[4:7], 0 offset:80
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[20:23], off, s[4:7], 0 offset:64
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[24:27], off, s[4:7], 0
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[28:31], off, s[4:7], 0 offset:16
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[32:35], off, s[4:7], 0 offset:32
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[36:39], off, s[4:7], 0 offset:48
-; GCN-NOHSA-VI-NEXT:    s_addc_u32 s89, s89, 0
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(6)
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v63, 16, v13
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(5)
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v59, 16, v17
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(4)
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v55, 16, v21
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(3)
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 16, v27
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 16, v26
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v27, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v26, 0, 16
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    buffer_store_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 16, v9
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 16, v8
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v9, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v8, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 16, v25
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v5, 16, v24
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v25, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v24, 0, 16
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(7)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v35, 16, v3
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v33, 16, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v34, v3, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v32, v2, 0, 16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v32, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v33, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v34, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v35, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v39, 16, v1
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v37, 16, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v38, v1, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v36, v0, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 16, v29
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 16, v28
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v29, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v28, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v35, 16, v7
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v33, 16, v6
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v34, v7, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v32, v6, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v43, 16, v5
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v41, 16, v4
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v42, v5, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v40, v4, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 16, v11
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v5, 16, v10
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v11, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v10, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v47, 16, v9
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v45, 16, v8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v46, v9, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v44, v8, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v11, 16, v15
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v9, 16, v14
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v10, v15, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v8, v14, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v51, 16, v13
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v49, 16, v12
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v50, v13, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v48, v12, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v15, 16, v19
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v13, 16, v18
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v14, v19, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v12, v18, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v55, 16, v17
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v53, 16, v16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v54, v17, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v52, v16, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 16, v23
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v17, 16, v22
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v23, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v16, v22, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v59, 16, v21
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v57, 16, v20
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v58, v21, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v56, v20, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v23, 16, v27
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v21, 16, v26
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v22, v27, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v20, v26, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v63, 16, v25
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v61, 16, v24
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v62, v25, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v60, v24, 0, 16
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v27, 16, v31
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v25, 16, v30
 ; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v26, v31, 0, 16
 ; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v24, v30, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v43, 16, v29
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v41, 16, v28
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v42, v29, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v40, v28, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v31, 16, v35
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v29, 16, v34
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v30, v35, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v28, v34, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v47, 16, v33
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v45, 16, v32
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v46, v33, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v44, v32, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v35, 16, v39
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v33, 16, v38
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v34, v39, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v32, v38, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v51, 16, v37
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v49, 16, v36
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v50, v37, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v48, v36, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v39, 16, v23
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v37, 16, v22
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v38, v23, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v36, v22, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v53, 16, v20
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v54, v21, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v52, v20, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v22, 16, v19
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v20, 16, v18
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v21, v19, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v19, v18, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v57, 16, v16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v58, v17, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v56, v16, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 16, v15
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 16, v14
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v15, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v14, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v61, 16, v12
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v62, v13, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v60, v12, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 16, v11
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v12, 16, v10
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v11, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v11, v10, 0, 16
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:240
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:192
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:208
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:160
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:128
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:144
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:112
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    buffer_load_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Reload
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
 ; GCN-NOHSA-VI-NEXT:    s_endpgm
 ;

llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir

body:             |
   ; GFX908-LABEL: name: test_occ_2_sink_for_3_occ
   ; GFX908: bb.0:
   ; GFX908-NEXT:   successors: %bb.1(0x80000000)
   ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
   ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
   ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
-  ; GFX908-NEXT:   [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
   ; GFX908-NEXT:   undef %4.sub1:sreg_64 = S_MOV_B32 0
-  ; GFX908-NEXT:   undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
   ; GFX908-NEXT:   %5:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 0, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX908-NEXT:   [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
   ; GFX908-NEXT:   %6:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %7:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %8:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %9:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 4, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %10:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 5, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %11:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 6, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %12:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 7, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %13:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 8, implicit $exec, implicit $mode, implicit-def $m0
body:             |
   ; GFX908-LABEL: name: test_occ_1_sink_for_2_occ
   ; GFX908: bb.0:
   ; GFX908-NEXT:   successors: %bb.1(0x80000000)
   ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
   ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
   ; GFX908-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
-  ; GFX908-NEXT:   [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
   ; GFX908-NEXT:   undef %4.sub1:sreg_64 = S_MOV_B32 0
-  ; GFX908-NEXT:   undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
   ; GFX908-NEXT:   %5:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 0, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX908-NEXT:   [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+  ; GFX908-NEXT:   undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
   ; GFX908-NEXT:   %6:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %7:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %8:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %9:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 4, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %10:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 5, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %11:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 6, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %12:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 7, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   %13:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 8, implicit $exec, implicit $mode, implicit-def $m0