[AMDGPU] Fixed stack pointer init with architected flat scratch

[rampitec]

Summary

Even if wave offset is not present we still need to do the rest
of the initialization. The mov into s32 was missing in the
kernels.

Fixes: SWDEV-310935

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

void SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF,
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const Function &F = MF.getFunction();
   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
 
   assert(MFI->isEntryFunction());
 
   Register PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
       AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET);
-  // FIXME: Hack to not crash in situations which emitted an error.
-  if (!PreloadedScratchWaveOffsetReg)
-    return;
 
   // We need to do the replacement of the private segment buffer register even
   // if there are no stack objects. There could be stores to undef or a
   // constant without an associated object.
   //
   // This will return `Register()` in cases where there are no actual
   // uses of the SRSRC.
   Register ScratchRsrcReg;
void SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF,
   MachineBasicBlock::iterator I = MBB.begin();
 
   // We found the SRSRC first because it needs four registers and has an
   // alignment requirement. If the SRSRC that we found is clobbering with
   // the scratch wave offset, which may be in a fixed SGPR or a free SGPR
   // chosen by SITargetLowering::allocateSystemSGPRs, COPY the scratch
   // wave offset to a free SGPR.
   Register ScratchWaveOffsetReg;
-  if (TRI->isSubRegisterEq(ScratchRsrcReg, PreloadedScratchWaveOffsetReg)) {
+  if (PreloadedScratchWaveOffsetReg &&
+      TRI->isSubRegisterEq(ScratchRsrcReg, PreloadedScratchWaveOffsetReg)) {
     ArrayRef<MCPhysReg> AllSGPRs = TRI->getAllSGPR32(MF);
     unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
     AllSGPRs = AllSGPRs.slice(
         std::min(static_cast<unsigned>(AllSGPRs.size()), NumPreloaded));
     Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
     for (MCPhysReg Reg : AllSGPRs) {
       if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg) &&
           !TRI->isSubRegisterEq(ScratchRsrcReg, Reg) && GITPtrLoReg != Reg) {
         ScratchWaveOffsetReg = Reg;
         BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg)
             .addReg(PreloadedScratchWaveOffsetReg, RegState::Kill);
         break;
       }
     }
   } else {
     ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
   }
-  assert(ScratchWaveOffsetReg);
+  assert(ScratchWaveOffsetReg || !PreloadedScratchWaveOffsetReg);
 
   if (requiresStackPointerReference(MF)) {
     Register SPReg = MFI->getStackPtrOffsetReg();
     assert(SPReg != AMDGPU::SP_REG);
     BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), SPReg)
         .addImm(FrameInfo.getStackSize() * getScratchScaleFactor(ST));
   }
 
   if (hasFP(MF)) {
     Register FPReg = MFI->getFrameOffsetReg();
     assert(FPReg != AMDGPU::FP_REG);
     BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), FPReg).addImm(0);
   }
 
   bool NeedsFlatScratchInit =
       MFI->hasFlatScratchInit() &&
       (MRI.isPhysRegUsed(AMDGPU::FLAT_SCR) || FrameInfo.hasCalls() ||
        (!allStackObjectsAreDead(FrameInfo) && ST.enableFlatScratch()));
 
   if ((NeedsFlatScratchInit || ScratchRsrcReg) &&
-      !ST.flatScratchIsArchitected()) {
+      PreloadedScratchWaveOffsetReg && !ST.flatScratchIsArchitected()) {
     MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
     MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
   }
 
   if (NeedsFlatScratchInit) {
     emitEntryFunctionFlatScratchInit(MF, MBB, I, DL, ScratchWaveOffsetReg);
   }
 

llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 --amdgpu-enable-flat-scratch < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 --amdgpu-enable-flat-scratch < %s | FileCheck -check-prefixes=GCN,FLAT_SCR_OPT %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 --mattr=+architected-flat-scratch < %s | FileCheck -check-prefixes=GCN,FLAT_SCR_ARCH %s
 
 declare void @extern_func() #0
 
 define amdgpu_kernel void @stack_object_addrspacecast_in_kernel_no_calls() {
-; GCN-LABEL: stack_object_addrspacecast_in_kernel_no_calls:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_add_u32 s0, s0, s3
-; GCN-NEXT:    s_addc_u32 s1, s1, 0
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
-; GCN-NEXT:    v_mov_b32_e32 v0, 4
-; GCN-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
-; GCN-NEXT:    v_mov_b32_e32 v2, 0
-; GCN-NEXT:    s_lshl_b32 s0, s0, 16
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc_lo, -1, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, s0, vcc_lo
-; GCN-NEXT:    flat_store_dword v[0:1], v2
-; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
-; GCN-NEXT:    s_endpgm
+; FLAT_SCR_OPT-LABEL: stack_object_addrspacecast_in_kernel_no_calls:
+; FLAT_SCR_OPT:       ; %bb.0:
+; FLAT_SCR_OPT-NEXT:    s_add_u32 s0, s0, s3
+; FLAT_SCR_OPT-NEXT:    s_addc_u32 s1, s1, 0
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
+; FLAT_SCR_OPT-NEXT:    v_mov_b32_e32 v0, 4
+; FLAT_SCR_OPT-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
+; FLAT_SCR_OPT-NEXT:    v_mov_b32_e32 v2, 0
+; FLAT_SCR_OPT-NEXT:    s_lshl_b32 s0, s0, 16
+; FLAT_SCR_OPT-NEXT:    v_cmp_ne_u32_e32 vcc_lo, -1, v0
+; FLAT_SCR_OPT-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; FLAT_SCR_OPT-NEXT:    v_cndmask_b32_e64 v1, 0, s0, vcc_lo
+; FLAT_SCR_OPT-NEXT:    flat_store_dword v[0:1], v2
+; FLAT_SCR_OPT-NEXT:    s_waitcnt_vscnt null, 0x0
+; FLAT_SCR_OPT-NEXT:    s_endpgm
+;
+; FLAT_SCR_ARCH-LABEL: stack_object_addrspacecast_in_kernel_no_calls:
+; FLAT_SCR_ARCH:       ; %bb.0:
+; FLAT_SCR_ARCH-NEXT:    v_mov_b32_e32 v0, 4
+; FLAT_SCR_ARCH-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_SH_MEM_BASES, 0, 16)
+; FLAT_SCR_ARCH-NEXT:    v_mov_b32_e32 v2, 0
+; FLAT_SCR_ARCH-NEXT:    s_lshl_b32 s0, s0, 16
+; FLAT_SCR_ARCH-NEXT:    v_cmp_ne_u32_e32 vcc_lo, -1, v0
+; FLAT_SCR_ARCH-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; FLAT_SCR_ARCH-NEXT:    v_cndmask_b32_e64 v1, 0, s0, vcc_lo
+; FLAT_SCR_ARCH-NEXT:    flat_store_dword v[0:1], v2
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt_vscnt null, 0x0
+; FLAT_SCR_ARCH-NEXT:    s_endpgm
   %alloca = alloca i32, addrspace(5)
   %cast = addrspacecast i32 addrspace(5)* %alloca to i32*
   store volatile i32 0, i32* %cast
   ret void
 }
 
 define amdgpu_kernel void @stack_object_in_kernel_no_calls() {
-; GCN-LABEL: stack_object_in_kernel_no_calls:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_add_u32 s0, s0, s3
-; GCN-NEXT:    s_addc_u32 s1, s1, 0
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_mov_b32 vcc_lo, 0
-; GCN-NEXT:    scratch_store_dword off, v0, vcc_lo offset:4
-; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
-; GCN-NEXT:    s_endpgm
+; FLAT_SCR_OPT-LABEL: stack_object_in_kernel_no_calls:
+; FLAT_SCR_OPT:       ; %bb.0:
+; FLAT_SCR_OPT-NEXT:    s_add_u32 s0, s0, s3
+; FLAT_SCR_OPT-NEXT:    s_addc_u32 s1, s1, 0
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
+; FLAT_SCR_OPT-NEXT:    v_mov_b32_e32 v0, 0
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 vcc_lo, 0
+; FLAT_SCR_OPT-NEXT:    scratch_store_dword off, v0, vcc_lo offset:4
+; FLAT_SCR_OPT-NEXT:    s_waitcnt_vscnt null, 0x0
+; FLAT_SCR_OPT-NEXT:    s_endpgm
+;
+; FLAT_SCR_ARCH-LABEL: stack_object_in_kernel_no_calls:
+; FLAT_SCR_ARCH:       ; %bb.0:
+; FLAT_SCR_ARCH-NEXT:    v_mov_b32_e32 v0, 0
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 vcc_lo, 0
+; FLAT_SCR_ARCH-NEXT:    scratch_store_dword off, v0, vcc_lo offset:4
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt_vscnt null, 0x0
+; FLAT_SCR_ARCH-NEXT:    s_endpgm
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   ret void
 }
 
 define amdgpu_kernel void @kernel_calls_no_stack() {
-; GCN-LABEL: kernel_calls_no_stack:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_add_u32 s6, s6, s11
-; GCN-NEXT:    s_mov_b32 s32, 0
-; GCN-NEXT:    s_addc_u32 s7, s7, 0
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s6
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s7
-; GCN-NEXT:    s_getpc_b64 s[0:1]
-; GCN-NEXT:    s_add_u32 s0, s0, extern_func@gotpcrel32@lo+4
-; GCN-NEXT:    s_addc_u32 s1, s1, extern_func@gotpcrel32@hi+12
-; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GCN-NEXT:    s_endpgm
+; FLAT_SCR_OPT-LABEL: kernel_calls_no_stack:
+; FLAT_SCR_OPT:       ; %bb.0:
+; FLAT_SCR_OPT-NEXT:    s_add_u32 s6, s6, s11
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s32, 0
+; FLAT_SCR_OPT-NEXT:    s_addc_u32 s7, s7, 0
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s6
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s7
+; FLAT_SCR_OPT-NEXT:    s_getpc_b64 s[0:1]
+; FLAT_SCR_OPT-NEXT:    s_add_u32 s0, s0, extern_func@gotpcrel32@lo+4
+; FLAT_SCR_OPT-NEXT:    s_addc_u32 s1, s1, extern_func@gotpcrel32@hi+12
+; FLAT_SCR_OPT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; FLAT_SCR_OPT-NEXT:    s_waitcnt lgkmcnt(0)
+; FLAT_SCR_OPT-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; FLAT_SCR_OPT-NEXT:    s_endpgm
+;
+; FLAT_SCR_ARCH-LABEL: kernel_calls_no_stack:
+; FLAT_SCR_ARCH:       ; %bb.0:
+; FLAT_SCR_ARCH-NEXT:    s_getpc_b64 s[0:1]
+; FLAT_SCR_ARCH-NEXT:    s_add_u32 s0, s0, extern_func@gotpcrel32@lo+4
+; FLAT_SCR_ARCH-NEXT:    s_addc_u32 s1, s1, extern_func@gotpcrel32@hi+12
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s32, 0
+; FLAT_SCR_ARCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt lgkmcnt(0)
+; FLAT_SCR_ARCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; FLAT_SCR_ARCH-NEXT:    s_endpgm
   call void @extern_func()
   ret void
 }
 
 define amdgpu_kernel void @test(i32 addrspace(1)* %out, i32 %in) {
-; GCN-LABEL: test:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_add_u32 s2, s2, s5
-; GCN-NEXT:    s_addc_u32 s3, s3, 0
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
-; GCN-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
-; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GCN-NEXT:    s_mov_b32 s104, exec_lo
-; GCN-NEXT:    s_mov_b32 exec_lo, 3
-; GCN-NEXT:    s_mov_b32 s105, 0
-; GCN-NEXT:    scratch_store_dword off, v72, s105
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_writelane_b32 v72, s2, 0
-; GCN-NEXT:    s_mov_b32 s105, 4
-; GCN-NEXT:    v_writelane_b32 v72, s3, 1
-; GCN-NEXT:    scratch_store_dword off, v72, s105 ; 4-byte Folded Spill
-; GCN-NEXT:    s_waitcnt_depctr 0xffe3
-; GCN-NEXT:    s_mov_b32 s105, 0
-; GCN-NEXT:    scratch_load_dword v72, off, s105
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    s_waitcnt_depctr 0xffe3
-; GCN-NEXT:    s_mov_b32 exec_lo, s104
-; GCN-NEXT:    s_load_dword vcc_lo, s[0:1], 0x8
-; GCN-NEXT:    ; kill: killed $sgpr0_sgpr1
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v0, vcc_lo
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    ;;#ASMSTART
-; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    s_mov_b32 s2, exec_lo
-; GCN-NEXT:    s_mov_b32 exec_lo, 3
-; GCN-NEXT:    s_mov_b32 s3, 0
-; GCN-NEXT:    scratch_store_dword off, v2, s3
-; GCN-NEXT:    s_waitcnt_depctr 0xffe3
-; GCN-NEXT:    s_mov_b32 s3, 4
-; GCN-NEXT:    scratch_load_dword v2, off, s3 ; 4-byte Folded Reload
-; GCN-NEXT:    s_waitcnt_depctr 0xffe3
-; GCN-NEXT:    s_mov_b32 s3, 0
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_readlane_b32 s0, v2, 0
-; GCN-NEXT:    v_readlane_b32 s1, v2, 1
-; GCN-NEXT:    scratch_load_dword v2, off, s3
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    s_waitcnt_depctr 0xffe3
-; GCN-NEXT:    s_mov_b32 exec_lo, s2
-; GCN-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-NEXT:    global_store_dword v1, v0, s[0:1]
-; GCN-NEXT:    s_endpgm
+; FLAT_SCR_OPT-LABEL: test:
+; FLAT_SCR_OPT:       ; %bb.0:
+; FLAT_SCR_OPT-NEXT:    s_add_u32 s2, s2, s5
+; FLAT_SCR_OPT-NEXT:    s_addc_u32 s3, s3, 0
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
+; FLAT_SCR_OPT-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
+; FLAT_SCR_OPT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s104, exec_lo
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 exec_lo, 3
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s105, 0
+; FLAT_SCR_OPT-NEXT:    scratch_store_dword off, v72, s105
+; FLAT_SCR_OPT-NEXT:    s_waitcnt lgkmcnt(0)
+; FLAT_SCR_OPT-NEXT:    v_writelane_b32 v72, s2, 0
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s105, 4
+; FLAT_SCR_OPT-NEXT:    v_writelane_b32 v72, s3, 1
+; FLAT_SCR_OPT-NEXT:    scratch_store_dword off, v72, s105 ; 4-byte Folded Spill
+; FLAT_SCR_OPT-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s105, 0
+; FLAT_SCR_OPT-NEXT:    scratch_load_dword v72, off, s105
+; FLAT_SCR_OPT-NEXT:    s_waitcnt vmcnt(0)
+; FLAT_SCR_OPT-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 exec_lo, s104
+; FLAT_SCR_OPT-NEXT:    s_load_dword vcc_lo, s[0:1], 0x8
+; FLAT_SCR_OPT-NEXT:    ; kill: killed $sgpr0_sgpr1
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    s_waitcnt lgkmcnt(0)
+; FLAT_SCR_OPT-NEXT:    v_mov_b32_e32 v0, vcc_lo
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    ;;#ASMSTART
+; FLAT_SCR_OPT-NEXT:    ;;#ASMEND
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s2, exec_lo
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 exec_lo, 3
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s3, 0
+; FLAT_SCR_OPT-NEXT:    scratch_store_dword off, v2, s3
+; FLAT_SCR_OPT-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s3, 4
+; FLAT_SCR_OPT-NEXT:    scratch_load_dword v2, off, s3 ; 4-byte Folded Reload
+; FLAT_SCR_OPT-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 s3, 0
+; FLAT_SCR_OPT-NEXT:    s_waitcnt vmcnt(0)
+; FLAT_SCR_OPT-NEXT:    v_readlane_b32 s0, v2, 0
+; FLAT_SCR_OPT-NEXT:    v_readlane_b32 s1, v2, 1
+; FLAT_SCR_OPT-NEXT:    scratch_load_dword v2, off, s3
+; FLAT_SCR_OPT-NEXT:    s_waitcnt vmcnt(0)
+; FLAT_SCR_OPT-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_OPT-NEXT:    s_mov_b32 exec_lo, s2
+; FLAT_SCR_OPT-NEXT:    v_mov_b32_e32 v1, 0
+; FLAT_SCR_OPT-NEXT:    global_store_dword v1, v0, s[0:1]
+; FLAT_SCR_OPT-NEXT:    s_endpgm
+;
+; FLAT_SCR_ARCH-LABEL: test:
+; FLAT_SCR_ARCH:       ; %bb.0:
+; FLAT_SCR_ARCH-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s104, exec_lo
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 exec_lo, 3
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s105, 0
+; FLAT_SCR_ARCH-NEXT:    scratch_store_dword off, v72, s105
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt lgkmcnt(0)
+; FLAT_SCR_ARCH-NEXT:    v_writelane_b32 v72, s2, 0
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s105, 4
+; FLAT_SCR_ARCH-NEXT:    v_writelane_b32 v72, s3, 1
+; FLAT_SCR_ARCH-NEXT:    scratch_store_dword off, v72, s105 ; 4-byte Folded Spill
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s105, 0
+; FLAT_SCR_ARCH-NEXT:    scratch_load_dword v72, off, s105
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt vmcnt(0)
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 exec_lo, s104
+; FLAT_SCR_ARCH-NEXT:    s_load_dword vcc_lo, s[0:1], 0x8
+; FLAT_SCR_ARCH-NEXT:    ; kill: killed $sgpr0_sgpr1
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt lgkmcnt(0)
+; FLAT_SCR_ARCH-NEXT:    v_mov_b32_e32 v0, vcc_lo
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMSTART
+; FLAT_SCR_ARCH-NEXT:    ;;#ASMEND
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s2, exec_lo
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 exec_lo, 3
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s3, 0
+; FLAT_SCR_ARCH-NEXT:    scratch_store_dword off, v2, s3
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s3, 4
+; FLAT_SCR_ARCH-NEXT:    scratch_load_dword v2, off, s3 ; 4-byte Folded Reload
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 s3, 0
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt vmcnt(0)
+; FLAT_SCR_ARCH-NEXT:    v_readlane_b32 s0, v2, 0
+; FLAT_SCR_ARCH-NEXT:    v_readlane_b32 s1, v2, 1
+; FLAT_SCR_ARCH-NEXT:    scratch_load_dword v2, off, s3
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt vmcnt(0)
+; FLAT_SCR_ARCH-NEXT:    s_waitcnt_depctr 0xffe3
+; FLAT_SCR_ARCH-NEXT:    s_mov_b32 exec_lo, s2
+; FLAT_SCR_ARCH-NEXT:    v_mov_b32_e32 v1, 0
+; FLAT_SCR_ARCH-NEXT:    global_store_dword v1, v0, s[0:1]
+; FLAT_SCR_ARCH-NEXT:    s_endpgm
   call void asm sideeffect "", "~{s[0:7]}" ()
   call void asm sideeffect "", "~{s[8:15]}" ()
   call void asm sideeffect "", "~{s[16:23]}" ()
   call void asm sideeffect "", "~{s[24:31]}" ()
   call void asm sideeffect "", "~{s[32:39]}" ()
   call void asm sideeffect "", "~{s[40:47]}" ()
   call void asm sideeffect "", "~{s[48:55]}" ()
   call void asm sideeffect "", "~{s[56:63]}" ()