[AMDGPU] Omit buffer resource with flat scratch.

[rampitec]

I have realized that SI_SPILL_* opcodes need a valid SReg_128, even if it is unused and will dropped after expansion. With this change ScratchRSrcReg is a $noreg which is not valid for the verifier.

I am using ttmp tuple as any reserved SReg_128 will work and at the end it will not be used. In general I could create a new unallocatable pseudo RC with SGPR_128 + PRIVATE_RSRC_REG just for that, but it looks like an overkill.

[arsenm]

We should drop the registers from the spill pseudos since we need reserved registers anyway and can insert them when frame indexes are handled

[rampitec]

In D90979#2380367, @arsenm wrote:

We should drop the registers from the spill pseudos since we need reserved registers anyway and can insert them when frame indexes are handled

Makes sense. I can do it as a separate patch, as it is really a separate thing.

llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp

if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) {
     // Special inputs come after user arguments.
     TLI.allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
   }
 
   // Start adding system SGPRs.
   if (IsEntryFunc) {
     TLI.allocateSystemSGPRs(CCInfo, MF, *Info, CC, IsShader);
   } else {
+    if (!Subtarget.enableFlatScratch())
       CCInfo.AllocateReg(Info->getScratchRSrcReg());
     TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
   }
 
   // Move back to the end of the basic block.
   B.setMBB(MBB);
 
   return true;
 }
bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   SmallVector<Register, 8> PhysRegs;
   AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
                                    AssignFnVarArg, false);
   if (!handleAssignments(CCInfo, ArgLocs, MIRBuilder, OutArgs, Handler))
     return false;
 
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
+  if (!ST.enableFlatScratch()) {
     // Insert copies for the SRD. In the HSA case, this should be an identity
     // copy.
     auto ScratchRSrcReg = MIRBuilder.buildCopy(LLT::vector(4, 32),
                                                MFI->getScratchRSrcReg());
     MIRBuilder.buildCopy(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
     MIB.addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Implicit);
+  }
 
   for (std::pair<MCRegister, Register> ArgReg : ImplicitArgRegs) {
     MIRBuilder.buildCopy((Register)ArgReg.first, ArgReg.second);
     MIB.addReg(ArgReg.first, RegState::Implicit);
   }
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = CCInfo.getNextStackOffset();

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

if (!PreloadedScratchWaveOffsetReg)
     return;
 
   // We need to do the replacement of the private segment buffer register even
   // if there are no stack objects. There could be stores to undef or a
   // constant without an associated object.
   //
   // This will return `Register()` in cases where there are no actual
   // uses of the SRSRC.
-  Register ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
+  Register ScratchRsrcReg;
+  if (!ST.enableFlatScratch())
+    ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
 
   // Make the selected register live throughout the function.
   if (ScratchRsrcReg) {
     for (MachineBasicBlock &OtherBB : MF) {
       if (&OtherBB != &MBB) {
         OtherBB.addLiveIn(ScratchRsrcReg);
       }
     }

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

static void reservePrivateMemoryRegs(const TargetMachine &TM,
   // almost certain that spilling will be required.
   if (TM.getOptLevel() == CodeGenOpt::None)
     HasStackObjects = true;
 
   // For now assume stack access is needed in any callee functions, so we need
   // the scratch registers to pass in.
   bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
 
+  if (!ST.enableFlatScratch()) {
     if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) {
       // If we have stack objects, we unquestionably need the private buffer
       // resource. For the Code Object V2 ABI, this will be the first 4 user
       // SGPR inputs. We can reserve those and use them directly.
 
       Register PrivateSegmentBufferReg =
           Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
       Info.setScratchRSrcReg(PrivateSegmentBufferReg);
     } else {
       unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
       // We tentatively reserve the last registers (skipping the last registers
       // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
       // we'll replace these with the ones immediately after those which were
       // really allocated. In the prologue copies will be inserted from the
       // argument to these reserved registers.
 
       // Without HSA, relocations are used for the scratch pointer and the
       // buffer resource setup is always inserted in the prologue. Scratch wave
       // offset is still in an input SGPR.
       Info.setScratchRSrcReg(ReservedBufferReg);
     }
+  }
 
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
   // For entry functions we have to set up the stack pointer if we use it,
   // whereas non-entry functions get this "for free". This means there is no
   // intrinsic advantage to using S32 over S34 in cases where we do not have
   // calls but do need a frame pointer (i.e. if we are requested to have one
   // because frame pointer elimination is disabled). To keep things simple we
SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   int32_t FPDiff = 0;
   MachineFrameInfo &MFI = MF.getFrameInfo();
 
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass
   if (!IsSibCall) {
     Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
 
+    if (!Subtarget->enableFlatScratch()) {
       SmallVector<SDValue, 4> CopyFromChains;
 
       // In the HSA case, this should be an identity copy.
       SDValue ScratchRSrcReg
         = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
       RegsToPass.emplace_back(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
       CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
       Chain = DAG.getTokenFactor(DL, CopyFromChains);
     }
+  }
 
   MVT PtrVT = MVT::i32;
 
   // Walk the register/memloc assignments, inserting copies/loads.
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
     CCValAssign &VA = ArgLocs[i];
     SDValue Arg = OutVals[i];
 

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

if (RI.isSGPRClass(RC)) {
 
     // The SGPR spill/restore instructions only work on number sgprs, so we need
     // to make sure we are using the correct register class.
     if (SrcReg.isVirtual() && SpillSize == 4) {
       MachineRegisterInfo &MRI = MF->getRegInfo();
       MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
     }
 
+    Register ScratchRSrc =
+      ST.enableFlatScratch() ? AMDGPU::TTMP0_TTMP1_TTMP2_TTMP3 // Dummy
+                             : MFI->getScratchRSrcReg();
     BuildMI(MBB, MI, DL, OpDesc)
       .addReg(SrcReg, getKillRegState(isKill)) // data
       .addFrameIndex(FrameIndex)               // addr
       .addMemOperand(MMO)
-      .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
+      .addReg(ScratchRSrc, RegState::Implicit)
       .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
     // Add the scratch resource registers as implicit uses because we may end up
     // needing them, and need to ensure that the reserved registers are
     // correctly handled.
     if (RI.spillSGPRToVGPR())
       FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
     return;
   }
 
   unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
                                     : getVGPRSpillSaveOpcode(SpillSize);
   MFI->setHasSpilledVGPRs();
 
+  Register ScratchRSrc =
+    ST.enableFlatScratch() ? AMDGPU::TTMP0_TTMP1_TTMP2_TTMP3 // Dummy
+                           : MFI->getScratchRSrcReg();
   BuildMI(MBB, MI, DL, get(Opcode))
     .addReg(SrcReg, getKillRegState(isKill)) // data
     .addFrameIndex(FrameIndex)               // addr
-    .addReg(MFI->getScratchRSrcReg())        // scratch_rsrc
+    .addReg(ScratchRSrc)                     // scratch_rsrc
     .addReg(MFI->getStackPtrOffsetReg())     // scratch_offset
     .addImm(0)                               // offset
     .addMemOperand(MMO);
 }
 
 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
   switch (Size) {
   case 4:
if (RI.isSGPRClass(RC)) {
     // FIXME: Maybe this should not include a memoperand because it will be
     // lowered to non-memory instructions.
     const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
     if (DestReg.isVirtual() && SpillSize == 4) {
       MachineRegisterInfo &MRI = MF->getRegInfo();
       MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
     }
 
+    Register ScratchRSrc =
+      ST.enableFlatScratch() ? AMDGPU::TTMP0_TTMP1_TTMP2_TTMP3 // Dummy
+                             : MFI->getScratchRSrcReg();
     if (RI.spillSGPRToVGPR())
       FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
     BuildMI(MBB, MI, DL, OpDesc, DestReg)
       .addFrameIndex(FrameIndex) // addr
       .addMemOperand(MMO)
-      .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
+      .addReg(ScratchRSrc, RegState::Implicit)
       .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
     return;
   }
 
   unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
                                     : getVGPRSpillRestoreOpcode(SpillSize);
+  Register ScratchRSrc =
+    ST.enableFlatScratch() ? AMDGPU::TTMP0_TTMP1_TTMP2_TTMP3 // Dummy
+                           : MFI->getScratchRSrcReg();
   BuildMI(MBB, MI, DL, get(Opcode), DestReg)
     .addFrameIndex(FrameIndex)        // vaddr
-    .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
+    .addReg(ScratchRSrc)              // scratch_rsrc
     .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
     .addImm(0)                           // offset
     .addMemOperand(MMO);
 }
 
 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator MI) const {
   insertNoops(MBB, MI, 1);

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

if (!F.arg_empty())
       KernargSegmentPtr = true;
     WorkGroupIDX = true;
     WorkItemIDX = true;
   } else if (CC == CallingConv::AMDGPU_PS) {
     PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
   }
 
   if (!isEntryFunction()) {
-    // Non-entry functions have no special inputs for now, other registers
-    // required for scratch access.
-    ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
-
     // TODO: Pick a high register, and shift down, similar to a kernel.
     FrameOffsetReg = AMDGPU::SGPR33;
     StackPtrOffsetReg = AMDGPU::SGPR32;
 
+    if (!ST.enableFlatScratch()) {
+      // Non-entry functions have no special inputs for now, other registers
+      // required for scratch access.
+      ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
+
       ArgInfo.PrivateSegmentBuffer =
         ArgDescriptor::createRegister(ScratchRSrcReg);
+    }
 
     if (F.hasFnAttribute("amdgpu-implicitarg-ptr"))
       ImplicitArgPtr = true;
   } else {
     if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) {
       KernargSegmentPtr = true;
       MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
                                  MaxKernArgAlign);
if (isEntryFunction()) {
     if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
         (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
       ArgInfo.PrivateSegmentWaveByteOffset =
           ArgDescriptor::createRegister(AMDGPU::SGPR5);
   }
 
   bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
   if (isAmdHsaOrMesa) {
+    if (!ST.enableFlatScratch())
       PrivateSegmentBuffer = true;
 
     if (UseFixedABI) {
       DispatchPtr = true;
       QueuePtr = true;
 
       // FIXME: We don't need this?
       DispatchID = true;
     } else {

llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll

 ; GCN: v_writelane_b32 v40, s35, 1
 ; GCN: v_writelane_b32 v40, s30, 2
 ; GCN: v_writelane_b32 v40, s31, 3
 
 ; GCN: s_swappc_b64
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NEXT: s_swappc_b64
-; GCN-DAG: v_readlane_b32 s4, v40, 2
-; GCN-DAG: v_readlane_b32 s5, v40, 3
+; MUBUF-DAG:   v_readlane_b32 s4, v40, 2
+; MUBUF-DAG:   v_readlane_b32 s5, v40, 3
+; FLATSCR-DAG: v_readlane_b32 s0, v40, 2
+; FLATSCR-DAG: v_readlane_b32 s1, v40, 3
 ; GCN: v_readlane_b32 s35, v40, 1
 ; GCN: v_readlane_b32 s34, v40, 0
 
 ; GCN: v_readlane_b32 s33, v40, 4
 ; MUBUF:   buffer_load_dword
 ; FLATSCR: scratch_load_dword
 ; GCN: s_setpc_b64
 define void @test_func_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void() #0 {
define amdgpu_kernel void @test_call_void_func_void_mayclobber_v31(i32 addrspace(1)* %out) #0 {
   call void @external_void_func_void()
   call void asm sideeffect "; use $0", "{v31}"(i32 %v31)
   ret void
 }
 
 ; FIXME: What is the expected behavior for reserved registers here?
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_preserves_s33:
-; GCN: s_getpc_b64 s[4:5]
-; GCN-NEXT: s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
-; GCN-NEXT: s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; MUBUF:        s_getpc_b64 s[4:5]
+; MUBUF-NEXT:   s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
+; MUBUF-NEXT:   s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; FLATSCR:      s_getpc_b64 s[0:1]
+; FLATSCR-NEXT: s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
+; FLATSCR-NEXT: s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
 ; GCN: s_mov_b32 s32, 0
 ; GCN: #ASMSTART
 ; GCN-NEXT: ; def s33
 ; GCN-NEXT: #ASMEND
-; GCN: s_swappc_b64 s[30:31], s[4:5]
+; MUBUF:   s_swappc_b64 s[30:31], s[4:5]
+; FLATSCR: s_swappc_b64 s[30:31], s[0:1]
 ; GCN: ;;#ASMSTART
 ; GCN-NEXT: ; use s33
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NOT: s33
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @test_call_void_func_void_preserves_s33(i32 addrspace(1)* %out) #0 {
   %s33 = call i32 asm sideeffect "; def $0", "={s33}"()
   call void @external_void_func_void()
   call void asm sideeffect "; use $0", "{s33}"(i32 %s33)
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_preserves_s34: {{.*}}
 ; GCN-NOT: s34
 
-; GCN: s_getpc_b64 s[4:5]
-; GCN-NEXT: s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
-; GCN-NEXT: s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; MUBUF:        s_getpc_b64 s[4:5]
+; MUBUF-NEXT:   s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
+; MUBUF-NEXT:   s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; FLATSCR:      s_getpc_b64 s[0:1]
+; FLATSCR-NEXT: s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
+; FLATSCR-NEXT: s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
 ; GCN: s_mov_b32 s32, 0
 
 ; GCN-NOT: s34
 ; GCN: ;;#ASMSTART
 ; GCN-NEXT: ; def s34
 ; GCN-NEXT: ;;#ASMEND
 
 ; GCN-NOT: s34
-; GCN: s_swappc_b64 s[30:31], s[4:5]
+; MUBUF:   s_swappc_b64 s[30:31], s[4:5]
+; FLATSCR: s_swappc_b64 s[30:31], s[0:1]
 
 ; GCN-NOT: s34
 
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: ; use s34
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @test_call_void_func_void_preserves_s34(i32 addrspace(1)* %out) #0 {
   %s34 = call i32 asm sideeffect "; def $0", "={s34}"()
   call void @external_void_func_void()
   call void asm sideeffect "; use $0", "{s34}"(i32 %s34)
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_call_void_func_void_preserves_v40: {{.*}}
 
 ; GCN-NOT: v32
-; GCN: s_getpc_b64 s[4:5]
-; GCN-NEXT: s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
-; GCN-NEXT: s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; MUBUF: s_getpc_b64 s[4:5]
+; MUBUF-NEXT:   s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
+; MUBUF-NEXT:   s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
+; FLATSCR:      s_getpc_b64 s[0:1]
+; FLATSCR-NEXT: s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
+; FLATSCR-NEXT: s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
 ; GCN: s_mov_b32 s32, 0
 ; GCN-NOT: v40
 
 ; GCN: ;;#ASMSTART
 ; GCN-NEXT: ; def v40
 ; GCN-NEXT: ;;#ASMEND
 
-; GCN: s_swappc_b64 s[30:31], s[4:5]
+; MUBUF:   s_swappc_b64 s[30:31], s[4:5]
+; FLATSCR: s_swappc_b64 s[30:31], s[0:1]
 
 ; GCN-NOT: v40
 
 ; GCN: ;;#ASMSTART
 ; GCN-NEXT: ; use v40
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @test_call_void_func_void_preserves_v40(i32 addrspace(1)* %out) #0 {

llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll

 ; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,CI,MUBUF %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,GFX9,MUBUF %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-flat-scratch < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,FLATSCR %s
 
 ; GCN-LABEL: {{^}}callee_no_stack:
 ; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @callee_no_stack() #0 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_all:
 ; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_mov_b32 s4, s33
+; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
+; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
 ; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: s_mov_b32 s33, s4
+; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
 ; GCN-NEXT: s_setpc_b64
 define void @callee_no_stack_no_fp_elim_all() #1 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_nonleaf:
 ; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
define void @callee_with_stack() #0 {
   ret void
 }
 
 ; Can use free call clobbered register to preserve original FP value.
 
 ; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_all:
 ; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
-; GCN-NEXT: s_mov_b32 s4, s33
+; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
+; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
 ; GCN-NEXT: s_mov_b32 s33, s32
 ; MUBUF-NEXT:   s_add_u32 s32, s32, 0x200
 ; FLATSCR-NEXT: s_add_u32 s32, s32, 8
 ; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
 ; MUBUF-NEXT:   buffer_store_dword v0, off, s[0:3], s33 offset:4{{$}}
 ; FLATSCR-NEXT: scratch_store_dword off, v0, s33 offset:4{{$}}
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x200
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 8
-; GCN-NEXT: s_mov_b32 s33, s4
+; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_no_fp_elim_all() #1 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   ret void
 }
 

 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30,
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31,
 
 ; MUBUF-DAG:   buffer_store_dword [[ZERO]], off, s[0:3], s33{{$}}
 ; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33{{$}}
 
 ; GCN: s_swappc_b64
 
-; GCN-DAG: v_readlane_b32 s5, [[CSR_VGPR]]
-; GCN-DAG: v_readlane_b32 s4, [[CSR_VGPR]]
+; MUBUF-DAG: v_readlane_b32 s5, [[CSR_VGPR]]
+; MUBUF-DAG: v_readlane_b32 s4, [[CSR_VGPR]]
+; FLATSCR-DAG: v_readlane_b32 s0, [[CSR_VGPR]]
+; FLATSCR-DAG: v_readlane_b32 s1, [[CSR_VGPR]]
 
 ; MUBUF:    s_sub_u32 s32, s32, 0x400{{$}}
 ; FLATSCR:  s_sub_u32 s32, s32, 16{{$}}
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:4 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]

 ; MUBUF-DAG:   s_add_u32 s32, s32, 0x400
 ; FLATSCR-DAG: s_add_u32 s32, s32, 16
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s33, [[FP_SPILL_LANE:[0-9]+]]
 
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; GCN: s_swappc_b64
 
-; GCN-DAG: v_readlane_b32 s4, v40, 0
-; GCN-DAG: v_readlane_b32 s5, v40, 1
+; MUBUF-DAG: v_readlane_b32 s4, v40, 0
+; MUBUF-DAG: v_readlane_b32 s5, v40, 1
+; FLATSCR-DAG: v_readlane_b32 s0, v40, 0
+; FLATSCR-DAG: v_readlane_b32 s1, v40, 1
 
 ; MUBUF:   s_sub_u32 s32, s32, 0x400
 ; FLATSCR: s_sub_u32 s32, s32, 16
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], [[FP_SPILL_LANE]]
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
 ; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]

 ; GCN:  ;;#ASMSTART
 ; GCN-NEXT: ; clobber v41
 ; GCN-NEXT: ;;#ASMEND
 
 ; MUBUF:   buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
 ; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
 ; MUBUF:        s_add_u32 s32, s32, 0x300
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300
+; MUBUF-NEXT:   s_mov_b32 s33, s4
 ; FLATSCR:      s_add_u32 s32, s32, 12
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 12
-; GCN-NEXT: s_mov_b32 s33, s4
+; FLATSCR-NEXT: s_mov_b32 s33, s0
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_no_fp_elim_csr_vgpr() #1 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   call void asm sideeffect "; clobber v41", "~{v41}"()
   ret void
 }
define void @no_new_vgpr_for_fp_csr() #1 {
 
   ret void
 }
 
 ; GCN-LABEL: {{^}}realign_stack_no_fp_elim:
 ; GCN: s_waitcnt
 ; MUBUF-NEXT:   s_add_u32 [[SCRATCH:s[0-9]+]], s32, 0x7ffc0
 ; FLATSCR-NEXT: s_add_u32 [[SCRATCH:s[0-9]+]], s32, 0x1fff
-; GCN-NEXT:     s_mov_b32 s4, s33
+; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
+; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
 ; MUBUF-NEXT:   s_and_b32 s33, [[SCRATCH]], 0xfff80000
 ; FLATSCR-NEXT: s_and_b32 s33, [[SCRATCH]], 0xffffe000
 ; MUBUF-NEXT:   s_add_u32 s32, s32, 0x100000
 ; FLATSCR-NEXT: s_add_u32 s32, s32, 0x4000
 ; GCN-NEXT:     v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
 ; MUBUF-NEXT:   buffer_store_dword [[ZERO]], off, s[0:3], s33
 ; FLATSCR-NEXT: scratch_store_dword off, [[ZERO]], s33
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x100000
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 0x4000
-; GCN-NEXT: s_mov_b32 s33, s4
+; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @realign_stack_no_fp_elim() #1 {
   %alloca = alloca i32, align 8192, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   ret void
 }
 
 ; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp:
 ; GCN: s_waitcnt
 ; GCN-NEXT: v_writelane_b32 v1, s33, 2
 ; GCN-NEXT: v_writelane_b32 v1, s30, 0
 ; GCN-NEXT: s_mov_b32 s33, s32
 ; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
 ; GCN: v_writelane_b32 v1, s31, 1
 ; MUBUF:   buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:4
 ; FLATSCR: scratch_store_dword off, [[ZERO]], s33 offset:4
 ; GCN: ;;#ASMSTART
-; GCN: v_readlane_b32 s4, v1, 0
+; MUBUF:        v_readlane_b32 s4, v1, 0
 ; MUBUF-NEXT:   s_add_u32 s32, s32, 0x200
+; MUBUF-NEXT:   v_readlane_b32 s5, v1, 1
+; FLATSCR:      v_readlane_b32 s0, v1, 0
 ; FLATSCR-NEXT: s_add_u32 s32, s32, 8
-; GCN-NEXT:     v_readlane_b32 s5, v1, 1
+; FLATSCR-NEXT: v_readlane_b32 s1, v1, 1
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x200
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 8
-; GCN-NEXT: v_readlane_b32 s33, v1, 2
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[4:5]
+; GCN-NEXT:     v_readlane_b32 s33, v1, 2
+; GCN-NEXT:     s_waitcnt vmcnt(0)
+; MUBUF-NEXT:   s_setpc_b64 s[4:5]
+; FLATSCR-NEXT: s_setpc_b64 s[0:1]
 define void @no_unused_non_csr_sgpr_for_fp() #1 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
 
   ; Use all clobberable registers, so FP has to spill to a VGPR.
   call void asm sideeffect "",
     "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}

 ; GCN-NEXT: s_mov_b32 s33, s32
 
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; MUBUF-DAG:   buffer_store_dword
 ; FLATSCR-DAG: scratch_store_dword
 ; MUBUF:       s_add_u32 s32, s32, 0x300{{$}}
 ; FLATSCR:     s_add_u32 s32, s32, 12{{$}}
 
-; GCN:          v_readlane_b32 s4, [[CSR_VGPR]], 0
+; MUBUF:        v_readlane_b32 s4, [[CSR_VGPR]], 0
+; FLATSCR:      v_readlane_b32 s0, [[CSR_VGPR]], 0
 ; GCN: ;;#ASMSTART
-; GCN:          v_readlane_b32 s5, [[CSR_VGPR]], 1
+; MUBUF:        v_readlane_b32 s5, [[CSR_VGPR]], 1
+; FLATSCR:      v_readlane_b32 s1, [[CSR_VGPR]], 1
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300{{$}}
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 12{{$}}
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:8 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)

 ; GCN-DAG:  v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-DAG:  s_mov_b32 s33, s32
 ; GCN-DAG:  v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; MUBUF-DAG:   s_add_u32 s32, s32, 0x40300{{$}}
 ; FLATSCR-DAG: s_add_u32 s32, s32, 0x100c{{$}}
 ; MUBUF-DAG:   buffer_store_dword
 ; FLATSCR-DAG: scratch_store_dword
 
-; GCN: v_readlane_b32 s4, [[CSR_VGPR]], 0
+; MUBUF:   v_readlane_b32 s4, [[CSR_VGPR]], 0
+; FLATSCR: v_readlane_b32 s0, [[CSR_VGPR]], 0
 ; GCN: ;;#ASMSTART
-; GCN: v_readlane_b32 s5, [[CSR_VGPR]], 1
+; MUBUF:   v_readlane_b32 s5, [[CSR_VGPR]], 1
+; FLATSCR: v_readlane_b32 s1, [[CSR_VGPR]], 1
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x40300{{$}}
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 0x100c{{$}}
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; MUBUF-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
 ; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Reload
 ; FLATSCR-NEXT: s_add_u32 [[SCRATCH_SGPR:s[0-9]+]], s32, 0x1008
 ; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, [[SCRATCH_SGPR]] ; 4-byte Folded Reload
define void @ipra_call_with_stack() #0 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   call void @local_empty_func()
   ret void
 }
 
 ; With no free registers, we must spill the FP to memory.
 ; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory:
-; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN: v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
+; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
+; MUBUF:   v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
 ; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:4
-; FLATSCR: scratch_store_dword off, [[TMP_VGPR1]], s32 offset:4
-; GCN: s_mov_b64 exec, [[COPY_EXEC1]]
-; GCN: s_mov_b32 s33, s32
-; GCN: s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
+; MUBUF:   s_mov_b64 exec, [[COPY_EXEC1]]
+; FLATSCR: s_mov_b32 s0, s33
+; GCN:     s_mov_b32 s33, s32
+; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:4
-; FLATSCR: scratch_load_dword [[TMP_VGPR2:v[0-9]+]], off, s32 offset:4
-; GCN: s_waitcnt vmcnt(0)
-; GCN: v_readfirstlane_b32 s33, [[TMP_VGPR2]]
-; GCN: s_mov_b64 exec, [[COPY_EXEC2]]
-; GCN: s_setpc_b64
-; GCN: ScratchSize: 8
+; FLATSCR: s_mov_b32 s33, s0
+; MUBUF:   s_waitcnt vmcnt(0)
+; MUBUF:   v_readfirstlane_b32 s33, [[TMP_VGPR2]]
+; MUBUF:   s_mov_b64 exec, [[COPY_EXEC2]]
+; GCN:     s_setpc_b64
+; MUBUF:   ScratchSize: 8
+; FLATSCR: ScratchSize: 0
 define void @callee_need_to_spill_fp_to_memory() #3 {
   call void asm sideeffect "; clobber nonpreserved SGPRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
     ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
     ,~{vcc}"()
 
   call void asm sideeffect "; clobber all VGPRs",
     "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
     ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
     ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
     ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"()
   ret void
 }
 
 ; If we have a reserved VGPR that can be used for SGPR spills, we may still
 ; need to spill the FP to memory if there are no free lanes in the reserved
 ; VGPR.
 ; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory_full_reserved_vgpr:
-; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN: v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
+; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
+; MUBUF:   v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
 ; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:[[OFF:[0-9]+]]
-; FLATSCR: scratch_store_dword off, [[TMP_VGPR1]], s32 offset:[[OFF:[0-9]+]]
-; GCN: s_mov_b64 exec, [[COPY_EXEC1]]
+; MUBUF:   s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NOT: v_writelane_b32 v40, s33
-; GCN: s_mov_b32 s33, s32
+; MUBUF:   s_mov_b32 s33, s32
+; FLATSCR: s_mov_b32 s33, s0
 ; GCN-NOT: v_readlane_b32 s33, v40
-; GCN: s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
+; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:[[OFF]]
-; FLATSCR: scratch_load_dword [[TMP_VGPR2:v[0-9]+]], off, s32 offset:[[OFF]]
-; GCN: v_readfirstlane_b32 s33, [[TMP_VGPR2]]
-; GCN: s_mov_b64 exec, [[COPY_EXEC2]]
-; GCN: s_setpc_b64
+; MUBUF:   v_readfirstlane_b32 s33, [[TMP_VGPR2]]
+; MUBUF:   s_mov_b64 exec, [[COPY_EXEC2]]
+; GCN:     s_setpc_b64
 define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
   call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
     ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
     ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
     ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
     ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
     ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
     ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
     ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
     ,~{s100},~{s101},~{s102},~{s39},~{vcc}"()
 
   call void asm sideeffect "; clobber all VGPRs except CSR v40",
     "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
     ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
     ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
     ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"()
   ret void
 }
 
 ; If the size of the offset exceeds the MUBUF offset field we need another
 ; scratch VGPR to hold the offset.
 ; GCN-LABEL: {{^}}spill_fp_to_memory_scratch_reg_needed_mubuf_offset
-; GCN: s_or_saveexec_b64 s[4:5], -1
+; MUBUF: s_or_saveexec_b64 s[4:5], -1
 ; MUBUF: v_mov_b32_e32 v0, s33
 ; GCN-NOT: v_mov_b32_e32 v0, 0x1008
 ; MUBUF-NEXT: v_mov_b32_e32 v1, 0x1008
 ; MUBUF-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen ; 4-byte Folded Spill
-; FLATSCR-NEXT: s_add_u32 [[SOFF:s[0-9]+]], s32, 0x1008
-; FLATSCR-NEXT: v_mov_b32_e32 v0, s33
-; FLATSCR-NEXT: scratch_store_dword off, v0, [[SOFF]] ; 4-byte Folded Spill
+; FLATSCR: s_add_u32 [[SOFF:s[0-9]+]], s33, 0x1004
+; FLATSCR: v_mov_b32_e32 v0, 0
+; FLATSCR: scratch_store_dword off, v0, [[SOFF]]
 define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval align 4 %arg) #3 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
 
   call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
     ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}

llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll

 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    buffer_load_short_d16_hi v0, off, s[0:3], 0
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: chain_hi_to_lo_private:
 ; FLATSCR:       ; %bb.0: ; %bb
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_mov_b32 s4, 2
-; FLATSCR-NEXT:    scratch_load_ushort v0, off, s4
-; FLATSCR-NEXT:    s_mov_b32 s4, 0
+; FLATSCR-NEXT:    s_mov_b32 s0, 2
+; FLATSCR-NEXT:    scratch_load_ushort v0, off, s0
+; FLATSCR-NEXT:    s_mov_b32 s0, 0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; FLATSCR-NEXT:    scratch_load_short_d16_hi v0, off, s4
+; FLATSCR-NEXT:    scratch_load_short_d16_hi v0, off, s0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %gep_lo = getelementptr inbounds half, half addrspace(5)* null, i64 1
   %load_lo = load half, half addrspace(5)* %gep_lo
   %gep_hi = getelementptr inbounds half, half addrspace(5)* null, i64 0
   %load_hi = load half, half addrspace(5)* %gep_hi
 

 ; GFX900-NEXT:    buffer_load_short_d16_hi v3, off, s[0:3], 0 offset:8
 ; GFX900-NEXT:    v_lshl_or_b32 v2, v4, 16, v2
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; GFX900-NEXT:    s_endpgm
 ;
 ; FLATSCR-LABEL: vload2_private:
 ; FLATSCR:       ; %bb.0: ; %entry
-; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s6, s9
-; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
-; FLATSCR-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
+; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
+; FLATSCR-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT:    v_mov_b32_e32 v0, s4
-; FLATSCR-NEXT:    v_mov_b32_e32 v1, s5
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, s0
+; FLATSCR-NEXT:    v_mov_b32_e32 v1, s1
 ; FLATSCR-NEXT:    global_load_ushort v2, v[0:1], off
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    scratch_store_short off, v2, vcc_hi offset:4
 ; FLATSCR-NEXT:    global_load_ushort v2, v[0:1], off offset:2
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    scratch_store_short off, v2, vcc_hi offset:6
 ; FLATSCR-NEXT:    global_load_ushort v2, v[0:1], off offset:4
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
-; FLATSCR-NEXT:    v_mov_b32_e32 v0, s6
-; FLATSCR-NEXT:    v_mov_b32_e32 v1, s7
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, s2
+; FLATSCR-NEXT:    v_mov_b32_e32 v1, s3
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    scratch_store_short off, v2, vcc_hi offset:8
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
 ; FLATSCR-NEXT:    scratch_load_ushort v2, off, vcc_hi offset:4
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
 ; FLATSCR-NEXT:    scratch_load_ushort v4, off, vcc_hi offset:6
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(1)

llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll

 ; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-FLASTSCR-LABEL: private_load_2xi16_align1:
 ; GFX9-FLASTSCR:       ; %bb.0:
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-FLASTSCR-NEXT:    scratch_load_dword v0, v0, off
 ; GFX9-FLASTSCR-NEXT:    v_mov_b32_e32 v1, 0xffff
-; GFX9-FLASTSCR-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-FLASTSCR-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-FLASTSCR-NEXT:    v_bfi_b32 v1, v1, 0, v0
-; GFX9-FLASTSCR-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-FLASTSCR-NEXT:    v_and_or_b32 v0, v0, s0, v1
 ; GFX9-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
   %gep.p = getelementptr i16, i16 addrspace(5)* %p, i64 1
   %p.0 = load i16, i16 addrspace(5)* %p, align 1
   %p.1 = load i16, i16 addrspace(5)* %gep.p, align 1
   %zext.0 = zext i16 %p.0 to i32
   %zext.1 = zext i16 %p.1 to i32
   %shl.1 = shl i32 %zext.1, 16
   %or = or i32 %zext.0, %shl.1

 ; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-FLASTSCR-LABEL: private_load_2xi16_align4:
 ; GFX9-FLASTSCR:       ; %bb.0:
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-FLASTSCR-NEXT:    scratch_load_dword v0, v0, off
 ; GFX9-FLASTSCR-NEXT:    v_mov_b32_e32 v1, 0xffff
-; GFX9-FLASTSCR-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-FLASTSCR-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-FLASTSCR-NEXT:    v_bfi_b32 v1, v1, 0, v0
-; GFX9-FLASTSCR-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-FLASTSCR-NEXT:    v_and_or_b32 v0, v0, s0, v1
 ; GFX9-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
   %gep.p = getelementptr i16, i16 addrspace(5)* %p, i64 1
   %p.0 = load i16, i16 addrspace(5)* %p, align 4
   %p.1 = load i16, i16 addrspace(5)* %gep.p, align 2
   %zext.0 = zext i16 %p.0 to i32
   %zext.1 = zext i16 %p.1 to i32
   %shl.1 = shl i32 %zext.1, 16
   %or = or i32 %zext.0, %shl.1

llvm/test/CodeGen/AMDGPU/flat-scratch.ll

bb:
   %i12 = load volatile i32, i32 addrspace(5)* %i10, align 4
   ret void
 }
 
 define void @store_load_large_imm_offset_foo() {
 ; GFX9-LABEL: store_load_large_imm_offset_foo:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s4, 0x3000
+; GFX9-NEXT:    s_movk_i32 s0, 0x3000
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX9-NEXT:    scratch_store_dword off, v0, s32
-; GFX9-NEXT:    s_add_u32 s4, s32, s4
+; GFX9-NEXT:    s_add_u32 s0, s32, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15
-; GFX9-NEXT:    scratch_store_dword off, v0, s4 offset:3712
-; GFX9-NEXT:    scratch_load_dword v0, off, s4 offset:3712
+; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:3712
+; GFX9-NEXT:    scratch_load_dword v0, off, s0 offset:3712
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: store_load_large_imm_offset_foo:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
-; GFX10-NEXT:    s_movk_i32 s4, 0x3800
+; GFX10-NEXT:    s_movk_i32 s0, 0x3800
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_add_u32 s4, s32, s4
+; GFX10-NEXT:    s_add_u32 s0, s32, s0
 ; GFX10-NEXT:    scratch_store_dword off, v0, s32
-; GFX10-NEXT:    scratch_store_dword off, v1, s4 offset:1664
-; GFX10-NEXT:    scratch_load_dword v0, off, s4 offset:1664
+; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
+; GFX10-NEXT:    scratch_load_dword v0, off, s0 offset:1664
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = alloca [4096 x i32], align 4, addrspace(5)
   %i1 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 undef
   store volatile i32 13, i32 addrspace(5)* %i1, align 4
   %i7 = getelementptr inbounds [4096 x i32], [4096 x i32] addrspace(5)* %i, i32 0, i32 4000

llvm/test/CodeGen/AMDGPU/load-lo16.ll

 ; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-FLATSCR-LABEL: load_private_lo_v2i16_reglo_vreg_nooff:
 ; GFX900-FLATSCR:       ; %bb.0: ; %entry
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-FLATSCR-NEXT:    s_movk_i32 s4, 0xffe
-; GFX900-FLATSCR-NEXT:    scratch_load_short_d16 v1, off, s4
+; GFX900-FLATSCR-NEXT:    s_movk_i32 s0, 0xffe
+; GFX900-FLATSCR-NEXT:    scratch_load_short_d16 v1, off, s0
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i16, i16 addrspace(5)* inttoptr (i32 4094 to i16 addrspace(5)*)
   %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0

 ; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-FLATSCR-LABEL: load_private_lo_v2i16_reghi_vreg_nooff:
 ; GFX900-FLATSCR:       ; %bb.0: ; %entry
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-FLATSCR-NEXT:    s_movk_i32 s4, 0xffe
-; GFX900-FLATSCR-NEXT:    scratch_load_short_d16 v1, off, s4
+; GFX900-FLATSCR-NEXT:    s_movk_i32 s0, 0xffe
+; GFX900-FLATSCR-NEXT:    scratch_load_short_d16 v1, off, s0
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i16, i16 addrspace(5)* inttoptr (i32 4094 to i16 addrspace(5)*)
   %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0

 ; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-FLATSCR-LABEL: load_private_lo_v2f16_reglo_vreg_nooff:
 ; GFX900-FLATSCR:       ; %bb.0: ; %entry
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-FLATSCR-NEXT:    s_movk_i32 s4, 0xffe
-; GFX900-FLATSCR-NEXT:    scratch_load_short_d16 v1, off, s4
+; GFX900-FLATSCR-NEXT:    s_movk_i32 s0, 0xffe
+; GFX900-FLATSCR-NEXT:    scratch_load_short_d16 v1, off, s0
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load volatile half, half addrspace(5)* inttoptr (i32 4094 to half addrspace(5)*)
   %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0

 ; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-FLATSCR-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_zexti8:
 ; GFX900-FLATSCR:       ; %bb.0: ; %entry
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-FLATSCR-NEXT:    s_movk_i32 s4, 0xffe
-; GFX900-FLATSCR-NEXT:    scratch_load_ubyte_d16 v1, off, s4
+; GFX900-FLATSCR-NEXT:    s_movk_i32 s0, 0xffe
+; GFX900-FLATSCR-NEXT:    scratch_load_ubyte_d16 v1, off, s0
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
   %ext = zext i8 %load to i16

 ; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-FLATSCR-LABEL: load_private_lo_v2i16_reglo_vreg_nooff_sexti8:
 ; GFX900-FLATSCR:       ; %bb.0: ; %entry
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-FLATSCR-NEXT:    s_movk_i32 s4, 0xffe
-; GFX900-FLATSCR-NEXT:    scratch_load_sbyte_d16 v1, off, s4
+; GFX900-FLATSCR-NEXT:    s_movk_i32 s0, 0xffe
+; GFX900-FLATSCR-NEXT:    scratch_load_sbyte_d16 v1, off, s0
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x i16>
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
   %ext = sext i8 %load to i16

 ; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-FLATSCR-LABEL: load_private_lo_v2f16_reglo_vreg_nooff_zexti8:
 ; GFX900-FLATSCR:       ; %bb.0: ; %entry
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-FLATSCR-NEXT:    s_movk_i32 s4, 0xffe
-; GFX900-FLATSCR-NEXT:    scratch_load_ubyte_d16 v1, off, s4
+; GFX900-FLATSCR-NEXT:    s_movk_i32 s0, 0xffe
+; GFX900-FLATSCR-NEXT:    scratch_load_ubyte_d16 v1, off, s0
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX900-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %reg.bc = bitcast i32 %reg to <2 x half>
   %load = load volatile i8, i8 addrspace(5)* inttoptr (i32 4094 to i8 addrspace(5)*)
   %ext = zext i8 %load to i16

llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll

 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
 ; MUBUF-NEXT:    v_mov_b32_e32 v3, s5
 ; MUBUF-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
 ; MUBUF-NEXT:    s_endpgm
 ;
 ; FLATSCR-LABEL: local_stack_offset_uses_sp:
 ; FLATSCR:       ; %bb.0: ; %entry
-; FLATSCR-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s6, s9
-; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
+; FLATSCR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
+; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
+; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    s_movk_i32 vcc_hi, 0x2000
-; FLATSCR-NEXT:    s_mov_b32 s6, 0
+; FLATSCR-NEXT:    s_mov_b32 s2, 0
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, vcc_hi
 ; FLATSCR-NEXT:  BB0_1: ; %loadstoreloop
 ; FLATSCR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; FLATSCR-NEXT:    s_add_u32 s7, 0x3000, s6
-; FLATSCR-NEXT:    s_add_i32 s6, s6, 1
-; FLATSCR-NEXT:    s_cmpk_lt_u32 s6, 0x2120
-; FLATSCR-NEXT:    scratch_store_byte off, v0, s7
+; FLATSCR-NEXT:    s_add_u32 s3, 0x3000, s2
+; FLATSCR-NEXT:    s_add_i32 s2, s2, 1
+; FLATSCR-NEXT:    s_cmpk_lt_u32 s2, 0x2120
+; FLATSCR-NEXT:    scratch_store_byte off, v0, s3
 ; FLATSCR-NEXT:    s_cbranch_scc1 BB0_1
 ; FLATSCR-NEXT:  ; %bb.2: ; %split
-; FLATSCR-NEXT:    s_movk_i32 s6, 0x20d0
-; FLATSCR-NEXT:    s_add_u32 s6, 0x3000, s6
-; FLATSCR-NEXT:    scratch_load_dword v1, off, s6 offset:4
-; FLATSCR-NEXT:    s_movk_i32 s6, 0x2000
-; FLATSCR-NEXT:    s_add_u32 s6, 0x3000, s6
-; FLATSCR-NEXT:    scratch_load_dword v0, off, s6 offset:208
-; FLATSCR-NEXT:    s_movk_i32 s6, 0x3000
-; FLATSCR-NEXT:    scratch_load_dword v2, off, s6 offset:68
-; FLATSCR-NEXT:    s_movk_i32 s6, 0x3000
-; FLATSCR-NEXT:    scratch_load_dword v3, off, s6 offset:64
+; FLATSCR-NEXT:    s_movk_i32 s2, 0x20d0
+; FLATSCR-NEXT:    s_add_u32 s2, 0x3000, s2
+; FLATSCR-NEXT:    scratch_load_dword v1, off, s2 offset:4
+; FLATSCR-NEXT:    s_movk_i32 s2, 0x2000
+; FLATSCR-NEXT:    s_add_u32 s2, 0x3000, s2
+; FLATSCR-NEXT:    scratch_load_dword v0, off, s2 offset:208
+; FLATSCR-NEXT:    s_movk_i32 s2, 0x3000
+; FLATSCR-NEXT:    scratch_load_dword v2, off, s2 offset:68
+; FLATSCR-NEXT:    s_movk_i32 s2, 0x3000
+; FLATSCR-NEXT:    scratch_load_dword v3, off, s2 offset:64
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v3
 ; FLATSCR-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT:    v_mov_b32_e32 v2, s4
-; FLATSCR-NEXT:    v_mov_b32_e32 v3, s5
+; FLATSCR-NEXT:    v_mov_b32_e32 v3, s1
+; FLATSCR-NEXT:    v_mov_b32_e32 v2, s0
 ; FLATSCR-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
 ; FLATSCR-NEXT:    s_endpgm
 entry:
   %pin.low = alloca i32, align 8192, addrspace(5)
   %local.area = alloca [1060 x i64], align 4096, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %pin.low
   %local.area.cast = bitcast [1060 x i64] addrspace(5)* %local.area to i8 addrspace(5)*
   call void @llvm.memset.p5i8.i32(i8 addrspace(5)* align 4 %local.area.cast, i8 0, i32 8480, i1 true)

 ; MUBUF-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v6, vcc
 ; MUBUF-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: func_local_stack_offset_uses_sp:
 ; FLATSCR:       ; %bb.0: ; %entry
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_add_u32 s4, s32, 0x1fff
-; FLATSCR-NEXT:    s_mov_b32 s6, s33
-; FLATSCR-NEXT:    s_and_b32 s33, s4, 0xffffe000
+; FLATSCR-NEXT:    s_add_u32 s0, s32, 0x1fff
+; FLATSCR-NEXT:    s_mov_b32 s2, s33
+; FLATSCR-NEXT:    s_and_b32 s33, s0, 0xffffe000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 0
-; FLATSCR-NEXT:    s_mov_b32 s4, 0
+; FLATSCR-NEXT:    s_mov_b32 s0, 0
 ; FLATSCR-NEXT:    s_add_u32 s32, s32, 0x6000
 ; FLATSCR-NEXT:    scratch_store_dword off, v2, s33
 ; FLATSCR-NEXT:  BB1_1: ; %loadstoreloop
 ; FLATSCR-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; FLATSCR-NEXT:    s_add_u32 vcc_hi, s33, 0x1000
-; FLATSCR-NEXT:    s_add_u32 s5, vcc_hi, s4
-; FLATSCR-NEXT:    s_add_i32 s4, s4, 1
-; FLATSCR-NEXT:    s_cmpk_lt_u32 s4, 0x2120
-; FLATSCR-NEXT:    scratch_store_byte off, v2, s5
+; FLATSCR-NEXT:    s_add_u32 s1, vcc_hi, s0
+; FLATSCR-NEXT:    s_add_i32 s0, s0, 1
+; FLATSCR-NEXT:    s_cmpk_lt_u32 s0, 0x2120
+; FLATSCR-NEXT:    scratch_store_byte off, v2, s1
 ; FLATSCR-NEXT:    s_cbranch_scc1 BB1_1
 ; FLATSCR-NEXT:  ; %bb.2: ; %split
-; FLATSCR-NEXT:    s_movk_i32 s4, 0x20d0
-; FLATSCR-NEXT:    s_add_u32 s5, s33, 0x1000
-; FLATSCR-NEXT:    s_add_u32 s4, s5, s4
-; FLATSCR-NEXT:    scratch_load_dword v3, off, s4 offset:4
-; FLATSCR-NEXT:    s_movk_i32 s4, 0x2000
-; FLATSCR-NEXT:    s_add_u32 s5, s33, 0x1000
-; FLATSCR-NEXT:    s_add_u32 s4, s5, s4
-; FLATSCR-NEXT:    scratch_load_dword v2, off, s4 offset:208
-; FLATSCR-NEXT:    s_add_u32 s4, s33, 0x1000
-; FLATSCR-NEXT:    scratch_load_dword v4, off, s4 offset:68
-; FLATSCR-NEXT:    s_add_u32 s4, s33, 0x1000
-; FLATSCR-NEXT:    scratch_load_dword v5, off, s4 offset:64
+; FLATSCR-NEXT:    s_movk_i32 s0, 0x20d0
+; FLATSCR-NEXT:    s_add_u32 s1, s33, 0x1000
+; FLATSCR-NEXT:    s_add_u32 s0, s1, s0
+; FLATSCR-NEXT:    scratch_load_dword v3, off, s0 offset:4
+; FLATSCR-NEXT:    s_movk_i32 s0, 0x2000
+; FLATSCR-NEXT:    s_add_u32 s1, s33, 0x1000
+; FLATSCR-NEXT:    s_add_u32 s0, s1, s0
+; FLATSCR-NEXT:    scratch_load_dword v2, off, s0 offset:208
+; FLATSCR-NEXT:    s_add_u32 s0, s33, 0x1000
+; FLATSCR-NEXT:    scratch_load_dword v4, off, s0 offset:68
+; FLATSCR-NEXT:    s_add_u32 s0, s33, 0x1000
+; FLATSCR-NEXT:    scratch_load_dword v5, off, s0 offset:64
 ; FLATSCR-NEXT:    s_sub_u32 s32, s32, 0x6000
-; FLATSCR-NEXT:    s_mov_b32 s33, s6
+; FLATSCR-NEXT:    s_mov_b32 s33, s2
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v5
 ; FLATSCR-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
 ; FLATSCR-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %pin.low = alloca i32, align 8192, addrspace(5)

llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll

 ; MUBUF-NEXT:    global_store_dword v[0:1], v2, off
 ; MUBUF-NEXT:  BB0_3: ; %bb.2
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
 ; MUBUF-NEXT:    global_store_dword v[0:1], v0, off
 ; MUBUF-NEXT:    s_endpgm
 ;
 ; FLATSCR-LABEL: kernel_non_entry_block_static_alloca_uniformly_reached_align4:
 ; FLATSCR:       ; %bb.0: ; %entry
-; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s6, s9
-; FLATSCR-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x8
-; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
+; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
+; FLATSCR-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x8
+; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, 16
 ; FLATSCR-NEXT:    s_mov_b32 s33, 0
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT:    s_cmp_lg_u32 s8, 0
+; FLATSCR-NEXT:    s_cmp_lg_u32 s4, 0
 ; FLATSCR-NEXT:    s_cbranch_scc1 BB0_3
 ; FLATSCR-NEXT:  ; %bb.1: ; %bb.0
-; FLATSCR-NEXT:    s_cmp_lg_u32 s9, 0
+; FLATSCR-NEXT:    s_cmp_lg_u32 s5, 0
 ; FLATSCR-NEXT:    s_cbranch_scc1 BB0_3
 ; FLATSCR-NEXT:  ; %bb.2: ; %bb.1
-; FLATSCR-NEXT:    s_mov_b32 s6, s32
-; FLATSCR-NEXT:    s_movk_i32 s7, 0x1000
-; FLATSCR-NEXT:    s_add_i32 s8, s6, s7
-; FLATSCR-NEXT:    s_add_u32 s6, s6, s7
+; FLATSCR-NEXT:    s_mov_b32 s2, s32
+; FLATSCR-NEXT:    s_movk_i32 s3, 0x1000
+; FLATSCR-NEXT:    s_add_i32 s4, s2, s3
+; FLATSCR-NEXT:    s_add_u32 s2, s2, s3
 ; FLATSCR-NEXT:    v_mov_b32_e32 v1, 0
-; FLATSCR-NEXT:    scratch_store_dword off, v1, s6
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s2
 ; FLATSCR-NEXT:    v_mov_b32_e32 v1, 1
-; FLATSCR-NEXT:    s_lshl_b32 s6, s10, 2
-; FLATSCR-NEXT:    s_mov_b32 s32, s8
-; FLATSCR-NEXT:    scratch_store_dword off, v1, s8 offset:4
-; FLATSCR-NEXT:    s_add_i32 s8, s8, s6
-; FLATSCR-NEXT:    scratch_load_dword v1, off, s8
-; FLATSCR-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; FLATSCR-NEXT:    s_lshl_b32 s2, s6, 2
+; FLATSCR-NEXT:    s_mov_b32 s32, s4
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s4 offset:4
+; FLATSCR-NEXT:    s_add_i32 s4, s4, s2
+; FLATSCR-NEXT:    scratch_load_dword v1, off, s4
+; FLATSCR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_u32_e32 v2, v1, v0
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT:    v_mov_b32_e32 v0, s4
-; FLATSCR-NEXT:    v_mov_b32_e32 v1, s5
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, s0
+; FLATSCR-NEXT:    v_mov_b32_e32 v1, s1
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v2, off
 ; FLATSCR-NEXT:  BB0_3: ; %bb.2
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v0, off
 ; FLATSCR-NEXT:    s_endpgm
 
 entry:
   %cond0 = icmp eq i32 %arg.cond0, 0

 ; MUBUF-NEXT:    global_store_dword v[0:1], v2, off
 ; MUBUF-NEXT:  BB1_2: ; %bb.1
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
 ; MUBUF-NEXT:    global_store_dword v[0:1], v0, off
 ; MUBUF-NEXT:    s_endpgm
 ;
 ; FLATSCR-LABEL: kernel_non_entry_block_static_alloca_uniformly_reached_align64:
 ; FLATSCR:       ; %bb.0: ; %entry
-; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s6, s9
-; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
-; FLATSCR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x8
+; FLATSCR-NEXT:    s_add_u32 flat_scratch_lo, s2, s5
+; FLATSCR-NEXT:    s_addc_u32 flat_scratch_hi, s3, 0
+; FLATSCR-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
 ; FLATSCR-NEXT:    s_mov_b32 s32, 64
 ; FLATSCR-NEXT:    s_mov_b32 s33, 0
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT:    s_cmp_lg_u32 s6, 0
+; FLATSCR-NEXT:    s_cmp_lg_u32 s2, 0
 ; FLATSCR-NEXT:    s_cbranch_scc1 BB1_2
 ; FLATSCR-NEXT:  ; %bb.1: ; %bb.0
-; FLATSCR-NEXT:    s_add_i32 s6, s32, 0x1000
-; FLATSCR-NEXT:    s_and_b32 s6, s6, 0xfffff000
+; FLATSCR-NEXT:    s_add_i32 s2, s32, 0x1000
+; FLATSCR-NEXT:    s_and_b32 s2, s2, 0xfffff000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v1, 0
-; FLATSCR-NEXT:    scratch_store_dword off, v1, s6
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s2
 ; FLATSCR-NEXT:    v_mov_b32_e32 v1, 1
-; FLATSCR-NEXT:    s_lshl_b32 s7, s7, 2
-; FLATSCR-NEXT:    s_mov_b32 s32, s6
-; FLATSCR-NEXT:    scratch_store_dword off, v1, s6 offset:4
-; FLATSCR-NEXT:    s_add_i32 s6, s6, s7
-; FLATSCR-NEXT:    scratch_load_dword v1, off, s6
-; FLATSCR-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; FLATSCR-NEXT:    s_lshl_b32 s3, s3, 2
+; FLATSCR-NEXT:    s_mov_b32 s32, s2
+; FLATSCR-NEXT:    scratch_store_dword off, v1, s2 offset:4
+; FLATSCR-NEXT:    s_add_i32 s2, s2, s3
+; FLATSCR-NEXT:    scratch_load_dword v1, off, s2
+; FLATSCR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_u32_e32 v2, v1, v0
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT:    v_mov_b32_e32 v0, s4
-; FLATSCR-NEXT:    v_mov_b32_e32 v1, s5
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, s0
+; FLATSCR-NEXT:    v_mov_b32_e32 v1, s1
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v2, off
 ; FLATSCR-NEXT:  BB1_2: ; %bb.1
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v0, off
 ; FLATSCR-NEXT:    s_endpgm
 entry:
   %cond = icmp eq i32 %arg.cond, 0
   br i1 %cond, label %bb.0, label %bb.1

 ; MUBUF-NEXT:    s_sub_u32 s32, s32, 0x400
 ; MUBUF-NEXT:    s_mov_b32 s33, s7
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: func_non_entry_block_static_alloca_align4:
 ; FLATSCR:       ; %bb.0: ; %entry
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_mov_b32 s9, s33
+; FLATSCR-NEXT:    s_mov_b32 s5, s33
 ; FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; FLATSCR-NEXT:    s_mov_b32 s33, s32
 ; FLATSCR-NEXT:    s_add_u32 s32, s32, 16
-; FLATSCR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; FLATSCR-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; FLATSCR-NEXT:    s_cbranch_execz BB2_3
 ; FLATSCR-NEXT:  ; %bb.1: ; %bb.0
 ; FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; FLATSCR-NEXT:    s_and_b64 exec, exec, vcc
 ; FLATSCR-NEXT:    s_cbranch_execz BB2_3
 ; FLATSCR-NEXT:  ; %bb.2: ; %bb.1
-; FLATSCR-NEXT:    s_mov_b32 s6, s32
-; FLATSCR-NEXT:    s_movk_i32 s7, 0x1000
-; FLATSCR-NEXT:    s_add_i32 s8, s6, s7
-; FLATSCR-NEXT:    s_add_u32 s6, s6, s7
+; FLATSCR-NEXT:    s_mov_b32 s2, s32
+; FLATSCR-NEXT:    s_movk_i32 s3, 0x1000
+; FLATSCR-NEXT:    s_add_i32 s4, s2, s3
+; FLATSCR-NEXT:    s_add_u32 s2, s2, s3
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 0
-; FLATSCR-NEXT:    scratch_store_dword off, v2, s6
+; FLATSCR-NEXT:    scratch_store_dword off, v2, s2
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 1
-; FLATSCR-NEXT:    scratch_store_dword off, v2, s8 offset:4
-; FLATSCR-NEXT:    v_lshl_add_u32 v2, v4, 2, s8
+; FLATSCR-NEXT:    scratch_store_dword off, v2, s4 offset:4
+; FLATSCR-NEXT:    v_lshl_add_u32 v2, v4, 2, s4
 ; FLATSCR-NEXT:    scratch_load_dword v2, v2, off
 ; FLATSCR-NEXT:    v_and_b32_e32 v3, 0x3ff, v5
-; FLATSCR-NEXT:    s_mov_b32 s32, s8
+; FLATSCR-NEXT:    s_mov_b32 s32, s4
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_u32_e32 v2, v2, v3
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v2, off
 ; FLATSCR-NEXT:  BB2_3: ; %bb.2
-; FLATSCR-NEXT:    s_or_b64 exec, exec, s[4:5]
+; FLATSCR-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v0, off
 ; FLATSCR-NEXT:    s_sub_u32 s32, s32, 16
-; FLATSCR-NEXT:    s_mov_b32 s33, s9
+; FLATSCR-NEXT:    s_mov_b32 s33, s5
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 
 entry:
   %cond0 = icmp eq i32 %arg.cond0, 0
   br i1 %cond0, label %bb.0, label %bb.2
 
 bb.0:

 ; MUBUF-NEXT:    s_sub_u32 s32, s32, 0x2000
 ; MUBUF-NEXT:    s_mov_b32 s33, s7
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: func_non_entry_block_static_alloca_align64:
 ; FLATSCR:       ; %bb.0: ; %entry
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_add_u32 s4, s32, 63
-; FLATSCR-NEXT:    s_mov_b32 s7, s33
-; FLATSCR-NEXT:    s_and_b32 s33, s4, 0xffffffc0
+; FLATSCR-NEXT:    s_add_u32 s0, s32, 63
+; FLATSCR-NEXT:    s_mov_b32 s3, s33
+; FLATSCR-NEXT:    s_and_b32 s33, s0, 0xffffffc0
 ; FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; FLATSCR-NEXT:    s_add_u32 s32, s32, 0x80
-; FLATSCR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; FLATSCR-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; FLATSCR-NEXT:    s_cbranch_execz BB3_2
 ; FLATSCR-NEXT:  ; %bb.1: ; %bb.0
-; FLATSCR-NEXT:    s_add_i32 s6, s32, 0x1000
-; FLATSCR-NEXT:    s_and_b32 s6, s6, 0xfffff000
+; FLATSCR-NEXT:    s_add_i32 s2, s32, 0x1000
+; FLATSCR-NEXT:    s_and_b32 s2, s2, 0xfffff000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 0
-; FLATSCR-NEXT:    scratch_store_dword off, v2, s6
+; FLATSCR-NEXT:    scratch_store_dword off, v2, s2
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 1
-; FLATSCR-NEXT:    scratch_store_dword off, v2, s6 offset:4
-; FLATSCR-NEXT:    v_lshl_add_u32 v2, v3, 2, s6
+; FLATSCR-NEXT:    scratch_store_dword off, v2, s2 offset:4
+; FLATSCR-NEXT:    v_lshl_add_u32 v2, v3, 2, s2
 ; FLATSCR-NEXT:    scratch_load_dword v2, v2, off
 ; FLATSCR-NEXT:    v_and_b32_e32 v3, 0x3ff, v4
-; FLATSCR-NEXT:    s_mov_b32 s32, s6
+; FLATSCR-NEXT:    s_mov_b32 s32, s2
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_u32_e32 v2, v2, v3
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v2, off
 ; FLATSCR-NEXT:  BB3_2: ; %bb.1
-; FLATSCR-NEXT:    s_or_b64 exec, exec, s[4:5]
+; FLATSCR-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v0, off
 ; FLATSCR-NEXT:    s_sub_u32 s32, s32, 0x80
-; FLATSCR-NEXT:    s_mov_b32 s33, s7
+; FLATSCR-NEXT:    s_mov_b32 s33, s3
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %cond = icmp eq i32 %arg.cond, 0
   br i1 %cond, label %bb.0, label %bb.1
 
 bb.0:
   %alloca = alloca [16 x i32], align 64, addrspace(5)

llvm/test/CodeGen/AMDGPU/spill-offset-calculation.ll

-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -post-RA-scheduler=0 -stress-regalloc=8 < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -post-RA-scheduler=0 -stress-regalloc=8 < %s | FileCheck -check-prefixes=GCN,MUBUF %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -post-RA-scheduler=0 -stress-regalloc=8 -amdgpu-enable-flat-scratch < %s | FileCheck -check-prefixes=GCN,FLATSCR %s
 
 ; Test that the VGPR spiller correctly switches to SGPR offsets when the
 ; instruction offset field would overflow, and that it accounts for memory
 ; swizzling.
 
-; CHECK-LABEL: test_inst_offset_kernel
+; GCN-LABEL: test_inst_offset_kernel
 define amdgpu_kernel void @test_inst_offset_kernel() {
 entry:
   ; Occupy 4092 bytes of scratch, so the offset of the spill of %a just fits in
   ; the instruction offset field.
   %alloca = alloca i8, i32 4088, align 4, addrspace(5)
   %buf = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
 
   %aptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 offset:4092 ; 4-byte Folded Spill
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 offset:4092 ; 4-byte Folded Spill
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s{{[0-9]+}} ; 4-byte Folded Spill
   %a = load volatile i32, i32 addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
   store volatile i32 %a, i32 addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: test_sgpr_offset_kernel
+; GCN-LABEL: test_sgpr_offset_kernel
 define amdgpu_kernel void @test_sgpr_offset_kernel() {
 entry:
   ; Occupy 4096 bytes of scratch, so the offset of the spill of %a does not
   ; fit in the instruction, and has to live in the SGPR offset.
   %alloca = alloca i8, i32 4092, align 4, addrspace(5)
   %buf = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
 
   %aptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
   ; 0x40000 / 64 = 4096 (for wave64)
-  ; CHECK: s_mov_b32 s6, 0x40000
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s6 ; 4-byte Folded Spill
+  ; MUBUF:   s_mov_b32 s6, 0x40000
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s6 ; 4-byte Folded Spill
+  ; FLATSCR: s_movk_i32 s2, 0x1000
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s2 ; 4-byte Folded Spill
   %a = load volatile i32, i32 addrspace(5)* %aptr
 
   ; Force %a to spill
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
   store volatile i32 %a, i32 addrspace(5)* %outptr
 
   ret void
 }
 
 ; FIXME: If we fail to scavenge an SGPR in a kernel we don't have a stack
 ; pointer to temporarily update, so we just crash.
 
-; CHECK-LABEL: test_sgpr_offset_function_scavenge_fail
+; GCN-LABEL: test_sgpr_offset_function_scavenge_fail
 define void @test_sgpr_offset_function_scavenge_fail() #2 {
 entry:
   ; Occupy 4096 bytes of scratch, so the offset of the spill of %a does not
   ; fit in the instruction, and has to live in the SGPR offset.
   %alloca = alloca i8, i32 4096, align 4, addrspace(5)
   %buf = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
 
   %aptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
 
   %asm.0 = call { i32, i32, i32, i32, i32, i32, i32, i32 } asm sideeffect "", "=s,=s,=s,=s,=s,=s,=s,=s"()
   %asm0.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 0
   %asm1.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 1
   %asm2.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 2
   %asm3.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 3
   %asm4.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 4
   %asm5.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 5
   %asm6.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 6
   %asm7.0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm.0, 7
 
   ; 0x40000 / 64 = 4096 (for wave64)
   %a = load volatile i32, i32 addrspace(5)* %aptr
 
-  ; CHECK: s_add_u32 s32, s32, 0x40000
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s32 ; 4-byte Folded Spill
-  ; CHECK: s_sub_u32 s32, s32, 0x40000
+  ; MUBUF:   s_add_u32 s32, s32, 0x40000
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s32 ; 4-byte Folded Spill
+  ; MUBUF:   s_sub_u32 s32, s32, 0x40000
+  ; FLATSCR: s_add_u32 [[SOFF:s[0-9+]]], s32, 0x1000
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF]] ; 4-byte Folded Spill
   call void asm sideeffect "", "s,s,s,s,s,s,s,s,v"(i32 %asm0.0, i32 %asm1.0, i32 %asm2.0, i32 %asm3.0, i32 %asm4.0, i32 %asm5.0, i32 %asm6.0, i32 %asm7.0, i32 %a)
 
   %asm = call { i32, i32, i32, i32, i32, i32, i32, i32 } asm sideeffect "", "=s,=s,=s,=s,=s,=s,=s,=s"()
   %asm0 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 0
   %asm1 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 1
   %asm2 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 2
   %asm3 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 3
   %asm4 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 4
   %asm5 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 5
   %asm6 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 6
   %asm7 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32 } %asm, 7
 
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"() #0
 
-  ; CHECK: s_add_u32 s32, s32, 0x40000
-  ; CHECK: buffer_load_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s32 ; 4-byte Folded Reload
-  ; CHECK: s_sub_u32 s32, s32, 0x40000
+  ; MUBUF:   s_add_u32 s32, s32, 0x40000
+  ; MUBUF:   buffer_load_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s32 ; 4-byte Folded Reload
+  ; MUBUF:   s_sub_u32 s32, s32, 0x40000
+  ; FLATSCR: s_add_u32 [[SOFF:s[0-9+]]], s32, 0x1000
+  ; FLATSCR: scratch_load_dword v{{[0-9]+}}, off, [[SOFF]] ; 4-byte Folded Reload
 
    ; Force %a to spill with no free SGPRs
   call void asm sideeffect "", "s,s,s,s,s,s,s,s,v"(i32 %asm0, i32 %asm1, i32 %asm2, i32 %asm3, i32 %asm4, i32 %asm5, i32 %asm6, i32 %asm7, i32 %a)
   ret void
 }
 
-; CHECK-LABEL: test_sgpr_offset_subregs_kernel
+; GCN-LABEL: test_sgpr_offset_subregs_kernel
 define amdgpu_kernel void @test_sgpr_offset_subregs_kernel() {
 entry:
   ; Occupy 4088 bytes of scratch, so that the spill of the last subreg of %a
   ; still fits below offset 4096 (4088 + 8 - 4 = 4092), and can be placed in
   ; the instruction offset field.
   %alloca = alloca i8, i32 4084, align 4, addrspace(5)
   %bufv1 = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
   %bufv2 = bitcast i8 addrspace(5)* %alloca to <2 x i32> addrspace(5)*
 
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 offset:4088 ; 4-byte Folded Spill
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 offset:4092 ; 4-byte Folded Spill
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 offset:4088 ; 4-byte Folded Spill
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 offset:4092 ; 4-byte Folded Spill
+  ; FLATSCR: s_movk_i32 [[SOFF:s[0-9]+]], 0xff8
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF]]          ; 4-byte Folded Spill
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF]] offset:4 ; 4-byte Folded Spill
   %aptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %bufv2, i32 1
   %a = load volatile <2 x i32>, <2 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   ; Ensure the alloca sticks around.
   %bptr = getelementptr i32, i32 addrspace(5)* %bufv1, i32 1
   %b = load volatile i32, i32 addrspace(5)* %bptr
 
   ; Ensure the spill is of the full super-reg.
   call void asm sideeffect "; $0", "r"(<2 x i32> %a)
 
   ret void
 }
 
-; CHECK-LABEL: test_inst_offset_subregs_kernel
+; GCN-LABEL: test_inst_offset_subregs_kernel
 define amdgpu_kernel void @test_inst_offset_subregs_kernel() {
 entry:
   ; Occupy 4092 bytes of scratch, so that the spill of the last subreg of %a
   ; does not fit below offset 4096 (4092 + 8 - 4 = 4096), and has to live
   ; in the SGPR offset.
   %alloca = alloca i8, i32 4088, align 4, addrspace(5)
   %bufv1 = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
   %bufv2 = bitcast i8 addrspace(5)* %alloca to <2 x i32> addrspace(5)*
 
   ; 0x3ff00 / 64 = 4092 (for wave64)
-  ; CHECK: s_mov_b32 s6, 0x3ff00
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s6 ; 4-byte Folded Spill
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s6 offset:4 ; 4-byte Folded Spill
+  ; MUBUF:   s_mov_b32 s6, 0x3ff00
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s6 ; 4-byte Folded Spill
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s6 offset:4 ; 4-byte Folded Spill
+  ; FLATSCR: s_movk_i32 [[SOFF:s[0-9]+]], 0xffc
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF]]          ; 4-byte Folded Spill
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF]] offset:4 ; 4-byte Folded Spill
   %aptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %bufv2, i32 1
   %a = load volatile <2 x i32>, <2 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   ; Ensure the alloca sticks around.
   %bptr = getelementptr i32, i32 addrspace(5)* %bufv1, i32 1
   %b = load volatile i32, i32 addrspace(5)* %bptr
 
   ; Ensure the spill is of the full super-reg.
   call void asm sideeffect "; $0", "r"(<2 x i32> %a)
 
   ret void
 }
 
-; CHECK-LABEL: test_inst_offset_function
+; GCN-LABEL: test_inst_offset_function
 define void @test_inst_offset_function() {
 entry:
   ; Occupy 4092 bytes of scratch, so the offset of the spill of %a just fits in
   ; the instruction offset field.
   %alloca = alloca i8, i32 4092, align 4, addrspace(5)
   %buf = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
 
   %aptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4092 ; 4-byte Folded Spill
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4092 ; 4-byte Folded Spill
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s{{[0-9]+}} offset:4092 ; 4-byte Folded Spill
   %a = load volatile i32, i32 addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
   store volatile i32 %a, i32 addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: test_sgpr_offset_function
+; GCN-LABEL: test_sgpr_offset_function
 define void @test_sgpr_offset_function() {
 entry:
   ; Occupy 4096 bytes of scratch, so the offset of the spill of %a does not
   ; fit in the instruction, and has to live in the SGPR offset.
   %alloca = alloca i8, i32 4096, align 4, addrspace(5)
   %buf = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
 
   %aptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
   ; 0x40000 / 64 = 4096 (for wave64)
-  ; CHECK: s_add_u32 s4, s32, 0x40000
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s4 ; 4-byte Folded Spill
+  ; MUBUF:   s_add_u32 s4, s32, 0x40000
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s4 ; 4-byte Folded Spill
+  ; FLATSCR: s_add_u32 s0, s32, 0x1000
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s0 ; 4-byte Folded Spill
   %a = load volatile i32, i32 addrspace(5)* %aptr
 
   ; Force %a to spill
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr i32, i32 addrspace(5)* %buf, i32 1
   store volatile i32 %a, i32 addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: test_sgpr_offset_subregs_function
+; GCN-LABEL: test_sgpr_offset_subregs_function
 define void @test_sgpr_offset_subregs_function() {
 entry:
   ; Occupy 4088 bytes of scratch, so that the spill of the last subreg of %a
   ; still fits below offset 4096 (4088 + 8 - 4 = 4092), and can be placed in
   ; the instruction offset field.
   %alloca = alloca i8, i32 4088, align 4, addrspace(5)
   %bufv1 = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
   %bufv2 = bitcast i8 addrspace(5)* %alloca to <2 x i32> addrspace(5)*
 
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4088 ; 4-byte Folded Spill
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4092 ; 4-byte Folded Spill
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4088 ; 4-byte Folded Spill
+  ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4092 ; 4-byte Folded Spill
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s32 offset:4088 ; 4-byte Folded Spill
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s32 offset:4092 ; 4-byte Folded Spill
   %aptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %bufv2, i32 1
   %a = load volatile <2 x i32>, <2 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   ; Ensure the alloca sticks around.
   %bptr = getelementptr i32, i32 addrspace(5)* %bufv1, i32 1
   %b = load volatile i32, i32 addrspace(5)* %bptr
 
   ; Ensure the spill is of the full super-reg.
   call void asm sideeffect "; $0", "r"(<2 x i32> %a)
 
   ret void
 }
 
-; CHECK-LABEL: test_inst_offset_subregs_function
+; GCN-LABEL: test_inst_offset_subregs_function
 define void @test_inst_offset_subregs_function() {
 entry:
   ; Occupy 4092 bytes of scratch, so that the spill of the last subreg of %a
   ; does not fit below offset 4096 (4092 + 8 - 4 = 4096), and has to live
   ; in the SGPR offset.
   %alloca = alloca i8, i32 4092, align 4, addrspace(5)
   %bufv1 = bitcast i8 addrspace(5)* %alloca to i32 addrspace(5)*
   %bufv2 = bitcast i8 addrspace(5)* %alloca to <2 x i32> addrspace(5)*
 
   ; 0x3ff00 / 64 = 4092 (for wave64)
-  ; CHECK: s_add_u32 s4, s32, 0x3ff00
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s4 ; 4-byte Folded Spill
-  ; CHECK: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s4 offset:4 ; 4-byte Folded Spill
+  ; MUBUF: s_add_u32 s4, s32, 0x3ff00
+  ; MUBUF: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s4 ; 4-byte Folded Spill
+  ; MUBUF: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s4 offset:4 ; 4-byte Folded Spill
+  ; FLATSCR: s_add_u32 [[SOFF:s[0-9]+]], s32, 0xffc
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF]]          ; 4-byte Folded Spill
+  ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF]] offset:4 ; 4-byte Folded Spill
   %aptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %bufv2, i32 1
   %a = load volatile <2 x i32>, <2 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   ; Ensure the alloca sticks around.
   %bptr = getelementptr i32, i32 addrspace(5)* %bufv1, i32 1

llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll

 ;
 ; FLATSCR-LABEL: kernel_background_evaluate:
 ; FLATSCR:       ; %bb.0: ; %entry
 ; FLATSCR-NEXT:    s_add_u32 s2, s2, s5
 ; FLATSCR-NEXT:    s_movk_i32 s32, 0x6000
 ; FLATSCR-NEXT:    s_addc_u32 s3, s3, 0
 ; FLATSCR-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
 ; FLATSCR-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
-; FLATSCR-NEXT:    s_load_dword s0, s[0:1], 0x24
-; FLATSCR-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
-; FLATSCR-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
-; FLATSCR-NEXT:    s_mov_b32 s38, -1
-; FLATSCR-NEXT:    s_mov_b32 s39, 0x31c16000
-; FLATSCR-NEXT:    s_add_u32 s36, s36, s5
-; FLATSCR-NEXT:    s_addc_u32 s37, s37, 0
+; FLATSCR-NEXT:    s_load_dword s2, s[0:1], 0x24
 ; FLATSCR-NEXT:    v_mov_b32_e32 v1, 0x2000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 0x4000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v3, 0
 ; FLATSCR-NEXT:    v_mov_b32_e32 v4, 0x400000
 ; FLATSCR-NEXT:    ; implicit-def: $vcc_hi
-; FLATSCR-NEXT:    s_getpc_b64 s[4:5]
-; FLATSCR-NEXT:    s_add_u32 s4, s4, svm_eval_nodes@rel32@lo+4
-; FLATSCR-NEXT:    s_addc_u32 s5, s5, svm_eval_nodes@rel32@hi+12
+; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
+; FLATSCR-NEXT:    s_add_u32 s0, s0, svm_eval_nodes@rel32@lo+4
+; FLATSCR-NEXT:    s_addc_u32 s1, s1, svm_eval_nodes@rel32@hi+12
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT:    v_mov_b32_e32 v0, s0
-; FLATSCR-NEXT:    s_mov_b64 s[0:1], s[36:37]
-; FLATSCR-NEXT:    s_mov_b64 s[2:3], s[38:39]
-; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; FLATSCR-NEXT:    v_mov_b32_e32 v0, s2
+; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; FLATSCR-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; FLATSCR-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; FLATSCR-NEXT:    s_cbranch_execz BB0_2
 ; FLATSCR-NEXT:  ; %bb.1: ; %if.then4.i
 ; FLATSCR-NEXT:    s_movk_i32 vcc_lo, 0x4000
 ; FLATSCR-NEXT:    s_nop 1
 ; FLATSCR-NEXT:    scratch_load_dword v0, off, vcc_lo offset:4
 ; FLATSCR-NEXT:    s_waitcnt_depctr 0xffe3