[AMDGPU] Fix function calls with flat scratch

[sebastian-ne]

Summary

When flat scratch is used, the stack pointer needs to be added when
writing arguments to the stack.
For buffer instructions, this is done in SelectMUBUFScratchOffen
and SelectMUBUFScratchOffset.

[foad]

Does globalisel handle this?

[rampitec]

In D103166#2782087, @foad wrote:

Does globalisel handle this?

Similar code is needed in the AMDGPUInstructionSelector::selectScratchSAddr().

[sebastian-ne]

In D103166#2782087, @foad wrote:

Does globalisel handle this?

Yes, it seems like globalisel handles adding the stack pointer in the AMDGPUOutgoingArgHandler. The code that adds the stack pointer in AMDGPUInstructionSelector::selectMUBUFScratchOffset is never hit, I’ll remove it.

I changed the handling in SDAG to match the one in GlobalISel. The stack pointer is now added when lowering call arguments and not when selecting instructions.

[arsenm]

LGTM. This is nicer than inspecting the memory operand

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

if (C->getSExtValue()) {
 
     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
     return true;
   }
 
   return false;
 }
 
-static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
-  auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
-  return PSV && PSV->isStack();
-}
-
 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
   SDLoc DL(N);
 
   auto *FI = dyn_cast<FrameIndexSDNode>(N);
   SDValue TFI =
       FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
 
   // We rebase the base address into an absolute stack address and hence
const int64_t NullPtr =
         AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
     // Don't fold null pointer.
     if (Imm != NullPtr) {
       SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
       MachineSDNode *MovHighBits = CurDAG->getMachineNode(
         AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
       VAddr = SDValue(MovHighBits, 0);
 
-      // In a call sequence, stores to the argument stack area are relative to the
-      // stack pointer.
-      const MachinePointerInfo &PtrInfo
-        = cast<MemSDNode>(Parent)->getPointerInfo();
-      SOffset = isStackPtrRelative(PtrInfo)
-        ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
-        : CurDAG->getTargetConstant(0, DL, MVT::i32);
+      SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
       ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
       return true;
     }
   }
 
   if (CurDAG->isBaseWithConstantOffset(Addr)) {
     // (add n0, c1)
 
bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
   }
 
   // (node)
   std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
   return true;
 }
 
+static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
+  if (Val.getOpcode() != ISD::CopyFromReg)
+    return false;
+  auto RC =
+      TRI.getPhysRegClass(cast<RegisterSDNode>(Val.getOperand(1))->getReg());
+  return RC && TRI.isSGPRClass(RC);
+}
+
 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
                                                   SDValue Addr,
                                                   SDValue &SRsrc,
                                                   SDValue &SOffset,
                                                   SDValue &Offset) const {
-  ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr);
-  if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
-    return false;
-
-  SDLoc DL(Addr);
+  const SIRegisterInfo *TRI =
+      static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
   MachineFunction &MF = CurDAG->getMachineFunction();
   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
+  SDLoc DL(Addr);
 
+  // CopyFromReg <sgpr>
+  if (IsCopyFromSGPR(*TRI, Addr)) {
     SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
+    SOffset = Addr;
+    Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
+    return true;
+  }
 
-  const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
+  ConstantSDNode *CAddr;
+  if (Addr.getOpcode() == ISD::ADD) {
+    // Add (CopyFromReg <sgpr>) <constant>
+    CAddr = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
+    if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
+      return false;
+    if (!IsCopyFromSGPR(*TRI, Addr.getOperand(0)))
+      return false;
+
+    SOffset = Addr.getOperand(0);
+  } else if ((CAddr = dyn_cast<ConstantSDNode>(Addr)) &&
+             SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) {
+    // <constant>
+    SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
+  } else {
+    return false;
+  }
 
-  // FIXME: Get from MachinePointerInfo? We should only be using the frame
-  // offset if we know this is in a call sequence.
-  SOffset = isStackPtrRelative(PtrInfo)
-                ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)
-                : CurDAG->getTargetConstant(0, DL, MVT::i32);
+  SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
 
   Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
   return true;
 }
 
 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
                                            SDValue &SOffset, SDValue &Offset
                                            ) const {
SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, SDLoc(SAddr),
                                            MVT::i32, TFI, SAddr.getOperand(1)),
                     0);
   }
 
   return SAddr;
 }
 
 // Match (32-bit SGPR base) + sext(imm offset)
-bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *N,
-                                            SDValue Addr,
+bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr,
                                             SDValue &SAddr,
                                             SDValue &Offset) const {
   if (Addr->isDivergent())
     return false;
 
-  SAddr = Addr;
+  SDLoc DL(Addr);
+
   int64_t COffsetVal = 0;
 
   if (CurDAG->isBaseWithConstantOffset(Addr)) {
     COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
     SAddr = Addr.getOperand(0);
+  } else {
+    SAddr = Addr;
   }
 
   SAddr = SelectSAddrFI(CurDAG, SAddr);
 
   const SIInstrInfo *TII = Subtarget->getInstrInfo();
 
   if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
                               SIInstrFlags::FlatScratch)) {
     int64_t SplitImmOffset, RemainderOffset;
     std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
         COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, SIInstrFlags::FlatScratch);
 
     COffsetVal = SplitImmOffset;
 
-    SDLoc DL(N);
     SDValue AddOffset =
-        getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
+        SAddr.getOpcode() == ISD::TargetFrameIndex
+            ? getMaterializedScalarImm32(Lo_32(RemainderOffset), DL)
+            : CurDAG->getTargetConstant(RemainderOffset, DL, MVT::i32);
     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_U32, DL, MVT::i32,
                                            SAddr, AddOffset), 0);
   }
 
-  Offset = CurDAG->getTargetConstant(COffsetVal, SDLoc(), MVT::i16);
+  Offset = CurDAG->getTargetConstant(COffsetVal, DL, MVT::i16);
 
   return true;
 }
 
 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
                                           SDValue &Offset, bool &Imm) const {
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
   if (!C) {

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

 
 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
                                                    const SDLoc &SL,
                                                    SDValue Chain,
                                                    SDValue ArgVal,
                                                    int64_t Offset) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
+  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
   SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
+  // Stores to the argument stack area are relative to the stack pointer.
+  SDValue SP =
+      DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32);
+  Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr);
   SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, Align(4),
                                MachineMemOperand::MODereferenceable);
   return Store;
 }
 
 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
                                              const TargetRegisterClass *RC,
                                              EVT VT, const SDLoc &SL,

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

if (!isSGPR(SAddr))
     return None;
 
   return {{
       [=](MachineInstrBuilder &MIB) { MIB.addReg(SAddr); }, // saddr
       [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
   }};
 }
 
-static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
-  auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
-  return PSV && PSV->isStack();
-}
-
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
   MachineInstr *MI = Root.getParent();
   MachineBasicBlock *MBB = MI->getParent();
   MachineFunction *MF = MBB->getParent();
   const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
 
   int64_t Offset = 0;
AMDGPUInstructionSelector::selectMUBUFScratchOffset(
 
   int64_t Offset = 0;
   if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) ||
       !SIInstrInfo::isLegalMUBUFImmOffset(Offset))
     return {};
 
   const MachineFunction *MF = MBB->getParent();
   const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
-  const MachineMemOperand *MMO = *MI->memoperands_begin();
-  const MachinePointerInfo &PtrInfo = MMO->getPointerInfo();
 
   return {{
       [=](MachineInstrBuilder &MIB) { // rsrc
         MIB.addReg(Info->getScratchRSrcReg());
       },
       [=](MachineInstrBuilder &MIB) { // soffset
-        if (isStackPtrRelative(PtrInfo))
-          MIB.addReg(Info->getStackPtrOffsetReg());
-        else
-          MIB.addImm(0);
       },
       [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } // offset
   }};
 }
 
 std::pair<Register, unsigned>
 AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const {
   const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

if (VA.isRegLoc()) {
         // are loaded before this eventual operation. Otherwise they'll be
         // clobbered.
 
         // FIXME: Why is this really necessary? This seems to just result in a
         // lot of code to copy the stack and write them back to the same
         // locations, which are supposed to be immutable?
         Chain = addTokenForArgument(Chain, DAG, MFI, FI);
       } else {
-        DstAddr = PtrOff;
+        // Stores to the argument stack area are relative to the stack pointer.
+        SDValue SP = DAG.getCopyFromReg(Chain, DL, Info->getStackPtrOffsetReg(),
+                                        MVT::i32);
+        DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff);
         DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
         Alignment =
             commonAlignment(Subtarget->getStackAlignment(), LocMemOffset);
       }
 
       if (Outs[i].Flags.isByVal()) {
         SDValue SizeNode =
             DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);

llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs-packed.ll

 }
 
 ; sp[0] = byval
 ; sp[1] = ??
 ; sp[2] = stack passed workitem ID x
 
 ; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_x_byval:
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
-; GCN-DAG: s_movk_i32 s32, 0x400
 
 ; GCN: buffer_store_dword [[K]], off, s[0:3], 0 offset:4
-; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], 0 offset:4
+; GCN: s_movk_i32 s32, 0x400
+; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:4
 
 ; GCN: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32{{$}}
 ; GCN: v_mov_b32_e32 [[RELOAD_BYVAL]],
 ; GCN: s_swappc_b64
 
 ; GCN: .amdhsa_system_vgpr_workitem_id 0
 define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_x_byval() #1 {
   %alloca = alloca i32, align 4, addrspace(5)

llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll

 ; var abi stack layout:
 ; sp[0] = byval
 ; sp[1] = ??
 ; sp[2] = stack passed workitem ID x
 
 ; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_x_byval:
 ; VARABI: enable_vgpr_workitem_id = 0
 ; VARABI: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
-; VARABI: s_movk_i32 s32, 0x400{{$}}
 ; VARABI: buffer_store_dword [[K]], off, s[0:3], 0 offset:4
-; VARABI: buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; VARABI: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], 0 offset:4
+; VARABI: s_movk_i32 s32, 0x400{{$}}
+; VARABI: buffer_store_dword v0, off, s[0:3], s32 offset:4
 
 ; VARABI: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32{{$}}
 ; VARABI: v_mov_b32_e32 [[RELOAD_BYVAL]],
 ; VARABI: s_swappc_b64
 
 
 ; FIXEDABI: v_mov_b32_e32 [[K0:v[0-9]+]], 0x3e7
 ; FIXEDABI: buffer_store_dword [[K0]], off, s[0:3], 0 offset:4{{$}}
call void @too_many_args_use_workitem_id_x_byval(
     i32 290, i32 300, i32 310, i32 320,
     i32 addrspace(5)* byval(i32) %alloca)
   ret void
 }
 
 ; GCN-LABEL: {{^}}func_call_too_many_args_use_workitem_id_x_byval:
 ; VARABI: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
 ; VARABI: buffer_store_dword [[K]], off, s[0:3], s33{{$}}
-; VARABI: buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; VARABI: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s33{{$}}
+; VARABI: buffer_store_dword v0, off, s[0:3], s32 offset:4
 ; VARABI: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32{{$}}
 ; VARABI: v_mov_b32_e32 [[RELOAD_BYVAL]],
 ; VARABI: s_swappc_b64
 
 
 ; FIXED-ABI-NOT: v31
 ; FIXEDABI: v_mov_b32_e32 [[K0:v[0-9]+]], 0x3e7{{$}}
 ; FIXEDABI: buffer_store_dword [[K0]], off, s[0:3], s33{{$}}

llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll

 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 0
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_i1@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_i1@rel32@hi+12
-; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s32
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s33, v40, 2
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s32 ; 4-byte Folded Reload

 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 0
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_i1_signext@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_i1_signext@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s2
+; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s32
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s33, v40, 2
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s32 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3

 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    global_load_ubyte v0, v[0:1], off glc dlc
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 0
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_i1_zeroext@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_i1_zeroext@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s2
+; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s32
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s33, v40, 2
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s32 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3

 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v32, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 0
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    global_load_dword v33, v[0:1], off
 ; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_clause 0x7
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[0:3], v32, s[0:1]
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
 ; GFX10-SCRATCH-NEXT:    global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_v32i32_i32@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_v32i32_i32@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(8)
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v33, s2
+; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v33, s32
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s33, v40, 2
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s32 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3

 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 0
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_i1_inreg@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_i1_inreg@rel32@hi+12
-; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    scratch_store_byte off, v0, s32
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s33, v40, 2
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s32 ; 4-byte Folded Reload

 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 18
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 20
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 16
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s20, 12
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s36, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s21, 8
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s22, 4
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s23, 0
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s37, 1
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s36, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s37, 1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s38, 2
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s39, 3
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s40, 4
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s41, 5
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s42, 6
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s43, 7
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s44, 8
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s45, 9
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s46, 10
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s47, 11
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s48, 12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s49, 13
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s50, 14
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s51, 15
 ; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_clause 0x1
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[36:51], s[0:1], 0x40
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[4:19], s[0:1], 0x0
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_v32i32_inreg@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_v32i32_inreg@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 17
 ; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s51
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s50
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, s49
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, s48
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, s47
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, s46
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s2
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s3
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s20
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v3, s21
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v4, s22
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v5, s23
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, s50
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, s51
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s46
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s47
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, s48
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, s49
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s20, s36
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s21, s37
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s22, s38
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s23, s39
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s24, s40
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s25, s41
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s26, s42
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s27, s43
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s28, s44
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s29, s45
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[4:5], s32 offset:16
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 17
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s51, v40, 15
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s50, v40, 14
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s49, v40, 13
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s48, v40, 12
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s47, v40, 11

 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 18
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 24
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s20, 20
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s21, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s36, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s22, 12
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s23, 4
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s24, 0
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s37, 1
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s36, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s37, 1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s38, 2
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s39, 3
-; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[4:19], s[0:1], 0x0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s40, 4
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s41, 5
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s42, 6
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s43, 7
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s44, 8
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s45, 9
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s46, 10
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s47, 11
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s48, 12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s49, 13
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s50, 14
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s51, 15
+; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_load_dword s2, s[0:1], 0x0
 ; GFX10-SCRATCH-NEXT:    ; kill: killed $sgpr0_sgpr1
 ; GFX10-SCRATCH-NEXT:    ; kill: killed $sgpr0_sgpr1
+; GFX10-SCRATCH-NEXT:    s_clause 0x1
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[36:51], s[0:1], 0x40
+; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[4:19], s[0:1], 0x0
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_v32i32_i32_inreg@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_v32i32_i32_inreg@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 17
 ; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 8
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s50
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, s49
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s3
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s51
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, s48
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, s47
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, s46
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s25, s41
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s20
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s21
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s22
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v3, s2
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v4, s23
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v5, s24
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, s50
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, s51
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s46
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s47
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, s48
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, s49
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s20, s36
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s21, s37
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s22, s38
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s23, s39
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s24, s40
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s25, s41
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s26, s42
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s27, s43
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s28, s44
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s29, s45
+; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v6, s32 offset:24
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[4:5], s32 offset:16
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 17
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s51, v40, 15
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s50, v40, 14
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s49, v40, 13
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s48, v40, 12
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s47, v40, 11

 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 offset:8 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 4
-; GFX10-SCRATCH-NEXT:    s_clause 0x1
-; GFX10-SCRATCH-NEXT:    scratch_load_dword v32, off, s33 offset:4
-; GFX10-SCRATCH-NEXT:    scratch_load_dword v33, off, s33
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 0
+; GFX10-SCRATCH-NEXT:    scratch_load_dwordx2 v[32:33], off, s33
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, stack_passed_f64_arg@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, stack_passed_f64_arg@rel32@hi+12
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v32, s2
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v33, s3
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[32:33], s32
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s33, v40, 2
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s32 offset:8 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3

 ; GFX10-SCRATCH-LABEL: stack_12xv3i32:
 ; GFX10-SCRATCH:       ; %bb.0: ; %entry
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 15
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 12
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 14
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 12
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 13
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 8
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s0
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 12
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 13
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 14
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, 15
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
+; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s1
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, 1
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, 1
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, 1
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, 1
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v6, 2
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v7, 2
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v8, 2
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v9, 3
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v10, 3
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v11, 3
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v12, 4

 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v24, 8
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v25, 8
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v26, 8
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v27, 9
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v28, 9
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v29, 9
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v30, 10
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v31, 11
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_12xv3i32@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_12xv3i32@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16

 ; GFX10-SCRATCH-LABEL: stack_8xv5i32:
 ; GFX10-SCRATCH:       ; %bb.0: ; %entry
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 15
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 14
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 28
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 24
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 11
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s1
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 13
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 12
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 20
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s1
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 10
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 12
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 9
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s0
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 8
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 8
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 4
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 12
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 13
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 14
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, 15
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, 8
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, 9
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v6, 10
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v7, 11
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
+; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s1
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[4:7], s32
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, 1
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v6, 1
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v7, 1

 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v24, 4
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v25, 5
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v26, 5
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v27, 5
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v28, 5
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v29, 5
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v30, 6
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v31, 7
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_8xv5i32@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_8xv5i32@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16

 ; GFX10-SCRATCH-LABEL: stack_8xv5f32:
 ; GFX10-SCRATCH:       ; %bb.0: ; %entry
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s0, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s32 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s0
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0x41700000
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0x41600000
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 28
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 24
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 0x41300000
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s1
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0x41500000
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0x41400000
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 20
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s1
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0x41200000
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 12
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0x41100000
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s0
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 8
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 0x41000000
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v0, s0
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 4
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s33, 2
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0x41400000
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0x41500000
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 0x41600000
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, 0x41700000
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, 0x41000000
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, 0x41100000
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v6, 0x41200000
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v7, 0x41300000
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
+; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v1, s0
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s1
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[0:3], s32 offset:16
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[4:7], s32
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, 1.0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v6, 1.0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v7, 1.0

 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v24, 4.0
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v25, 0x40a00000
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v26, 0x40a00000
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v27, 0x40a00000
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v28, 0x40a00000
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v29, 0x40a00000
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v30, 0x40c00000
 ; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v31, 0x40e00000
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_add_u32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-SCRATCH-NEXT:    s_add_u32 s0, s0, external_void_func_8xv5f32@rel32@lo+4
 ; GFX10-SCRATCH-NEXT:    s_addc_u32 s1, s1, external_void_func_8xv5f32@rel32@hi+12
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s1, v40, 1
 ; GFX10-SCRATCH-NEXT:    s_sub_u32 s32, s32, 16

llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll

 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
 ; FLATSCR-NEXT:    s_cmp_lg_u32 s4, 0
 ; FLATSCR-NEXT:    s_cbranch_scc1 BB0_3
 ; FLATSCR-NEXT:  ; %bb.1: ; %bb.0
 ; FLATSCR-NEXT:    s_cmp_lg_u32 s5, 0
 ; FLATSCR-NEXT:    s_cbranch_scc1 BB0_3
 ; FLATSCR-NEXT:  ; %bb.2: ; %bb.1
 ; FLATSCR-NEXT:    s_mov_b32 s2, s32
-; FLATSCR-NEXT:    s_movk_i32 s3, 0x1000
-; FLATSCR-NEXT:    s_add_i32 s4, s2, s3
+; FLATSCR-NEXT:    s_add_i32 s3, s2, 0x1000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v1, 0
+; FLATSCR-NEXT:    s_add_u32 s2, s2, 0x1000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 1
-; FLATSCR-NEXT:    s_add_u32 s2, s2, s3
 ; FLATSCR-NEXT:    scratch_store_dwordx2 off, v[1:2], s2
 ; FLATSCR-NEXT:    s_lshl_b32 s2, s6, 2
-; FLATSCR-NEXT:    s_mov_b32 s32, s4
-; FLATSCR-NEXT:    s_add_i32 s4, s4, s2
-; FLATSCR-NEXT:    scratch_load_dword v2, off, s4
+; FLATSCR-NEXT:    s_mov_b32 s32, s3
+; FLATSCR-NEXT:    s_add_i32 s3, s3, s2
+; FLATSCR-NEXT:    scratch_load_dword v2, off, s3
 ; FLATSCR-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_u32_e32 v0, v2, v0
 ; FLATSCR-NEXT:    s_waitcnt lgkmcnt(0)
 ; FLATSCR-NEXT:    global_store_dword v1, v0, s[0:1]
 ; FLATSCR-NEXT:  BB0_3: ; %bb.2
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v0, off

 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_sub_u32 s32, s32, 0x400
 ; MUBUF-NEXT:    s_mov_b32 s33, s7
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: func_non_entry_block_static_alloca_align4:
 ; FLATSCR:       ; %bb.0: ; %entry
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_mov_b32 s5, s33
+; FLATSCR-NEXT:    s_mov_b32 s4, s33
 ; FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; FLATSCR-NEXT:    s_mov_b32 s33, s32
 ; FLATSCR-NEXT:    s_add_u32 s32, s32, 16
 ; FLATSCR-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; FLATSCR-NEXT:    s_cbranch_execz BB2_3
 ; FLATSCR-NEXT:  ; %bb.1: ; %bb.0
 ; FLATSCR-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; FLATSCR-NEXT:    s_and_b64 exec, exec, vcc
 ; FLATSCR-NEXT:    s_cbranch_execz BB2_3
 ; FLATSCR-NEXT:  ; %bb.2: ; %bb.1
 ; FLATSCR-NEXT:    s_mov_b32 s2, s32
-; FLATSCR-NEXT:    s_movk_i32 s3, 0x1000
-; FLATSCR-NEXT:    s_add_i32 s4, s2, s3
+; FLATSCR-NEXT:    s_add_i32 s3, s2, 0x1000
+; FLATSCR-NEXT:    s_add_u32 s2, s2, 0x1000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 0
 ; FLATSCR-NEXT:    v_mov_b32_e32 v3, 1
-; FLATSCR-NEXT:    s_add_u32 s2, s2, s3
 ; FLATSCR-NEXT:    scratch_store_dwordx2 off, v[2:3], s2
-; FLATSCR-NEXT:    v_lshl_add_u32 v2, v4, 2, s4
+; FLATSCR-NEXT:    v_lshl_add_u32 v2, v4, 2, s3
 ; FLATSCR-NEXT:    scratch_load_dword v2, v2, off
 ; FLATSCR-NEXT:    v_and_b32_e32 v3, 0x3ff, v5
-; FLATSCR-NEXT:    s_mov_b32 s32, s4
+; FLATSCR-NEXT:    s_mov_b32 s32, s3
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    v_add_u32_e32 v2, v2, v3
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v2, off
 ; FLATSCR-NEXT:  BB2_3: ; %bb.2
 ; FLATSCR-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    global_store_dword v[0:1], v0, off
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_sub_u32 s32, s32, 16
-; FLATSCR-NEXT:    s_mov_b32 s33, s5
+; FLATSCR-NEXT:    s_mov_b32 s33, s4
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 
 entry:
   %cond0 = icmp eq i32 %arg.cond0, 0
   br i1 %cond0, label %bb.0, label %bb.2
 
 bb.0:
   %alloca = alloca [16 x i32], align 4, addrspace(5)