[AMDGPU] Spilling using flat scratch

[rampitec]

Summary

In theory at this point flat scratch should be functional.

[rampitec]

It needs ST addressing mode on GFX10.

[rampitec]

A fantastic result that it has passed PSDB on gfx9 with flat scratch enabled. I did not expect it to start working from the first attempt.

[arsenm]

In D89424#2333376, @rampitec wrote:

A fantastic result that it has passed PSDB on gfx9 with flat scratch enabled. I did not expect it to start working from the first attempt.

Should try with -O0 too

[rampitec]

In D89424#2333380, @arsenm wrote:

In D89424#2333376, @rampitec wrote:

A fantastic result that it has passed PSDB on gfx9 with flat scratch enabled. I did not expect it to start working from the first attempt.

Should try with -O0 too

I do not think it will pass with -O0 without my changes.

[rampitec]

Switched to ST mode on GFX10 is no SOffset is used.

[rampitec]

Rebased to parent.

[rampitec]

Rebased.

[rampitec]

I am integrating it into parent revision.

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

if (!TempSGPR) {
     LLVM_DEBUG(dbgs() << "Saving " << (IsFP ? "FP" : "BP") << " with copy to "
                       << printReg(TempSGPR, TRI) << '\n');
   }
 }
 
 // We need to specially emit stack operations here because a different frame
 // register is used than in the rest of the function, as getFrameRegister would
 // use.
-static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
+static void buildPrologSpill(const GCNSubtarget &ST, LivePhysRegs &LiveRegs,
+                             MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator I,
                              const SIInstrInfo *TII, Register SpillReg,
                              Register ScratchRsrcReg, Register SPReg, int FI) {
   MachineFunction *MF = MBB.getParent();
   MachineFrameInfo &MFI = MF->getFrameInfo();
 
   int64_t Offset = MFI.getObjectOffset(FI);
 
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 4,
       MFI.getObjectAlign(FI));
 
-  if (SIInstrInfo::isLegalMUBUFImmOffset(Offset)) {
+  if (ST.enableFlatScratch()) {
+    if (TII->isLegalFLATOffset(Offset, AMDGPUAS::PRIVATE_ADDRESS, true)) {
+      BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::SCRATCH_STORE_DWORD_SADDR))
+        .addReg(SpillReg, RegState::Kill)
+        .addReg(SPReg)
+        .addImm(Offset)
+        .addImm(0) // glc
+        .addImm(0) // slc
+        .addImm(0) // dlc
+        .addMemOperand(MMO);
+      return;
+    }
+  } else if (SIInstrInfo::isLegalMUBUFImmOffset(Offset)) {
     BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::BUFFER_STORE_DWORD_OFFSET))
       .addReg(SpillReg, RegState::Kill)
       .addReg(ScratchRsrcReg)
       .addReg(SPReg)
       .addImm(Offset)
       .addImm(0) // glc
       .addImm(0) // slc
       .addImm(0) // tfe
       .addImm(0) // dlc
       .addImm(0) // swz
       .addMemOperand(MMO);
     return;
   }
 
   // Don't clobber the TmpVGPR if we also need a scratch reg for the stack
   // offset in the spill.
   LiveRegs.addReg(SpillReg);
 
+  if (ST.enableFlatScratch()) {
+    MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister(
+      MF->getRegInfo(), LiveRegs, AMDGPU::SReg_32_XM0RegClass);
+
+    BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_ADD_U32), OffsetReg)
+      .addReg(SPReg)
+      .addImm(Offset);
+
+    BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::SCRATCH_STORE_DWORD_SADDR))
+      .addReg(SpillReg, RegState::Kill)
+      .addReg(OffsetReg, RegState::Kill)
+      .addImm(0)
+      .addImm(0) // glc
+      .addImm(0) // slc
+      .addImm(0) // dlc
+      .addMemOperand(MMO);
+  } else {
     MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister(
       MF->getRegInfo(), LiveRegs, AMDGPU::VGPR_32RegClass);
 
     BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::V_MOV_B32_e32), OffsetReg)
       .addImm(Offset);
 
     BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::BUFFER_STORE_DWORD_OFFEN))
       .addReg(SpillReg, RegState::Kill)
       .addReg(OffsetReg, RegState::Kill)
       .addReg(ScratchRsrcReg)
       .addReg(SPReg)
       .addImm(0)
       .addImm(0) // glc
       .addImm(0) // slc
       .addImm(0) // tfe
       .addImm(0) // dlc
       .addImm(0) // swz
       .addMemOperand(MMO);
+  }
 
   LiveRegs.removeReg(SpillReg);
 }
 
-static void buildEpilogReload(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
+static void buildEpilogReload(const GCNSubtarget &ST, LivePhysRegs &LiveRegs,
+                              MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator I,
                               const SIInstrInfo *TII, Register SpillReg,
                               Register ScratchRsrcReg, Register SPReg, int FI) {
   MachineFunction *MF = MBB.getParent();
   MachineFrameInfo &MFI = MF->getFrameInfo();
   int64_t Offset = MFI.getObjectOffset(FI);
 
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 4,
       MFI.getObjectAlign(FI));
 
+  if (ST.enableFlatScratch()) {
+    if (TII->isLegalFLATOffset(Offset, AMDGPUAS::PRIVATE_ADDRESS, true)) {
+      BuildMI(MBB, I, DebugLoc(),
+              TII->get(AMDGPU::SCRATCH_LOAD_DWORD_SADDR), SpillReg)
+        .addReg(SPReg)
+        .addImm(Offset)
+        .addImm(0) // glc
+        .addImm(0) // slc
+        .addImm(0) // dlc
+        .addMemOperand(MMO);
+      return;
+    }
+    MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister(
+      MF->getRegInfo(), LiveRegs, AMDGPU::SReg_32_XM0RegClass);
+
+      BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_ADD_U32), OffsetReg)
+        .addReg(SPReg)
+        .addImm(Offset);
+      BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::SCRATCH_LOAD_DWORD_SADDR),
+              SpillReg)
+      .addReg(OffsetReg, RegState::Kill)
+      .addImm(0)
+      .addImm(0) // glc
+      .addImm(0) // slc
+      .addImm(0) // dlc
+      .addMemOperand(MMO);
+      return;
+  }
+
   if (SIInstrInfo::isLegalMUBUFImmOffset(Offset)) {
     BuildMI(MBB, I, DebugLoc(),
             TII->get(AMDGPU::BUFFER_LOAD_DWORD_OFFSET), SpillReg)
       .addReg(ScratchRsrcReg)
       .addReg(SPReg)
       .addImm(Offset)
       .addImm(0) // glc
       .addImm(0) // slc
void SIFrameLowering::emitPrologue(MachineFunction &MF,
   for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg
          : FuncInfo->getSGPRSpillVGPRs()) {
     if (!Reg.FI.hasValue())
       continue;
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
 
-    buildPrologSpill(LiveRegs, MBB, MBBI, TII, Reg.VGPR,
+    buildPrologSpill(ST, LiveRegs, MBB, MBBI, TII, Reg.VGPR,
                      FuncInfo->getScratchRSrcReg(),
                      StackPtrReg,
                      Reg.FI.getValue());
   }
 
   if (HasFPSaveIndex && SpillFPToMemory) {
     assert(!MFI.isDeadObjectIndex(FuncInfo->FramePointerSaveIndex.getValue()));
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
 
     MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
         MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
 
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
         .addReg(FramePtrReg);
 
-    buildPrologSpill(LiveRegs, MBB, MBBI, TII, TmpVGPR,
+    buildPrologSpill(ST, LiveRegs, MBB, MBBI, TII, TmpVGPR,
                      FuncInfo->getScratchRSrcReg(), StackPtrReg,
                      FuncInfo->FramePointerSaveIndex.getValue());
   }
 
   if (HasBPSaveIndex && SpillBPToMemory) {
     assert(!MFI.isDeadObjectIndex(*FuncInfo->BasePointerSaveIndex));
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
 
     MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
         MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
 
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
         .addReg(BasePtrReg);
 
-    buildPrologSpill(LiveRegs, MBB, MBBI, TII, TmpVGPR,
+    buildPrologSpill(ST, LiveRegs, MBB, MBBI, TII, TmpVGPR,
                      FuncInfo->getScratchRSrcReg(), StackPtrReg,
                      *FuncInfo->BasePointerSaveIndex);
   }
 
   if (ScratchExecCopy) {
     // FIXME: Split block and make terminator.
     unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
     MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
if (HasFPSaveIndex) {
     const int FI = FuncInfo->FramePointerSaveIndex.getValue();
     assert(!MFI.isDeadObjectIndex(FI));
     if (SpillFPToMemory) {
       if (!ScratchExecCopy)
         ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
 
       MCPhysReg TempVGPR = findScratchNonCalleeSaveRegister(
           MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
-      buildEpilogReload(LiveRegs, MBB, MBBI, TII, TempVGPR,
+      buildEpilogReload(ST, LiveRegs, MBB, MBBI, TII, TempVGPR,
                         FuncInfo->getScratchRSrcReg(), StackPtrReg, FI);
       BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), FramePtrReg)
           .addReg(TempVGPR, RegState::Kill);
     } else {
       // Reload from VGPR spill.
       assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
       ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
           FuncInfo->getSGPRToVGPRSpills(FI);
if (HasBPSaveIndex) {
     const int BasePtrFI = *FuncInfo->BasePointerSaveIndex;
     assert(!MFI.isDeadObjectIndex(BasePtrFI));
     if (SpillBPToMemory) {
       if (!ScratchExecCopy)
         ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
 
       MCPhysReg TempVGPR = findScratchNonCalleeSaveRegister(
           MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
-      buildEpilogReload(LiveRegs, MBB, MBBI, TII, TempVGPR,
+      buildEpilogReload(ST, LiveRegs, MBB, MBBI, TII, TempVGPR,
                         FuncInfo->getScratchRSrcReg(), StackPtrReg, BasePtrFI);
       BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), BasePtrReg)
           .addReg(TempVGPR, RegState::Kill);
     } else {
       // Reload from VGPR spill.
       assert(MFI.getStackID(BasePtrFI) == TargetStackID::SGPRSpill);
       ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
           FuncInfo->getSGPRToVGPRSpills(BasePtrFI);
       assert(Spill.size() == 1);
       BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32),
               BasePtrReg)
           .addReg(Spill[0].VGPR)
           .addImm(Spill[0].Lane);
     }
   }
 
   for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg :
        FuncInfo->getSGPRSpillVGPRs()) {
     if (!Reg.FI.hasValue())
       continue;
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
 
-    buildEpilogReload(LiveRegs, MBB, MBBI, TII, Reg.VGPR,
+    buildEpilogReload(ST, LiveRegs, MBB, MBBI, TII, Reg.VGPR,
                       FuncInfo->getScratchRSrcReg(), StackPtrReg,
                       Reg.FI.getValue());
   }
 
   if (ScratchExecCopy) {
     // FIXME: Split block and make terminator.
     unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
     MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;

llvm/lib/Target/AMDGPU/SIInstrInfo.h

public:
 
   // Is a FLAT encoded instruction which accesses a specific segment,
   // i.e. global_* or scratch_*.
   static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
     auto Flags = MI.getDesc().TSFlags;
     return (Flags & SIInstrFlags::FLAT) && !(Flags & SIInstrFlags::LGKM_CNT);
   }
 
+  bool isSegmentSpecificFLAT(uint16_t Opcode) const {
+    auto Flags = get(Opcode).TSFlags;
+    return (Flags & SIInstrFlags::FLAT) && !(Flags & SIInstrFlags::LGKM_CNT);
+  }
+
   // FIXME: Make this more precise
   static bool isFLATScratch(const MachineInstr &MI) {
     return isSegmentSpecificFLAT(MI);
   }
 
+  bool isFLATScratch(uint16_t Opcode) const {
+    return isSegmentSpecificFLAT(Opcode);
+  }
+
   // Any FLAT encoded instruction, including global_* and scratch_*.
   bool isFLAT(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::FLAT;
   }
 
   static bool isEXP(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::EXP;
   }

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
                                          MachineMemOperand *MMO,
                                          RegScavenger *RS) const {
   MachineBasicBlock *MBB = MI->getParent();
   MachineFunction *MF = MI->getParent()->getParent();
   const SIInstrInfo *TII = ST.getInstrInfo();
   const MachineFrameInfo &MFI = MF->getFrameInfo();
   const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
 
-  const MCInstrDesc &Desc = TII->get(LoadStoreOp);
+  const MCInstrDesc *Desc = &TII->get(LoadStoreOp);
   const DebugLoc &DL = MI->getDebugLoc();
-  bool IsStore = Desc.mayStore();
+  bool IsStore = Desc->mayStore();
+  bool IsFlat = TII->isFLATScratch(LoadStoreOp);
 
   bool Scavenged = false;
   MCRegister SOffset = ScratchOffsetReg;
 
   const unsigned EltSize = 4;
   const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg);
   unsigned NumSubRegs = AMDGPU::getRegBitWidth(RC->getID()) / (EltSize * CHAR_BIT);
   unsigned Size = NumSubRegs * EltSize;
   int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
+  int64_t MaxOffset = Offset + Size - EltSize;
   int64_t ScratchOffsetRegDelta = 0;
 
   Align Alignment = MFI.getObjectAlign(Index);
   const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo();
 
   assert((Offset % EltSize) == 0 && "unexpected VGPR spill offset");
 
-  if (!SIInstrInfo::isLegalMUBUFImmOffset(Offset + Size - EltSize)) {
+  if (IsFlat
+        ? !TII->isLegalFLATOffset(MaxOffset, AMDGPUAS::PRIVATE_ADDRESS, true)
+        : !SIInstrInfo::isLegalMUBUFImmOffset(MaxOffset)) {
     SOffset = MCRegister();
 
     // We currently only support spilling VGPRs to EltSize boundaries, meaning
     // we can simplify the adjustment of Offset here to just scale with
     // WavefrontSize.
+    if (!IsFlat)
       Offset *= ST.getWavefrontSize();
 
     // We don't have access to the register scavenger if this function is called
     // during  PEI::scavengeFrameVirtualRegs().
     if (RS)
       SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false);
 
     if (!SOffset) {
       // There are no free SGPRs, and since we are in the process of spilling
if (ScratchOffsetReg == AMDGPU::NoRegister) {
       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset)
           .addReg(ScratchOffsetReg)
           .addImm(Offset);
     }
 
     Offset = 0;
   }
 
+  if (IsFlat && SOffset == AMDGPU::NoRegister) {
+    assert(AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0
+           && "Unexpected vaddr for flat scratch with a FI operand");
+
+    if (ST.hasFlatScratchSTMode()) {
+      LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp);
+      Desc = &TII->get(LoadStoreOp);
+    } else {
+      // We need a register for SADDR even if to just hold a zero.
+      if (RS)
+        SOffset = RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0,
+                                       false);
+      // This can fail unless we have a way to spill without extra registers
+      // needed during spilling. There is no such problem on GFX10 though
+      // because we can use NULL register.
+      if (!SOffset)
+        report_fatal_error("could not scavenge SGPR to spill in entry function");
+
+      BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset)
+          .addImm(0);
+      Scavenged = true;
+    }
+  }
+
   Register TmpReg;
 
+  // FIXME: Flat scratch does not have to be limited to a dword per store.
   for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) {
     Register SubReg = NumSubRegs == 1
                           ? Register(ValueReg)
                           : getSubReg(ValueReg, getSubRegFromChannel(i));
 
     unsigned SOffsetRegState = 0;
     unsigned SrcDstRegState = getDefRegState(!IsStore);
     if (i + 1 == e) {
if (!MIB.getInstr()) {
         SubReg = TmpReg;
       }
 
       MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i);
       MachineMemOperand *NewMMO =
           MF->getMachineMemOperand(PInfo, MMO->getFlags(), EltSize,
                                    commonAlignment(Alignment, EltSize * i));
 
-      MIB = BuildMI(*MBB, MI, DL, Desc)
+      MIB = BuildMI(*MBB, MI, DL, *Desc)
                 .addReg(SubReg,
-                        getDefRegState(!IsStore) | getKillRegState(IsKill))
-                .addReg(ScratchRsrcReg);
+                        getDefRegState(!IsStore) | getKillRegState(IsKill));
+      if (!IsFlat)
+        MIB.addReg(ScratchRsrcReg);
+
       if (SOffset == AMDGPU::NoRegister) {
+        if (!IsFlat)
           MIB.addImm(0);
       } else {
         MIB.addReg(SOffset, SOffsetRegState);
       }
       MIB.addImm(Offset)
           .addImm(0) // glc
           .addImm(0) // slc
-          .addImm(0) // tfe
-          .addImm(0) // dlc
-          .addImm(0) // swz
-          .addMemOperand(NewMMO);
+          .addImm(0); // tfe for MUBUF or dlc for FLAT
+      if (!IsFlat)
+        MIB.addImm(0) // dlc
+           .addImm(0); // swz
+      MIB.addMemOperand(NewMMO);
 
       if (!IsAGPR && NeedSuperRegDef)
         MIB.addReg(ValueReg, RegState::ImplicitDefine);
 
       if (!IsStore && TmpReg != AMDGPU::NoRegister)
         MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32),
                       FinalReg)
           .addReg(TmpReg, RegState::Kill);
void SIRegisterInfo::buildSGPRSpillLoadStore(MachineBasicBlock::iterator MI,
   Align Alignment = FrameInfo.getObjectAlign(Index);
   MachinePointerInfo PtrInfo =
       MachinePointerInfo::getFixedStack(*MF, Index);
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       PtrInfo, IsLoad ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore,
       EltSize, Alignment);
 
   if (IsLoad) {
-    buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET,
+    unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
+                                          : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
+    buildSpillLoadStore(MI, Opc,
           Index,
           VGPR, false,
           MFI->getScratchRSrcReg(), FrameReg,
           Offset * EltSize, MMO,
           RS);
   } else {
-    buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET, Index, VGPR,
+    unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
+                                          : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
+    buildSpillLoadStore(MI, Opc, Index, VGPR,
                         IsKill, MFI->getScratchRSrcReg(), FrameReg,
                         Offset * EltSize, MMO, RS);
     // This only ever adds one VGPR spill
     MFI->addToSpilledVGPRs(1);
   }
 
   // Restore EXEC
   BuildMI(*MBB, MI, DL, TII->get(ExecMovOpc), ExecReg)
switch (MI->getOpcode()) {
     case AMDGPU::SI_SPILL_A96_SAVE:
     case AMDGPU::SI_SPILL_A64_SAVE:
     case AMDGPU::SI_SPILL_A32_SAVE: {
       const MachineOperand *VData = TII->getNamedOperand(*MI,
                                                          AMDGPU::OpName::vdata);
       assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
              MFI->getStackPtrOffsetReg());
 
-      buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET,
+      unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
+                                            : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
+      buildSpillLoadStore(MI, Opc,
             Index,
             VData->getReg(), VData->isKill(),
             TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(),
             FrameReg,
             TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
             *MI->memoperands_begin(),
             RS);
       MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode()));
switch (MI->getOpcode()) {
     case AMDGPU::SI_SPILL_A256_RESTORE:
     case AMDGPU::SI_SPILL_A512_RESTORE:
     case AMDGPU::SI_SPILL_A1024_RESTORE: {
       const MachineOperand *VData = TII->getNamedOperand(*MI,
                                                          AMDGPU::OpName::vdata);
       assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
              MFI->getStackPtrOffsetReg());
 
-      buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET,
+      unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
+                                            : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
+      buildSpillLoadStore(MI, Opc,
             Index,
             VData->getReg(), VData->isKill(),
             TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(),
             FrameReg,
             TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(),
             *MI->memoperands_begin(),
             RS);
       MI->eraseFromParent();

llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll

 define amdgpu_kernel void @test_kernel_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void() #0 {
   call void @external_void_func_void()
   call void asm sideeffect "", ""() #0
   call void @external_void_func_void()
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_func_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void:
-; GCN: buffer_store_dword
+; MUBUF:   buffer_store_dword
+; FLATSCR: scratch_store_dword
 ; GCN: v_writelane_b32 v40, s33, 4
 ; GCN: v_writelane_b32 v40, s34, 0
 ; GCN: v_writelane_b32 v40, s35, 1
 ; GCN: v_writelane_b32 v40, s30, 2
 ; GCN: v_writelane_b32 v40, s31, 3
 
 ; GCN: s_swappc_b64
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NEXT: s_swappc_b64
 ; GCN-DAG: v_readlane_b32 s4, v40, 2
 ; GCN-DAG: v_readlane_b32 s5, v40, 3
 ; GCN: v_readlane_b32 s35, v40, 1
 ; GCN: v_readlane_b32 s34, v40, 0
 
 ; GCN: v_readlane_b32 s33, v40, 4
-; GCN: buffer_load_dword
+; MUBUF:   buffer_load_dword
+; FLATSCR: scratch_load_dword
 ; GCN: s_setpc_b64
 define void @test_func_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void() #0 {
   call void @external_void_func_void()
   call void asm sideeffect "", ""() #0
   call void @external_void_func_void()
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_func_call_external_void_funcx2:
-; GCN: buffer_store_dword v40
+; MUBUF:   buffer_store_dword v40
+; FLATSCR: scratch_store_dword off, v40
 ; GCN: v_writelane_b32 v40, s33, 4
 
 ; GCN: s_mov_b32 s33, s32
 ; MUBUF:   s_add_u32 s32, s32, 0x400
 ; FLATSCR: s_add_u32 s32, s32, 16
 ; GCN: s_swappc_b64
 ; GCN-NEXT: s_swappc_b64
 
 ; GCN: v_readlane_b32 s33, v40, 4
-; GCN: buffer_load_dword v40,
+; MUBUF:   buffer_load_dword v40
+; FLATSCR: scratch_load_dword v40
 define void @test_func_call_external_void_funcx2() #0 {
   call void @external_void_func_void()
   call void @external_void_func_void()
   ret void
 }
 
 ; GCN-LABEL: {{^}}void_func_void_clobber_s30_s31:
 ; GCN: s_waitcnt

llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll

define void @callee_with_stack_no_fp_elim_non_leaf() #2 {
   store volatile i32 0, i32 addrspace(5)* %alloca
   ret void
 }
 
 ; GCN-LABEL: {{^}}callee_with_stack_and_call:
 ; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 offset:4 ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN: v_writelane_b32 [[CSR_VGPR]], s33, 2
 ; GCN-DAG: s_mov_b32 s33, s32
 ; MUBUF-DAG:   s_add_u32 s32, s32, 0x400{{$}}
 ; FLATSCR-DAG: s_add_u32 s32, s32, 16{{$}}
 ; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30,
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31,
 
 ; MUBUF-DAG:   buffer_store_dword [[ZERO]], off, s[0:3], s33{{$}}
 ; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33{{$}}
 
 ; GCN: s_swappc_b64
 
 ; GCN-DAG: v_readlane_b32 s5, [[CSR_VGPR]]
 ; GCN-DAG: v_readlane_b32 s4, [[CSR_VGPR]]
 
 ; MUBUF:    s_sub_u32 s32, s32, 0x400{{$}}
 ; FLATSCR:  s_sub_u32 s32, s32, 16{{$}}
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:4 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 
 ; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_and_call() #0 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   call void @external_void_func_void()
   ret void
 }
 
 ; Should be able to copy incoming stack pointer directly to inner
 ; call's stack pointer argument.
 
 ; There is stack usage only because of the need to evict a VGPR for
 ; spilling CSR SGPRs.
 
 ; GCN-LABEL: {{^}}callee_no_stack_with_call:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill
+; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill
+; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; MUBUF-DAG:   s_add_u32 s32, s32, 0x400
 ; FLATSCR-DAG: s_add_u32 s32, s32, 16
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s33, [[FP_SPILL_LANE:[0-9]+]]
 
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; GCN: s_swappc_b64
 
 ; GCN-DAG: v_readlane_b32 s4, v40, 0
 ; GCN-DAG: v_readlane_b32 s5, v40, 1
 
 ; MUBUF:   s_sub_u32 s32, s32, 0x400
 ; FLATSCR: s_sub_u32 s32, s32, 16
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], [[FP_SPILL_LANE]]
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
+; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
+; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @callee_no_stack_with_call() #0 {
   call void @external_void_func_void()
   ret void
 }
 
 declare hidden void @external_void_func_void() #0
 
 ; Make sure if a CSR vgpr is used for SGPR spilling, it is saved and
 ; restored. No FP is required.
 ;
 ; GCN-LABEL: {{^}}callee_func_sgpr_spill_no_calls:
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill
+; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill
+; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN: v_writelane_b32 [[CSR_VGPR]], s
 ; GCN: v_writelane_b32 [[CSR_VGPR]], s
 
 ; GCN: ;;#ASMSTART
 ; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]]
 ; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]]
 
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
+; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
+; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"() #0
   call void asm sideeffect "", "~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15}"() #0
   call void asm sideeffect "", "~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23}"() #0
   call void asm sideeffect "", "~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"() #0

 }
 
 ; TODO: Can the SP inc/deec be remvoed?
 ; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_csr_vgpr:
 ; GCN: s_waitcnt
 ; GCN-NEXT:s_mov_b32 [[FP_COPY:s[0-9]+]], s33
 ; GCN-NEXT: s_mov_b32 s33, s32
 ; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; GCN-DAG: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-DAG:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
 ; MUBUF-DAG:   buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:8
 ; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33 offset:8
 
 ; GCN:  ;;#ASMSTART
 ; GCN-NEXT: ; clobber v41
 ; GCN-NEXT: ;;#ASMEND
 
-; GCN: buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF:   buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
 ; MUBUF:        s_add_u32 s32, s32, 0x300
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300
 ; FLATSCR:      s_add_u32 s32, s32, 12
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 12
 ; GCN-NEXT: s_mov_b32 s33, s4
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_no_fp_elim_csr_vgpr() #1 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   call void asm sideeffect "; clobber v41", "~{v41}"()
   ret void
 }
 
 ; Use a copy to a free SGPR instead of introducing a second CSR VGPR.
 ; GCN-LABEL: {{^}}last_lane_vgpr_for_fp_csr:
 ; GCN: s_waitcnt
 ; GCN-NEXT: v_writelane_b32 v1, s33, 63
 ; GCN-NEXT: s_mov_b32 s33, s32
-; GCN: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; FLATSCR: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
 ; GCN-COUNT-63: v_writelane_b32 v1
 ; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33 offset:8
 ; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s33 offset:8
 ; GCN: ;;#ASMSTART
 ; GCN-COUNT-63: v_readlane_b32 s{{[0-9]+}}, v1
 
 ; MUBUF:        s_add_u32 s32, s32, 0x300
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300
define void @last_lane_vgpr_for_fp_csr() #1 {
   ret void
 }
 
 ; Use a copy to a free SGPR instead of introducing a second CSR VGPR.
 ; GCN-LABEL: {{^}}no_new_vgpr_for_fp_csr:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_mov_b32 [[FP_COPY:s[0-9]+]], s33
 ; GCN-NEXT: s_mov_b32 s33, s32
-; GCN-NEXT: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; MUBUF-NEXT:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; FLATSCR-NEXT: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
 ; GCN-COUNT-64: v_writelane_b32 v1,
 
 ; MUBUF:   buffer_store_dword
 ; FLATSCR: scratch_store_dword
 ; GCN: ;;#ASMSTART
 ; GCN-COUNT-64: v_readlane_b32 s{{[0-9]+}}, v1
 
-; GCN: buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; MUBUF:   buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
 ; MUBUF:        s_add_u32 s32, s32, 0x300
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300
 ; FLATSCR:      s_add_u32 s32, s32, 12
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 12
 ; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @no_new_vgpr_for_fp_csr() #1 {
define void @no_unused_non_csr_sgpr_for_fp() #1 {
 
   ret void
 }
 
 ; Need a new CSR VGPR to satisfy the FP spill.
 ; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 offset:8 ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2
 ; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-NEXT: s_mov_b32 s33, s32
 
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; MUBUF-DAG:   buffer_store_dword
 ; FLATSCR-DAG: scratch_store_dword
 ; MUBUF:       s_add_u32 s32, s32, 0x300{{$}}
 ; FLATSCR:     s_add_u32 s32, s32, 12{{$}}
 
 ; GCN: ;;#ASMSTART
 
 ; GCN:          v_readlane_b32 s4, [[CSR_VGPR]], 0
 ; GCN-NEXT:     v_readlane_b32 s5, [[CSR_VGPR]], 1
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300{{$}}
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 12{{$}}
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:8 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
 
   ; Use all clobberable registers, so FP has to spill to a VGPR.
define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
   ret void
 }
 
 ; The byval argument exceeds the MUBUF constant offset, so a scratch
 ; register is needed to access the CSR VGPR slot.
 ; GCN-LABEL: {{^}}scratch_reg_needed_mubuf_offset:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
-; GCN-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Spill
+; MUBUF-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
+; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Spill
+; FLATSCR-NEXT: s_add_u32 [[SCRATCH_SGPR:s[0-9]+]], s32, 0x1008
+; FLATSCR-NEXT: s_nop 0
+; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], [[SCRATCH_SGPR]] ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2
 ; GCN-DAG:  v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-DAG:  s_mov_b32 s33, s32
 ; GCN-DAG:  v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; MUBUF-DAG:   s_add_u32 s32, s32, 0x40300{{$}}
 ; FLATSCR-DAG: s_add_u32 s32, s32, 0x100c{{$}}
 ; MUBUF-DAG:   buffer_store_dword
 ; FLATSCR-DAG: scratch_store_dword
 
 ; GCN: ;;#ASMSTART
 
 ; GCN: v_readlane_b32 s4, [[CSR_VGPR]], 0
 ; GCN-NEXT: v_readlane_b32 s5, [[CSR_VGPR]], 1
 ; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x40300{{$}}
 ; FLATSCR-NEXT: s_sub_u32 s32, s32, 0x100c{{$}}
 ; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
-; GCN-NEXT: buffer_load_dword [[CSR_VGPR]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Reload
+; MUBUF-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
+; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Reload
+; FLATSCR-NEXT: s_add_u32 [[SCRATCH_SGPR:s[0-9]+]], s32, 0x1008
+; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, [[SCRATCH_SGPR]] ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval align 4 %arg) #1 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
 
   ; Use all clobberable registers, so FP has to spill to a VGPR.
define void @ipra_call_with_stack() #0 {
   call void @local_empty_func()
   ret void
 }
 
 ; With no free registers, we must spill the FP to memory.
 ; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory:
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN: v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
-; GCN: buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:4
+; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:4
+; FLATSCR: scratch_store_dword off, [[TMP_VGPR1]], s32 offset:4
 ; GCN: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN: s_mov_b32 s33, s32
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN: buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:4
+; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:4
+; FLATSCR: scratch_load_dword [[TMP_VGPR2:v[0-9]+]], off, s32 offset:4
 ; GCN: s_waitcnt vmcnt(0)
 ; GCN: v_readfirstlane_b32 s33, [[TMP_VGPR2]]
 ; GCN: s_mov_b64 exec, [[COPY_EXEC2]]
 ; GCN: s_setpc_b64
 ; GCN: ScratchSize: 8
 define void @callee_need_to_spill_fp_to_memory() #3 {
   call void asm sideeffect "; clobber nonpreserved SGPRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}

 }
 
 ; If we have a reserved VGPR that can be used for SGPR spills, we may still
 ; need to spill the FP to memory if there are no free lanes in the reserved
 ; VGPR.
 ; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory_full_reserved_vgpr:
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN: v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
-; GCN: buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:[[OFF:[0-9]+]]
+; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:[[OFF:[0-9]+]]
+; FLATSCR: scratch_store_dword off, [[TMP_VGPR1]], s32 offset:[[OFF:[0-9]+]]
 ; GCN: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NOT: v_writelane_b32 v40, s33
 ; GCN: s_mov_b32 s33, s32
 ; GCN-NOT: v_readlane_b32 s33, v40
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
-; GCN: buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:[[OFF]]
+; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:[[OFF]]
+; FLATSCR: scratch_load_dword [[TMP_VGPR2:v[0-9]+]], off, s32 offset:[[OFF]]
 ; GCN: v_readfirstlane_b32 s33, [[TMP_VGPR2]]
 ; GCN: s_mov_b64 exec, [[COPY_EXEC2]]
 ; GCN: s_setpc_b64
 define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
   call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
     ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
call void asm sideeffect "; clobber all VGPRs except CSR v40",
     ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"()
   ret void
 }
 
 ; If the size of the offset exceeds the MUBUF offset field we need another
 ; scratch VGPR to hold the offset.
 ; GCN-LABEL: {{^}}spill_fp_to_memory_scratch_reg_needed_mubuf_offset
 ; GCN: s_or_saveexec_b64 s[4:5], -1
-; GCN: v_mov_b32_e32 v0, s33
+; MUBUF: v_mov_b32_e32 v0, s33
 ; GCN-NOT: v_mov_b32_e32 v0, 0x1008
-; GCN-NEXT: v_mov_b32_e32 v1, 0x1008
-; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen
+; MUBUF-NEXT: v_mov_b32_e32 v1, 0x1008
+; MUBUF-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen ; 4-byte Folded Spill
+; FLATSCR-NEXT: s_add_u32 [[SOFF:s[0-9]+]], s32, 0x1008
+; FLATSCR-NEXT: v_mov_b32_e32 v0, s33
+; FLATSCR-NEXT: scratch_store_dword off, v0, [[SOFF]] ; 4-byte Folded Spill
 define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval align 4 %arg) #3 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
 
   call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
     "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
     ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
     ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}

llvm/test/CodeGen/AMDGPU/multi-dword-vgpr-spill.ll

-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -post-RA-scheduler=0 -stress-regalloc=8 < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -post-RA-scheduler=0 -stress-regalloc=8 < %s | FileCheck %s -check-prefixes=GCN,MUBUF
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -post-RA-scheduler=0 -stress-regalloc=8 -amdgpu-enable-flat-scratch < %s | FileCheck %s -check-prefixes=GCN,FLATSCR
 
-; CHECK-LABEL: spill_v2i32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:16 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:20 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:16 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:20 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v2i32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:16 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:20 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:16 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:20 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:16 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:20 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:16 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:20 ; 4-byte Folded Reload
 
 define void @spill_v2i32() {
 entry:
   %alloca = alloca <2 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <2 x i32>, <2 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %alloca, i32 1
   store volatile <2 x i32> %a, <2 x i32> addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: spill_v2f32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:16 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:20 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:16 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:20 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v2f32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:16 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:20 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:16 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:20 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:16 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:20 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:16 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:20 ; 4-byte Folded Reload
 
 define void @spill_v2f32() {
 entry:
   %alloca = alloca <2 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <2 x i32>, <2 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <2 x i32>, <2 x i32> addrspace(5)* %alloca, i32 1
   store volatile <2 x i32> %a, <2 x i32> addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: spill_v3i32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v3i32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:32 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:36 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:40 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
 
 define void @spill_v3i32() {
 entry:
   %alloca = alloca <3 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <3 x i32>, <3 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <3 x i32>, <3 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <3 x i32>, <3 x i32> addrspace(5)* %alloca, i32 1
   store volatile <3 x i32> %a, <3 x i32> addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: spill_v3f32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v3f32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:32 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:36 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:40 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
 
 define void @spill_v3f32() {
 entry:
   %alloca = alloca <3 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <3 x i32>, <3 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <3 x i32>, <3 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <3 x i32>, <3 x i32> addrspace(5)* %alloca, i32 1
   store volatile <3 x i32> %a, <3 x i32> addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: spill_v4i32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:44 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:44 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v4i32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:44 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:32 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:36 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:40 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:44 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:44 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:44 ; 4-byte Folded Reload
 
 define void @spill_v4i32() {
 entry:
   %alloca = alloca <4 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <4 x i32>, <4 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <4 x i32>, <4 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <4 x i32>, <4 x i32> addrspace(5)* %alloca, i32 1
   store volatile <4 x i32> %a, <4 x i32> addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: spill_v4f32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:44 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:44 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v4f32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:32 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:36 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:40 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:44 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:32 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:36 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:40 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:44 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:44 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:32 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:36 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:40 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:44 ; 4-byte Folded Reload
 
 define void @spill_v4f32() {
 entry:
   %alloca = alloca <4 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <4 x i32>, <4 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <4 x i32>, <4 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <4 x i32>, <4 x i32> addrspace(5)* %alloca, i32 1
   store volatile <4 x i32> %a, <4 x i32> addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: spill_v5i32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:64 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:68 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:72 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:76 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:64 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:68 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:72 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:76 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v5i32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:64 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:68 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:72 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:76 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:64 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:68 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:72 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:76 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:64 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:68 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:72 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:76 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:64 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:68 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:72 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:76 ; 4-byte Folded Reload
 define void @spill_v5i32() {
 entry:
   %alloca = alloca <5 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <5 x i32>, <5 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <5 x i32>, <5 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <5 x i32>, <5 x i32> addrspace(5)* %alloca, i32 1
   store volatile <5 x i32> %a, <5 x i32> addrspace(5)* %outptr
 
   ret void
 }
 
-; CHECK-LABEL: spill_v5f32:
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:64 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:68 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:72 ; 4-byte Folded Spill
-; CHECK-DAG: buffer_store_dword v{{.*}} offset:76 ; 4-byte Folded Spill
-; CHECK: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:64 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:68 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:72 ; 4-byte Folded Reload
-; CHECK-DAG: buffer_load_dword v{{.*}} offset:76 ; 4-byte Folded Reload
+; GCN-LABEL: spill_v5f32:
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:64 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:68 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:72 ; 4-byte Folded Spill
+; MUBUF-DAG: buffer_store_dword v{{.*}} offset:76 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:64 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:68 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:72 ; 4-byte Folded Spill
+; FLATSCR-DAG: scratch_store_dword off, v{{.*}} offset:76 ; 4-byte Folded Spill
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:64 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:68 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:72 ; 4-byte Folded Reload
+; MUBUF-DAG: buffer_load_dword v{{.*}} offset:76 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:64 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:68 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:72 ; 4-byte Folded Reload
+; FLATSCR-DAG: scratch_load_dword v{{.*}} offset:76 ; 4-byte Folded Reload
 define void @spill_v5f32() {
 entry:
   %alloca = alloca <5 x i32>, i32 2, align 4, addrspace(5)
 
   %aptr = getelementptr <5 x i32>, <5 x i32> addrspace(5)* %alloca, i32 1
   %a = load volatile <5 x i32>, <5 x i32> addrspace(5)* %aptr
 
   ; Force %a to spill.
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}" ()
 
   %outptr = getelementptr <5 x i32>, <5 x i32> addrspace(5)* %alloca, i32 1
   store volatile <5 x i32> %a, <5 x i32> addrspace(5)* %outptr
 
   ret void
 }

llvm/test/CodeGen/AMDGPU/sgpr-spill.mir

-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=prologepilog %s -o - | FileCheck -check-prefix=CHECK -check-prefix=GCN64 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -verify-machineinstrs -run-pass=prologepilog %s -o - | FileCheck -check-prefix=CHECK -check-prefix=GCN32 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=prologepilog %s -o - | FileCheck -check-prefixes=CHECK,GCN64,MUBUF %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -verify-machineinstrs -run-pass=prologepilog %s -o - | FileCheck -check-prefixes=CHECK,GCN32,MUBUF %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-flat-scratch -run-pass=prologepilog %s -o - | FileCheck -check-prefixes=CHECK,GCN64,FLATSCR %s
 
 
 # CHECK-LABEL: name: check_spill
 
 # S32 with kill
 # CHECK: V_WRITELANE
 # CHECK: $sgpr12 = S_MOV_B32 $exec_lo
 # CHECK: $exec_lo = S_MOV_B32 1
-# CHECK: BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 4
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 4
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr{{[0-9]+}}, $sgpr33, 4
 # CHECK: $exec_lo = S_MOV_B32 killed $sgpr12
 
 # S32 without kill
 # CHECK: V_WRITELANE
 # CHECK: $sgpr12 = S_MOV_B32 $exec_lo
 # CHECK: $exec_lo = S_MOV_B32 1
-# CHECK: BUFFER_STORE_DWORD_OFFSET $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 4
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 4
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR $vgpr{{[0-9]+}}, $sgpr33, 4
 # CHECK: $sgpr12 = V_READLANE
 
 # S64 with kill
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 3
 # GCN64: $exec = S_MOV_B64 3
-# CHECK: BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 8
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 8
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr{{[0-9]+}}, $sgpr33, 8
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 
 # S64 without kill
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 3
 # GCN64: $exec = S_MOV_B64 3
-# CHECK: BUFFER_STORE_DWORD_OFFSET $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 8
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 8
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR $vgpr{{[0-9]+}}, $sgpr33, 8
 # GCN32: $exec_lo = S_MOV_B32 $sgpr12
 # GCN64: $exec = S_MOV_B64 $sgpr12_sgpr13
 # GCN64: $sgpr13 = V_READLANE
 # CHECK: $sgpr12 = V_READLANE
 
 # S96
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 7
 # GCN64: $exec = S_MOV_B64 7
-# CHECK: BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 16
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 16
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr{{[0-9]+}}, $sgpr33, 16
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 
 # S128
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 15
 # GCN64: $exec = S_MOV_B64 15
-# CHECK: BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 28
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET killed $vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 28
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr{{[0-9]+}}, $sgpr33, 28
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 
 # S160
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 31
 # GCN64: $exec = S_MOV_B64 31
-# CHECK: BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 44
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 44
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR {{(killed )?}}$vgpr{{[0-9]+}}, $sgpr33, 44
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 
 # S256
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 255
 # GCN64: $exec = S_MOV_B64 255
-# CHECK: BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 64
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 64
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR {{(killed )?}}$vgpr{{[0-9]+}}, $sgpr33, 64
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 
 # S512
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 65535
 # GCN64: $exec = S_MOV_B64 65535
-# CHECK: BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 96
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 96
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR {{(killed )?}}$vgpr{{[0-9]+}}, $sgpr33, 96
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 
 # S1024
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE

 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # CHECK: V_WRITELANE
 # GCN32: $sgpr64 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr64_sgpr65 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 4294967295
 # GCN64: $exec = S_MOV_B64 4294967295
-# CHECK: BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 160
+# MUBUF:   BUFFER_STORE_DWORD_OFFSET {{(killed )?}}$vgpr{{[0-9]+}}, ${{(sgpr[0-9_]+)*}}, $sgpr33, 160
+# FLATSCR: SCRATCH_STORE_DWORD_SADDR {{(killed )?}}$vgpr{{[0-9]+}}, $sgpr33, 160
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr64
 # GCN64: $exec = S_MOV_B64 killed $sgpr64_sgpr65
 
 --- |
 
   define amdgpu_kernel void @check_spill() #0 {
     ret void
   }
bb.0:
     SI_SPILL_S1024_SAVE killed $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, %stack.7, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr32
 
 
 # CHECK-LABEL: name: check_reload
 
 # S32
 # CHECK: $sgpr12 = S_MOV_B32 $exec_lo
 # CHECK: $exec_lo = S_MOV_B32 1
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 4
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 4
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 4
 # CHECK: $exec_lo = S_MOV_B32 killed $sgpr12
 # CHECK: $sgpr12 = V_READLANE
 
 # S64
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 3
 # GCN64: $exec = S_MOV_B64 3
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 8
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 8
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 8
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 # CHECK: $sgpr12 = V_READLANE
 # CHECK: $sgpr13 = V_READLANE
 
 # S96
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 7
 # GCN64: $exec = S_MOV_B64 7
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 16
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 16
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 16
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 # CHECK: $sgpr12 = V_READLANE
 # CHECK: $sgpr13 = V_READLANE
 # CHECK: $sgpr14 = V_READLANE
 
 # S128
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 15
 # GCN64: $exec = S_MOV_B64 15
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 28
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 28
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 28
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 # CHECK: $sgpr12 = V_READLANE
 # CHECK: $sgpr13 = V_READLANE
 # CHECK: $sgpr14 = V_READLANE
 # CHECK: $sgpr15 = V_READLANE
 
 # S160
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 31
 # GCN64: $exec = S_MOV_B64 31
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 44
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 44
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 44
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 # CHECK: $sgpr12 = V_READLANE
 # CHECK: $sgpr13 = V_READLANE
 # CHECK: $sgpr14 = V_READLANE
 # CHECK: $sgpr15 = V_READLANE
 # CHECK: $sgpr16 = V_READLANE
 
 # S256
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 255
 # GCN64: $exec = S_MOV_B64 255
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 64
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 64
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 64
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 # CHECK: $sgpr12 = V_READLANE
 # CHECK: $sgpr13 = V_READLANE
 # CHECK: $sgpr14 = V_READLANE
 # CHECK: $sgpr15 = V_READLANE
 # CHECK: $sgpr16 = V_READLANE
 # CHECK: $sgpr17 = V_READLANE
 # CHECK: $sgpr18 = V_READLANE
 # CHECK: $sgpr19 = V_READLANE
 
 # S512
 # GCN32: $sgpr12 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr12_sgpr13 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 65535
 # GCN64: $exec = S_MOV_B64 65535
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 96
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 96
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 96
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr12
 # GCN64: $exec = S_MOV_B64 killed $sgpr12_sgpr13
 # CHECK: $sgpr12 = V_READLANE
 # CHECK: $sgpr13 = V_READLANE
 # CHECK: $sgpr14 = V_READLANE
 # CHECK: $sgpr15 = V_READLANE
 # CHECK: $sgpr16 = V_READLANE
 # CHECK: $sgpr17 = V_READLANE
 # CHECK: $sgpr18 = V_READLANE
 # CHECK: $sgpr19 = V_READLANE
 # CHECK: $sgpr20 = V_READLANE
 # CHECK: $sgpr21 = V_READLANE
 # CHECK: $sgpr22 = V_READLANE
 # CHECK: $sgpr23 = V_READLANE
 # CHECK: $sgpr24 = V_READLANE
 # CHECK: $sgpr25 = V_READLANE
 # CHECK: $sgpr26 = V_READLANE
 # CHECK: $sgpr27 = V_READLANE
 
 # S1024
 # GCN32: $sgpr64 = S_MOV_B32 $exec_lo
 # GCN64: $sgpr64_sgpr65 = S_MOV_B64 $exec
 # GCN32: $exec_lo = S_MOV_B32 4294967295
 # GCN64: $exec = S_MOV_B64 4294967295
-# CHECK: BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 160
+# MUBUF:   BUFFER_LOAD_DWORD_OFFSET ${{(sgpr[0-9_]+)*}}, $sgpr33, 160
+# FLATSCR: SCRATCH_LOAD_DWORD_SADDR $sgpr33, 160
 # GCN32: $exec_lo = S_MOV_B32 killed $sgpr64
 # GCN64: $exec = S_MOV_B64 killed $sgpr64_sgpr65
 # CHECK: $sgpr64 = V_READLANE
 # CHECK: $sgpr65 = V_READLANE
 # CHECK: $sgpr66 = V_READLANE
 # CHECK: $sgpr67 = V_READLANE
 # CHECK: $sgpr68 = V_READLANE
 # CHECK: $sgpr69 = V_READLANE

llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll

 ; RUN: llc -march=amdgcn -mcpu=verde -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX6 %s
 ; RUN: llc -regalloc=basic -march=amdgcn -mcpu=tonga -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX7 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-enable-flat-scratch < %s | FileCheck -check-prefixes=CHECK,GFX9-FLATSCR,FLATSCR %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-enable-flat-scratch < %s | FileCheck -check-prefixes=CHECK,GFX10-FLATSCR,FLATSCR %s
 ;
 ; There is something about Tonga that causes this test to spend a lot of time
 ; in the default register allocator.
 
 
 ; When the offset of VGPR spills into scratch space gets too large, an additional SGPR
 ; is used to calculate the scratch load/store address. Make sure that this
 ; mechanism works even when many spills happen.
 
 ; Just test that it compiles successfully.
 ; CHECK-LABEL: test
+
+; GFX9-FLATSCR: s_mov_b32 [[SOFF1:s[0-9]+]], 0{{$}}
+; GFX9-FLATSCR: scratch_store_dword off, v{{[0-9]+}}, [[SOFF1]] offset:{{[0-9]+}} ; 4-byte Folded Spill
+; GFX9-FLATSCR: s_mov_b32 [[SOFF2:s[0-9]+]], 0{{$}}
+; GFX9-FLATSCR: scratch_load_dword v{{[0-9]+}}, off, [[SOFF2]] offset:{{[0-9]+}} ; 4-byte Folded Reload
+
+; GFX10-FLATSCR: scratch_store_dword off, v{{[0-9]+}}, off offset:{{[0-9]+}} ; 4-byte Folded Spill
+; GFX10-FLATSCR: scratch_load_dword v{{[0-9]+}}, off, off offset:{{[0-9]+}} ; 4-byte Folded Reload
 define amdgpu_kernel void @test(<1280 x i32> addrspace(1)* %out, <1280 x i32> addrspace(1)* %in) {
 entry:
   %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
   %tid = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
 
   %aptr = getelementptr <1280 x i32>, <1280 x i32> addrspace(1)* %in, i32 %tid
   %a = load <1280 x i32>, <1280 x i32> addrspace(1)* %aptr
 
 ; mark most VGPR registers as used to increase register pressure
   call void asm sideeffect "", "~{v4},~{v8},~{v12},~{v16},~{v20},~{v24},~{v28},~{v32}" ()
   call void asm sideeffect "", "~{v36},~{v40},~{v44},~{v48},~{v52},~{v56},~{v60},~{v64}" ()
   call void asm sideeffect "", "~{v68},~{v72},~{v76},~{v80},~{v84},~{v88},~{v92},~{v96}" ()
   call void asm sideeffect "", "~{v100},~{v104},~{v108},~{v112},~{v116},~{v120},~{v124},~{v128}" ()
   call void asm sideeffect "", "~{v132},~{v136},~{v140},~{v144},~{v148},~{v152},~{v156},~{v160}" ()
   call void asm sideeffect "", "~{v164},~{v168},~{v172},~{v176},~{v180},~{v184},~{v188},~{v192}" ()
   call void asm sideeffect "", "~{v196},~{v200},~{v204},~{v208},~{v212},~{v216},~{v220},~{v224}" ()
 
   %outptr = getelementptr <1280 x i32>, <1280 x i32> addrspace(1)* %out, i32 %tid
   store <1280 x i32> %a, <1280 x i32> addrspace(1)* %outptr
 
   ret void
 }
 
 ; CHECK-LABEL: test_limited_sgpr
-; GFX6: s_add_u32 s32, s32, 0x[[OFFSET:[0-9]+]]
+; GFX6: s_add_u32 s32, s32, 0x[[OFFSET:[0-9a-f]+]]
 ; GFX6-NEXT: buffer_load_dword v{{[0-9]+}}, off, s[{{[0-9:]+}}], s32
-; GFX6-NEXT: s_sub_u32 s32, s32, 0x[[OFFSET:[0-9]+]]
+; GFX6-NEXT: s_sub_u32 s32, s32, 0x[[OFFSET:[0-9a-f]+]]
 ; GFX6: NumSgprs: 48
 ; GFX6: ScratchSize: 8608
+
+; FLATSCR:           s_movk_i32 [[SOFF1:s[0-9]+]], 0x
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:      scratch_store_dword off, v{{[0-9]+}}, [[SOFF1]] ; 4-byte Folded Spill
+; FLATSCR:           s_movk_i32 [[SOFF2:s[0-9]+]], 0x
+; FLATSCR:           scratch_load_dword v{{[0-9]+}}, off, [[SOFF2]] ; 4-byte Folded Reload
 define amdgpu_kernel void @test_limited_sgpr(<64 x i32> addrspace(1)* %out, <64 x i32> addrspace(1)* %in) #0 {
 entry:
   %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
   %tid = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
 
 ; allocate enough scratch to go beyond 2^12 addressing
   %scratch = alloca <1280 x i32>, align 8, addrspace(5)