[AMDGPU] Implement -amdgpu-spill-cfi-saved-regs

[scott.linder]

Summary

These spills need special CFI anyway, so implementing them directly
where CFI is emitted avoids the need to invent a mechanism to track them
from ISel.

[scott.linder]

git clang-format

[arsenm]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

823

Avoid hardcoding by checking SIRegisterInfo::getReturnAddressReg?

1283

Register

1548

Register

1586

FIXME what?

[scott.linder]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

1586

The hard-coded SGPR size, I meant to come back and actually check relative to the SGPR.

[scott.linder]

Rebase and address feedback.

[scott.linder]

Rebase

[arsenm]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

855

You can just hardcode AMDGPU::sub0 instead of TRI.getSubRegFromChannel(0)

860

AMDGPU::sub1

[scott.linder]

Refer to subreg index directly

[RamNalamothu]

Rebase and fix position of spill for -amdgpu-spill-cfi-saved-regs

[RamNalamothu]

Rebase

[RamNalamothu]

Rebase onto LLVM master. Tidy up code little bit and couple of fixes.
And also, handle cases where no free VGPR is available to spill into and we have to spill into memeory.

[cameron.mcinally]

LGTM with moderate confidence. Does anyone see a reason that this shouldn't land?

[greened]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

143

Add a comment about why this is here. What is DwordOff about?

660

Needs a comment explaining what this does.

716

Needs a comment explaining what this does.

1232–1233

Can this be simplified by splitting into multiple if statements and commented? All of the chained logical operations are difficult to parse.

1283

Needs a comment.

1482

Needs a comment.

1546

Needs a comment.

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

428 ↗

(On Diff #290938)

Needs a comment.

[RamNalamothu]

Rebase onto LLVM master and address feedback

[RamNalamothu]

Rebase onto LLVM trunk and couple of bug fixes where we need to save FP due to CFI register spills but did not save it.

[RamNalamothu]

Rebase onto LLVM mainline top of the tree.

[RamNalamothu]

Incorporated all the downstream changes.

llvm/lib/Target/AMDGPU/SIFrameLowering.h

 //===--------------------- SIFrameLowering.h --------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H
 #define LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H
 
 #include "AMDGPUFrameLowering.h"
+#include "SIMachineFunctionInfo.h"
 
 namespace llvm {
 
 class SIInstrInfo;
-class SIMachineFunctionInfo;
 class SIRegisterInfo;
 class GCNSubtarget;
 
 class SIFrameLowering final : public AMDGPUFrameLowering {
 public:
   SIFrameLowering(StackDirection D, Align StackAl, int LAO,
                   Align TransAl = Align(1))
       : AMDGPUFrameLowering(D, StackAl, LAO, TransAl) {}
public:
   void buildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                 const DebugLoc &DL, const MCCFIInstruction &CFIInst) const;
   /// Create a CFI index describing a spill of an SGPR to a single lane of
   /// a VGPR and build a MachineInstr around it.
   void buildCFIForSGPRToVGPRSpill(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator MBBI,
                                   const DebugLoc &DL, const Register SGPR,
                                   const Register VGPR, const int Lane) const;
+  /// Create a CFI index describing a spill of an SGPR to multiple lanes of
+  /// VGPRs and build a MachineInstr around it.
+  void buildCFIForSGPRToVGPRSpill(
+      MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+      const DebugLoc &DL, Register SGPR,
+      ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills) const;
   /// Create a CFI index describing a spill of a VGPR to VMEM and
   /// build a MachineInstr around it.
   void buildCFIForVGPRToVMEMSpill(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator MBBI,
                                   const DebugLoc &DL, unsigned VGPR,
                                   int64_t Offset) const;
 };
 
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

 // use.
 static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator I,
                              const SIInstrInfo *TII, Register SpillReg,
                              Register ScratchRsrcReg, Register SPReg, int FI) {
   MachineFunction *MF = MBB.getParent();
   MachineFrameInfo &MFI = MF->getFrameInfo();
 
   int64_t Offset = MFI.getObjectOffset(FI);
 
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 4,
       MFI.getObjectAlign(FI));
 
   if (isUInt<12>(Offset)) {
     BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::BUFFER_STORE_DWORD_OFFSET))
       .addReg(SpillReg, RegState::Kill)
bool SIFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
   case TargetStackID::SGPRSpill:
     return true;
   case TargetStackID::SVEVector:
     return false;
   }
   llvm_unreachable("Invalid TargetStackID::Value");
 }
 
 // Activate all lanes, returns saved exec.
 static Register buildScratchExecCopy(LivePhysRegs &LiveRegs,
                                      MachineFunction &MF,
                                      MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator MBBI,
                                      bool IsProlog) {
   Register ScratchExecCopy;
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
void SIFrameLowering::emitPrologueEntryCFI(MachineBasicBlock &MBB,
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   const MCRegisterInfo *MCRI = MF.getMMI().getContext().getRegisterInfo();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo();
   Register StackPtrReg =
       MF.getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg();
 
   // DW_ASPACE_AMDGPU_private_wave FIXME: should be defined elsewhere
   buildCFI(MBB, MBBI, DL,
            MCCFIInstruction::createLLVMDefAspaceCfa(
                nullptr, MCRI->getDwarfRegNum(StackPtrReg, false), 0, 6));
 
   static const char PCEncodedInst[] = {
       dwarf::DW_CFA_expression,
       16, // PC 64
       8,  // length
       static_cast<char>(dwarf::DW_OP_regx),
auto ProcessReg = [&](MCPhysReg Reg) {
     if (IsCalleeSaved.test(Reg) || !MRI.isPhysRegModified(Reg))
       return;
     MCRegister DwarfReg = MCRI->getDwarfRegNum(Reg, false);
     buildCFI(MBB, MBBI, DL,
              MCCFIInstruction::createUndefined(nullptr, DwarfReg));
   };
   for_each(TRI.getAllVGPR32(MF), ProcessReg);
   for_each(TRI.getAllSGPR32(MF), ProcessReg);
-};
+}
 
 void SIFrameLowering::emitPrologue(MachineFunction &MF,
                                    MachineBasicBlock &MBB) const {
   SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
   if (FuncInfo->isEntryFunction()) {
     emitEntryFunctionPrologue(MF, MBB);
     return;
   }
void SIFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   // Emit the copy if we need a BP, and are using a free SGPR to save it.
   if (FuncInfo->SGPRForBPSaveRestoreCopy) {
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY),
             FuncInfo->SGPRForBPSaveRestoreCopy)
         .addReg(BasePtrReg)
         .setMIFlag(MachineInstr::FrameSetup);
     buildCFI(
         MBB, MBBI, DL,
         MCCFIInstruction::createRegister(
             nullptr, MCRI->getDwarfRegNum(BasePtrReg, false),
             MCRI->getDwarfRegNum(FuncInfo->SGPRForBPSaveRestoreCopy, false)));
   }
 
   // If a copy has been emitted for FP and/or BP, Make the SGPRs
   // used in the copy instructions live throughout the function.
void SIFrameLowering::emitPrologue(MachineFunction &MF,
 
   for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg
          : FuncInfo->getSGPRSpillVGPRs()) {
     if (!Reg.FI.hasValue())
       continue;
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
 
     int FI = Reg.FI.getValue();
 
     buildPrologSpill(LiveRegs, MBB, MBBI, TII, Reg.VGPR,
                      FuncInfo->getScratchRSrcReg(), StackPtrReg, FI);
 
     // We spill the entire VGPR, so we can get away with just cfi_offset
     buildCFI(MBB, MBBI, DL,
              MCCFIInstruction::createOffset(
                  nullptr, MCRI->getDwarfRegNum(Reg.VGPR, false),
                  MFI.getObjectOffset(FI) * ST.getWavefrontSize()));
   }
 
   if (HasFPSaveIndex && SpillFPToMemory) {
if (ScratchExecCopy) {
     // FIXME: Split block and make terminator.
     unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
     MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
     BuildMI(MBB, MBBI, DL, TII->get(ExecMov), Exec)
         .addReg(ScratchExecCopy, RegState::Kill);
     LiveRegs.addReg(ScratchExecCopy);
   }
 
+  if (TRI.isCFISavedRegsSpillEnabled()) {
+    MCRegister ReturnAddressReg = TRI.getReturnAddressReg(MF);
+    ArrayRef<SIMachineFunctionInfo::SpilledReg> ReturnAddressSpill =
+        FuncInfo->getSGPRToVGPRSpills(FuncInfo->ReturnAddressSaveIndex);
+    assert(ReturnAddressSpill.size() == 2);
+    BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+            ReturnAddressSpill[0].VGPR)
+        .addReg(TRI.getSubReg(ReturnAddressReg, TRI.getSubRegFromChannel(0)))
+        .addImm(ReturnAddressSpill[0].Lane)
+        .addReg(ReturnAddressSpill[0].VGPR, RegState::Undef);
+    BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+            ReturnAddressSpill[1].VGPR)
+        .addReg(TRI.getSubReg(ReturnAddressReg, TRI.getSubRegFromChannel(1)))
+        .addImm(ReturnAddressSpill[1].Lane)
+        .addReg(ReturnAddressSpill[1].VGPR, RegState::Undef);
+    buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, AMDGPU::PC_REG,
+                               ReturnAddressSpill);
+
+    ArrayRef<SIMachineFunctionInfo::SpilledReg> EXECSpill =
+        FuncInfo->getSGPRToVGPRSpills(FuncInfo->EXECSaveIndex);
+    assert(EXECSpill.size());
+    BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+            EXECSpill[0].VGPR)
+        .addReg(AMDGPU::EXEC_LO)
+        .addImm(EXECSpill[0].Lane)
+        .addReg(EXECSpill[0].VGPR, RegState::Undef);
+    if (!ST.isWave32()) {
+      assert(EXECSpill.size() == 2);
+      BuildMI(MBB, MBBI, DL,
+              TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+              EXECSpill[1].VGPR)
+          .addReg(AMDGPU::EXEC_HI)
+          .addImm(EXECSpill[1].Lane)
+          .addReg(EXECSpill[1].VGPR, RegState::Undef);
+    }
+    buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, AMDGPU::EXEC, EXECSpill);
+  }
+
   // In this case, spill the FP to a reserved VGPR.
   if (HasFPSaveIndex && !SpillFPToMemory) {
     const int FI = *FuncInfo->FramePointerSaveIndex;
     assert(!MFI.isDeadObjectIndex(FI));
 
     assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
     ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
         FuncInfo->getSGPRToVGPRSpills(FI);

 #ifndef NDEBUG
 static bool allSGPRSpillsAreDead(const MachineFrameInfo &MFI,
                                  Optional<int> FramePointerSaveIndex,
                                  Optional<int> BasePointerSaveIndex) {
   for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
        I != E; ++I) {
     if (!MFI.isDeadObjectIndex(I) &&
         MFI.getStackID(I) == TargetStackID::SGPRSpill &&
         ((FramePointerSaveIndex && I != FramePointerSaveIndex) ||
          (BasePointerSaveIndex && I != BasePointerSaveIndex))) {
       return false;
     }
   }
 
   return true;
 }
 #endif
 
if (FuncInfo->isEntryFunction()) {
         TRI->getSpillSize(AMDGPU::SGPR_32RegClass),
         TRI->getSpillAlignment(AMDGPU::SGPR_32RegClass),
         false);
       RS->addScavengingFrameIndex(ScavengeFI);
     }
   }
 }
 
+static void allocateCFISave(MachineFunction &MF, int &FI, Register Reg) {
+  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  const SIRegisterInfo *TRI = ST.getRegisterInfo();
+  const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+  int NewFI = MF.getFrameInfo().CreateStackObject(
+      TRI->getSpillSize(*RC), TRI->getSpillAlignment(*RC), true);
+  if (!MFI->allocateSGPRSpillToVGPR(MF, NewFI))
+    llvm_unreachable("allocate SGPR spill should have worked");
+  FI = NewFI;
+}
+
 // Only report VGPRs to generic code.
 void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
                                            BitVector &SavedVGPRs,
                                            RegScavenger *RS) const {
   TargetFrameLowering::determineCalleeSaves(MF, SavedVGPRs, RS);
   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   if (MFI->isEntryFunction())
     return;
const bool WillHaveFP =
       FrameInfo.hasCalls() &&
       (SavedVGPRs.any() || !allStackObjectsAreDead(FrameInfo));
 
   // VGPRs used for SGPR spilling need to be specially inserted in the prolog,
   // so don't allow the default insertion to handle them.
   for (auto SSpill : MFI->getSGPRSpillVGPRs())
     SavedVGPRs.reset(SSpill.VGPR);
 
+  if (TRI->isCFISavedRegsSpillEnabled()) {
+    allocateCFISave(MF, MFI->ReturnAddressSaveIndex,
+                    TRI->getReturnAddressReg(MF));
+    allocateCFISave(MF, MFI->EXECSaveIndex,
+                    ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC);
+  }
+
   LivePhysRegs LiveRegs;
   LiveRegs.init(*TRI);
 
   if (WillHaveFP || hasFP(MF)) {
     getVGPRSpillLaneOrTempRegister(MF, LiveRegs, MFI->SGPRForFPSaveRestoreCopy,
                                    MFI->FramePointerSaveIndex, true);
   }
 
bool SIFrameLowering::hasFP(const MachineFunction &MF) const {
   }
 
   return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
     MFI.hasStackMap() || MFI.hasPatchPoint() ||
     MF.getSubtarget<GCNSubtarget>().getRegisterInfo()->needsStackRealignment(MF) ||
     MF.getTarget().Options.DisableFramePointerElim(MF);
 }
 
 void SIFrameLowering::buildCFI(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator MBBI,
                                const DebugLoc &DL,
                                const MCCFIInstruction &CFIInst) const {
   MachineFunction &MF = *MBB.getParent();
   const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
   BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
       .addCFIIndex(MF.addFrameInst(CFIInst))
       .setMIFlag(MachineInstr::FrameSetup);
void SIFrameLowering::buildCFIForSGPRToVGPRSpill(
 
   encodeULEB128(Block.size(), OSCFIInst);
   OSCFIInst << Block;
 
   buildCFI(MBB, MBBI, DL,
            MCCFIInstruction::createEscape(nullptr, OSCFIInst.str()));
 }
 
+void SIFrameLowering::buildCFIForSGPRToVGPRSpill(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    const DebugLoc &DL, Register SGPR,
+    ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills) const {
+  MachineFunction &MF = *MBB.getParent();
+  const MCRegisterInfo &MCRI = *MF.getMMI().getContext().getRegisterInfo();
+  int DwarfSGPR = MCRI.getDwarfRegNum(SGPR, false);
+
+  // CFI for an SGPR spilled to a multiple lanes of VGPRs is implemented as an
+  // expression(E) rule where E is a composite location description
+  // with multiple parts each referencing
+  // VGPR register location storage with a bit offset of the lane index
+  // multiplied by the size of an SGPR (32 bits). In other words we generate
+  // the following DWARF:
+  //
+  // DW_CFA_expression: <SGPR>,
+  //    (DW_OP_regx <VGPR[0]>) (DW_OP_bit_piece 32, <Lane[0]>*32)
+  //    (DW_OP_regx <VGPR[1]>) (DW_OP_bit_piece 32, <Lane[1]>*32)
+  //    ...
+  //    (DW_OP_regx <VGPR[N]>) (DW_OP_bit_piece 32, <Lane[N]>*32)
+  //
+  // The memory location description for the current CFA is pushed on the
+  // stack before E is evaluated, but we choose not to drop it as it would
+  // require a longer expression E and DWARF defines the result of the
+  // evaulation to be the location description on the top of the stack (i.e. the
+  // implictly pushed one is just ignored.)
+  SmallString<20> CFIInst;
+  raw_svector_ostream OSCFIInst(CFIInst);
+  SmallString<20> Block;
+  raw_svector_ostream OSBlock(Block);
+
+  OSCFIInst << uint8_t(dwarf::DW_CFA_expression);
+  encodeULEB128(DwarfSGPR, OSCFIInst);
+
+  // TODO: Detect when we can merge multiple adjacent pieces, or even reduce
+  // this to a register location description (when all pieces are adjacent).
+  for (SIMachineFunctionInfo::SpilledReg Spill : VGPRSpills) {
+    encodeDwarfRegisterLocation(MCRI.getDwarfRegNum(Spill.VGPR, false),
+                                OSBlock);
+    OSBlock << uint8_t(dwarf::DW_OP_bit_piece);
+    // FIXME: Can this be a function of the SGPR?
+    const unsigned SGPRBitSize = 32;
+    encodeULEB128(SGPRBitSize, OSBlock);
+    encodeULEB128(SGPRBitSize * Spill.Lane, OSBlock);
+  }
+
+  encodeULEB128(Block.size(), OSCFIInst);
+  OSCFIInst << Block;
+
+  buildCFI(MBB, MBBI, DL,
+           MCCFIInstruction::createEscape(nullptr, OSCFIInst.str()));
+}
+
 void SIFrameLowering::buildCFIForVGPRToVMEMSpill(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     const DebugLoc &DL, unsigned VGPR, int64_t Offset) const {
   MachineFunction &MF = *MBB.getParent();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const MCRegisterInfo &MCRI = *MF.getMMI().getContext().getRegisterInfo();
   int DwarfVGPR = MCRI.getDwarfRegNum(VGPR, false);
 

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h

public: // FIXME
   Register SGPRForFPSaveRestoreCopy;
   Optional<int> FramePointerSaveIndex;
 
   /// If this is set, an SGPR used for save/restore of the register used for the
   /// base pointer.
   Register SGPRForBPSaveRestoreCopy;
   Optional<int> BasePointerSaveIndex;
 
+  int ReturnAddressSaveIndex;
+  int EXECSaveIndex;
+
   Register VGPRReservedForSGPRSpill;
   bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg);
 
 public:
   SIMachineFunctionInfo(const MachineFunction &MF);
 
   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI);
 

llvm/lib/Target/AMDGPU/SIRegisterInfo.h

public:
   /// \returns the sub reg enum value for the given \p Channel
   /// (e.g. getSubRegFromChannel(0) -> AMDGPU::sub0)
   static unsigned getSubRegFromChannel(unsigned Channel, unsigned NumRegs = 1);
 
   bool spillSGPRToVGPR() const {
     return SpillSGPRToVGPR;
   }
 
+  bool isCFISavedRegsSpillEnabled() const;
+
   /// Return the end register initially reserved for the scratch buffer in case
   /// spilling is needed.
   MCRegister reservedPrivateSegmentBufferReg(const MachineFunction &MF) const;
 
   BitVector getReservedRegs(const MachineFunction &MF) const override;
 
   const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
   const MCPhysReg *getCalleeSavedRegsViaCopy(const MachineFunction *MF) const;

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

 static cl::opt<bool> EnableSpillSGPRToVGPR(
   "amdgpu-spill-sgpr-to-vgpr",
   cl::desc("Enable spilling VGPRs to SGPRs"),
   cl::ReallyHidden,
   cl::init(true));
 
 std::array<std::vector<int16_t>, 16> SIRegisterInfo::RegSplitParts;
 
+static cl::opt<bool> EnableSpillCFISavedRegs(
+    "amdgpu-spill-cfi-saved-regs",
+    cl::desc("Enable spilling the registers required for CFI emission"),
+    cl::ReallyHidden, cl::init(false));
+
 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST)
     : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST),
       SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) {
 
   assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 &&
          getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) &&
          (getSubRegIndexLaneMask(AMDGPU::lo16) |
           getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() ==
unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel,
   const unsigned NumRegIndex = NumRegs - 1;
 
   assert(NumRegIndex < array_lengthof(SubRegFromChannelTable) &&
          "Not implemented");
   assert(Channel < array_lengthof(SubRegFromChannelTable[0]));
   return SubRegFromChannelTable[NumRegIndex][Channel];
 }
 
+bool SIRegisterInfo::isCFISavedRegsSpillEnabled() const {
+  return EnableSpillCFISavedRegs;
+}
+
 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg(
   const MachineFunction &MF) const {
   unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4;
   MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
   return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass);
 }
 
 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {

llvm/test/CodeGen/AMDGPU/amdgpu-spill-cfi-saved-regs.ll

+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=asm -amdgpu-spill-cfi-saved-regs -o - %s | FileCheck --check-prefixes=CHECK,WAVE64 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -filetype=asm -amdgpu-spill-cfi-saved-regs -o - %s | FileCheck --check-prefixes=CHECK,WAVE32 %s
+
+; CHECK-LABEL: kern:
+; CHECK: .cfi_startproc
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: %bb.0:
+; CHECK-NEXT: .cfi_escape 0x0f, 0x03, 0x30, 0x36, 0xe1
+; CHECK-NEXT: .cfi_undefined 16
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: .cfi_endproc
+define protected amdgpu_kernel void @kern() #0 {
+entry:
+  ret void
+}
+
+; CHECK-LABEL: func:
+; CHECK: .cfi_startproc
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: %bb.0:
+; SGPR32 = 64
+; CHECK-NEXT: .cfi_llvm_def_aspace_cfa 64, 0, 6
+; CHECK-NEXT: .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
+
+
+; FIXME: ideally this would not care what VGPR we spill to, but since we are
+; using .cfi_escape it isn't trivial/possible to make this general yet
+
+; CHECK: v_writelane_b32 v0, s30, 0
+; CHECK-NEXT: v_writelane_b32 v0, s31, 1
+
+; DW_CFA_expression [0x10]
+;   PC_64 ULEB128(17)=[0x10]
+;   BLOCK_LENGTH ULEB128(12)=[0x0c]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x00]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x20]
+; WAVE64-NEXT: .cfi_escape 0x10, 0x10, 0x0c, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x00, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x20
+
+; DW_CFA_expression [0x10]
+;   PC_64 ULEB128(17)=[0x10]
+;   BLOCK_LENGTH ULEB128(12)=[0x0c]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave32 ULEB128(1536)=[0x80, 0x0c]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x00]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave32 ULEB128(1536)=[0x80, 0x0c]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x20]
+; WAVE32-NEXT: .cfi_escape 0x10, 0x10, 0x0c, 0x90, 0x80, 0x0c, 0x9d, 0x20, 0x00, 0x90, 0x80, 0x0c, 0x9d, 0x20, 0x20
+
+
+; WAVE64: v_writelane_b32 v0, exec_lo, 2
+; WAVE64-NEXT: v_writelane_b32 v0, exec_hi, 3
+; DW_CFA_expression [0x10]
+;   EXEC_MASK_wave64 ULEB128(17)=[0x11]
+;   BLOCK_LENGTH ULEB128(12)=[0x0c]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x40]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x60]
+; WAVE64-NEXT: .cfi_escape 0x10, 0x11, 0x0c, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x40, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x60
+
+; WAVE32: v_writelane_b32 v0, exec_lo, 2
+; DW_CFA_expression [0x10]
+;   EXEC_MASK_wave32 ULEB128(1)=[0x01]
+;   BLOCK_LENGTH ULEB128(6)=[0x06]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave32 ULEB128(1536)=[0x80, 0x0c]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x40]
+; WAVE32-NEXT: .cfi_escape 0x10, 0x01, 0x06, 0x90, 0x80, 0x0c, 0x9d, 0x20, 0x40
+
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: .cfi_endproc
+define hidden void @func() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "filename", directory: "directory")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}