[AMDGPU] Implement -amdgpu-spill-cfi-saved-regs

[scott.linder]

Summary

These spills need special CFI anyway, so implementing them directly
where CFI is emitted avoids the need to invent a mechanism to track them
from ISel.

[scott.linder]

git clang-format

[arsenm]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

714

Avoid hardcoding by checking SIRegisterInfo::getReturnAddressReg?

1038

Register

1313

Register

1351

FIXME what?

[scott.linder]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

1351

The hard-coded SGPR size, I meant to come back and actually check relative to the SGPR.

[scott.linder]

Rebase and address feedback.

[scott.linder]

Rebase

[arsenm]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

715

You can just hardcode AMDGPU::sub0 instead of TRI.getSubRegFromChannel(0)

720

AMDGPU::sub1

[scott.linder]

Refer to subreg index directly

[RamNalamothu]

Rebase and fix position of spill for -amdgpu-spill-cfi-saved-regs

[RamNalamothu]

Rebase

[RamNalamothu]

Rebase onto LLVM master. Tidy up code little bit and couple of fixes.
And also, handle cases where no free VGPR is available to spill into and we have to spill into memeory.

[cameron.mcinally]

LGTM with moderate confidence. Does anyone see a reason that this shouldn't land?

[greened]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

97

Add a comment about why this is here. What is DwordOff about?

607

Needs a comment explaining what this does.

619

Needs a comment explaining what this does.

988

Can this be simplified by splitting into multiple if statements and commented? All of the chained logical operations are difficult to parse.

1038

Needs a comment.

1247

Needs a comment.

1311

Needs a comment.

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

428 ↗

(On Diff #290938)

Needs a comment.

[RamNalamothu]

Rebase onto LLVM master and address feedback

[RamNalamothu]

Rebase onto LLVM trunk and couple of bug fixes where we need to save FP due to CFI register spills but did not save it.

[RamNalamothu]

Rebase onto LLVM mainline top of the tree.

[RamNalamothu]

Incorporated all the downstream changes.

llvm/lib/Target/AMDGPU/SIFrameLowering.h

 //===--------------------- SIFrameLowering.h --------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H
 #define LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H
 
 #include "AMDGPUFrameLowering.h"
+#include "SIMachineFunctionInfo.h"
 
 namespace llvm {
 
 class SIInstrInfo;
-class SIMachineFunctionInfo;
 class SIRegisterInfo;
 class GCNSubtarget;
 
 class SIFrameLowering final : public AMDGPUFrameLowering {
 public:
   SIFrameLowering(StackDirection D, Align StackAl, int LAO,
                   Align TransAl = Align(1))
       : AMDGPUFrameLowering(D, StackAl, LAO, TransAl) {}
public:
   void buildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                 const DebugLoc &DL, const MCCFIInstruction &CFIInst) const;
   /// Create a CFI index describing a spill of an SGPR to a single lane of
   /// a VGPR and build a MachineInstr around it.
   void buildCFIForSGPRToVGPRSpill(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator MBBI,
                                   const DebugLoc &DL, const Register SGPR,
                                   const Register VGPR, const int Lane) const;
+  /// Create a CFI index describing a spill of an SGPR to multiple lanes of
+  /// VGPRs and build a MachineInstr around it.
+  void buildCFIForSGPRToVGPRSpill(
+      MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+      const DebugLoc &DL, Register SGPR,
+      ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills) const;
   /// Create a CFI index describing a spill of a VGPR to VMEM and
   /// build a MachineInstr around it.
   void buildCFIForVGPRToVMEMSpill(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator MBBI,
                                   const DebugLoc &DL, unsigned VGPR,
                                   int64_t Offset) const;
 };
 
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

 // use.
 static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator I,
                              const SIInstrInfo *TII, Register SpillReg,
                              Register ScratchRsrcReg, Register SPReg, int FI) {
   MachineFunction *MF = MBB.getParent();
   MachineFrameInfo &MFI = MF->getFrameInfo();
 
   int64_t Offset = MFI.getObjectOffset(FI);
 
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 4,
       MFI.getObjectAlign(FI));
 
   if (isUInt<12>(Offset)) {
     BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::BUFFER_STORE_DWORD_OFFSET))
       .addReg(SpillReg, RegState::Kill)
bool SIFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
   case TargetStackID::SGPRSpill:
     return true;
   case TargetStackID::SVEVector:
     return false;
   }
   llvm_unreachable("Invalid TargetStackID::Value");
 }
 
 void SIFrameLowering::emitPrologueEntryCFI(MachineBasicBlock &MBB,
                                            MachineBasicBlock::iterator MBBI,
                                            const DebugLoc &DL) const {
   const MachineFunction &MF = *MBB.getParent();
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   const MCRegisterInfo *MCRI = MF.getMMI().getContext().getRegisterInfo();
   Register StackPtrReg =
       MF.getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg();
 
   // DW_ASPACE_AMDGPU_private_wave FIXME: should be defined elsewhere
   buildCFI(MBB, MBBI, DL,
            MCCFIInstruction::createLLVMDefAspaceCfa(
                nullptr, MCRI->getDwarfRegNum(StackPtrReg, false), 0, 6));
 
   static const char PCEncodedInst[] = {
       dwarf::DW_CFA_expression,
       16, // PC 64
       8,  // length
       static_cast<char>(dwarf::DW_OP_regx),
       62, // SGPR30
       static_cast<char>(dwarf::DW_OP_piece),
BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FuncInfo->SGPRForFPSaveRestoreCopy)
       .setMIFlag(MachineInstr::FrameSetup);
     buildCFI(
         MBB, MBBI, DL,
         MCCFIInstruction::createRegister(
             nullptr, MCRI->getDwarfRegNum(FramePtrReg, false),
             MCRI->getDwarfRegNum(FuncInfo->SGPRForFPSaveRestoreCopy, false)));
   }
 
+  if (TRI.isCFISavedRegsSpillEnabled()) {
+    MCRegister ReturnAddressReg = TRI.getReturnAddressReg(MF);
+    ArrayRef<SIMachineFunctionInfo::SpilledReg> ReturnAddressSpill =
+        FuncInfo->getSGPRToVGPRSpills(FuncInfo->ReturnAddressSaveIndex);
+    assert(ReturnAddressSpill.size() == 2);
+    BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+            ReturnAddressSpill[0].VGPR)
+        .addReg(TRI.getSubReg(ReturnAddressReg, TRI.getSubRegFromChannel(0)))
+        .addImm(ReturnAddressSpill[0].Lane)
+        .addReg(ReturnAddressSpill[0].VGPR, RegState::Undef);
+    BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+            ReturnAddressSpill[1].VGPR)
+        .addReg(TRI.getSubReg(ReturnAddressReg, TRI.getSubRegFromChannel(1)))
+        .addImm(ReturnAddressSpill[1].Lane)
+        .addReg(ReturnAddressSpill[1].VGPR, RegState::Undef);
+    buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, AMDGPU::PC_REG,
+                               ReturnAddressSpill);
+
+    ArrayRef<SIMachineFunctionInfo::SpilledReg> EXECSpill =
+        FuncInfo->getSGPRToVGPRSpills(FuncInfo->EXECSaveIndex);
+    assert(EXECSpill.size());
+    BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+            EXECSpill[0].VGPR)
+        .addReg(AMDGPU::EXEC_LO)
+        .addImm(EXECSpill[0].Lane)
+        .addReg(EXECSpill[0].VGPR, RegState::Undef);
+    if (!ST.isWave32()) {
+      assert(EXECSpill.size() == 2);
+      BuildMI(MBB, MBBI, DL,
+              TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
+              EXECSpill[1].VGPR)
+          .addReg(AMDGPU::EXEC_HI)
+          .addImm(EXECSpill[1].Lane)
+          .addReg(EXECSpill[1].VGPR, RegState::Undef);
+    }
+    buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, AMDGPU::EXEC, EXECSpill);
+  }
+
   for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg
          : FuncInfo->getSGPRSpillVGPRs()) {
     if (!Reg.FI.hasValue())
       continue;
 
     if (!ScratchExecCopy) {
       if (LiveRegs.empty()) {
         LiveRegs.init(TRI);

 
 #ifndef NDEBUG
 static bool allSGPRSpillsAreDead(const MachineFrameInfo &MFI,
                                  Optional<int> FramePointerSaveIndex) {
   for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
        I != E; ++I) {
     if (!MFI.isDeadObjectIndex(I) &&
         MFI.getStackID(I) == TargetStackID::SGPRSpill &&
         FramePointerSaveIndex && I != FramePointerSaveIndex) {
       return false;
     }
   }
 
   return true;
 }
 #endif
 
if (FuncInfo->isEntryFunction()) {
         TRI->getSpillSize(AMDGPU::SGPR_32RegClass),
         TRI->getSpillAlignment(AMDGPU::SGPR_32RegClass),
         false);
       RS->addScavengingFrameIndex(ScavengeFI);
     }
   }
 }
 
+static void allocateCFISave(MachineFunction &MF, int &FI, Register Reg) {
+  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  const SIRegisterInfo *TRI = ST.getRegisterInfo();
+  const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+  int NewFI = MF.getFrameInfo().CreateStackObject(
+      TRI->getSpillSize(*RC), TRI->getSpillAlignment(*RC), true);
+  if (!MFI->allocateSGPRSpillToVGPR(MF, NewFI))
+    llvm_unreachable("allocate SGPR spill should have worked");
+  FI = NewFI;
+}
+
 // Only report VGPRs to generic code.
 void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
                                            BitVector &SavedVGPRs,
                                            RegScavenger *RS) const {
   TargetFrameLowering::determineCalleeSaves(MF, SavedVGPRs, RS);
   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   if (MFI->isEntryFunction())
     return;
const bool WillHaveFP =
       FrameInfo.hasCalls() &&
       (SavedVGPRs.any() || !allStackObjectsAreDead(FrameInfo));
 
   // VGPRs used for SGPR spilling need to be specially inserted in the prolog,
   // so don't allow the default insertion to handle them.
   for (auto SSpill : MFI->getSGPRSpillVGPRs())
     SavedVGPRs.reset(SSpill.VGPR);
 
+  if (TRI->isCFISavedRegsSpillEnabled()) {
+    allocateCFISave(MF, MFI->ReturnAddressSaveIndex,
+                    TRI->getReturnAddressReg(MF));
+    allocateCFISave(MF, MFI->EXECSaveIndex,
+                    ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC);
+  }
+
   const bool HasFP = WillHaveFP || hasFP(MF);
   if (!HasFP)
     return;
 
   if (MFI->haveFreeLanesForSGPRSpill(MF, 1)) {
     int NewFI = MF.getFrameInfo().CreateStackObject(4, 4, true, nullptr,
                                                     TargetStackID::SGPRSpill);
 
bool SIFrameLowering::hasFP(const MachineFunction &MF) const {
   }
 
   return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
     MFI.hasStackMap() || MFI.hasPatchPoint() ||
     MF.getSubtarget<GCNSubtarget>().getRegisterInfo()->needsStackRealignment(MF) ||
     MF.getTarget().Options.DisableFramePointerElim(MF);
 }
 
 void SIFrameLowering::buildCFI(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator MBBI,
                                const DebugLoc &DL,
                                const MCCFIInstruction &CFIInst) const {
   MachineFunction &MF = *MBB.getParent();
   const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
   BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
       .addCFIIndex(MF.addFrameInst(CFIInst))
       .setMIFlag(MachineInstr::FrameSetup);
void SIFrameLowering::buildCFIForSGPRToVGPRSpill(
 
   encodeULEB128(Block.size(), OSCFIInst);
   OSCFIInst << Block;
 
   buildCFI(MBB, MBBI, DL,
            MCCFIInstruction::createEscape(nullptr, OSCFIInst.str()));
 }
 
+void SIFrameLowering::buildCFIForSGPRToVGPRSpill(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    const DebugLoc &DL, Register SGPR,
+    ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills) const {
+  MachineFunction &MF = *MBB.getParent();
+  const MCRegisterInfo &MCRI = *MF.getMMI().getContext().getRegisterInfo();
+  int DwarfSGPR = MCRI.getDwarfRegNum(SGPR, false);
+
+  // CFI for an SGPR spilled to a multiple lanes of VGPRs is implemented as an
+  // expression(E) rule where E is a composite location description
+  // with multiple parts each referencing
+  // VGPR register location storage with a bit offset of the lane index
+  // multiplied by the size of an SGPR (32 bits). In other words we generate
+  // the following DWARF:
+  //
+  // DW_CFA_expression: <SGPR>,
+  //    (DW_OP_regx <VGPR[0]>) (DW_OP_bit_piece 32, <Lane[0]>*32)
+  //    (DW_OP_regx <VGPR[1]>) (DW_OP_bit_piece 32, <Lane[1]>*32)
+  //    ...
+  //    (DW_OP_regx <VGPR[N]>) (DW_OP_bit_piece 32, <Lane[N]>*32)
+  //
+  // The memory location description for the current CFA is pushed on the
+  // stack before E is evaluated, but we choose not to drop it as it would
+  // require a longer expression E and DWARF defines the result of the
+  // evaulation to be the location description on the top of the stack (i.e. the
+  // implictly pushed one is just ignored.)
+  SmallString<20> CFIInst;
+  raw_svector_ostream OSCFIInst(CFIInst);
+  SmallString<20> Block;
+  raw_svector_ostream OSBlock(Block);
+
+  OSCFIInst << uint8_t(dwarf::DW_CFA_expression);
+  encodeULEB128(DwarfSGPR, OSCFIInst);
+
+  // TODO: Detect when we can merge multiple adjacent pieces, or even reduce
+  // this to a register location description (when all pieces are adjacent).
+  for (SIMachineFunctionInfo::SpilledReg Spill : VGPRSpills) {
+    encodeDwarfRegisterLocation(MCRI.getDwarfRegNum(Spill.VGPR, false),
+                                OSBlock);
+    OSBlock << uint8_t(dwarf::DW_OP_bit_piece);
+    // FIXME:
+    const unsigned SGPRBitSize = 32;
+    encodeULEB128(SGPRBitSize, OSBlock);
+    encodeULEB128(SGPRBitSize * Spill.Lane, OSBlock);
+  }
+
+  encodeULEB128(Block.size(), OSCFIInst);
+  OSCFIInst << Block;
+
+  buildCFI(MBB, MBBI, DL,
+           MCCFIInstruction::createEscape(nullptr, OSCFIInst.str()));
+}
+
 void SIFrameLowering::buildCFIForVGPRToVMEMSpill(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     const DebugLoc &DL, unsigned VGPR, int64_t Offset) const {
   MachineFunction &MF = *MBB.getParent();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const MCRegisterInfo &MCRI = *MF.getMMI().getContext().getRegisterInfo();
   int DwarfVGPR = MCRI.getDwarfRegNum(VGPR, false);
 

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h

private:
   SmallVector<MCPhysReg, 32> SpillVGPR;
 
 public: // FIXME
   /// If this is set, an SGPR used for save/restore of the register used for the
   /// frame pointer.
   Register SGPRForFPSaveRestoreCopy;
   Optional<int> FramePointerSaveIndex;
 
+  int ReturnAddressSaveIndex;
+  int EXECSaveIndex;
+
 public:
   SIMachineFunctionInfo(const MachineFunction &MF);
 
   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI);
 
   ArrayRef<SpilledReg> getSGPRToVGPRSpills(int FrameIndex) const {
     auto I = SGPRToVGPRSpills.find(FrameIndex);
     return (I == SGPRToVGPRSpills.end()) ?

llvm/lib/Target/AMDGPU/SIRegisterInfo.h

public:
   /// \returns the sub reg enum value for the given \p Channel
   /// (e.g. getSubRegFromChannel(0) -> AMDGPU::sub0)
   static unsigned getSubRegFromChannel(unsigned Channel, unsigned NumRegs = 1);
 
   bool spillSGPRToVGPR() const {
     return SpillSGPRToVGPR;
   }
 
+  bool isCFISavedRegsSpillEnabled() const;
+
   /// Return the end register initially reserved for the scratch buffer in case
   /// spilling is needed.
   MCRegister reservedPrivateSegmentBufferReg(const MachineFunction &MF) const;
 
   BitVector getReservedRegs(const MachineFunction &MF) const override;
 
   const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
   const MCPhysReg *getCalleeSavedRegsViaCopy(const MachineFunction *MF) const;

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

 #include "AMDGPUGenRegisterInfo.inc"
 
 static cl::opt<bool> EnableSpillSGPRToVGPR(
   "amdgpu-spill-sgpr-to-vgpr",
   cl::desc("Enable spilling VGPRs to SGPRs"),
   cl::ReallyHidden,
   cl::init(true));
 
+static cl::opt<bool> EnableSpillCFISavedRegs(
+    "amdgpu-spill-cfi-saved-regs",
+    cl::desc("Enable spilling the registers required for CFI emission"),
+    cl::ReallyHidden, cl::init(false));
+
 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST)
     : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST),
       SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) {
 
   assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 &&
          getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) &&
          (getSubRegIndexLaneMask(AMDGPU::lo16) |
           getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() ==
unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel,
   const unsigned NumRegIndex = NumRegs - 1;
 
   assert(NumRegIndex < array_lengthof(SubRegFromChannelTable) &&
          "Not implemented");
   assert(Channel < array_lengthof(SubRegFromChannelTable[0]));
   return SubRegFromChannelTable[NumRegIndex][Channel];
 }
 
+bool SIRegisterInfo::isCFISavedRegsSpillEnabled() const {
+  return EnableSpillCFISavedRegs;
+}
+
 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg(
   const MachineFunction &MF) const {
   unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4;
   MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
   return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass);
 }
 
 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {

llvm/test/CodeGen/AMDGPU/amdgpu-spill-cfi-saved-regs.ll

+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=asm -amdgpu-spill-cfi-saved-regs -o - %s | FileCheck --check-prefixes=CHECK,WAVE64 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -filetype=asm -amdgpu-spill-cfi-saved-regs -o - %s | FileCheck --check-prefixes=CHECK,WAVE32 %s
+
+; CHECK-LABEL: kern:
+; CHECK: .cfi_startproc
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: %bb.0:
+; CHECK-NEXT: .cfi_escape 0x0f, 0x03, 0x30, 0x36, 0xe1
+; CHECK-NEXT: .cfi_undefined 16
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: .cfi_endproc
+define protected amdgpu_kernel void @kern() #0 {
+entry:
+  ret void
+}
+
+; CHECK-LABEL: func:
+; CHECK: .cfi_startproc
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: %bb.0:
+; SGPR32 = 64
+; CHECK-NEXT: .cfi_llvm_def_aspace_cfa 64, 0, 6
+; CHECK-NEXT: .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
+
+
+; FIXME: ideally this would not care what VGPR we spill to, but since we are
+; using .cfi_escape it isn't trivial/possible to make this general yet
+
+; CHECK: v_writelane_b32 v0, s30, 0
+; CHECK-NEXT: v_writelane_b32 v0, s31, 1
+
+; DW_CFA_expression [0x10]
+;   PC_64 ULEB128(17)=[0x10]
+;   BLOCK_LENGTH ULEB128(12)=[0x0c]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x00]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x20]
+; WAVE64-NEXT: .cfi_escape 0x10, 0x10, 0x0c, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x00, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x20
+
+; DW_CFA_expression [0x10]
+;   PC_64 ULEB128(17)=[0x10]
+;   BLOCK_LENGTH ULEB128(12)=[0x0c]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave32 ULEB128(1536)=[0x80, 0x0c]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x00]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave32 ULEB128(1536)=[0x80, 0x0c]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x20]
+; WAVE32-NEXT: .cfi_escape 0x10, 0x10, 0x0c, 0x90, 0x80, 0x0c, 0x9d, 0x20, 0x00, 0x90, 0x80, 0x0c, 0x9d, 0x20, 0x20
+
+
+; WAVE64: v_writelane_b32 v0, exec_lo, 2
+; WAVE64-NEXT: v_writelane_b32 v0, exec_hi, 3
+; DW_CFA_expression [0x10]
+;   EXEC_MASK_wave64 ULEB128(17)=[0x11]
+;   BLOCK_LENGTH ULEB128(12)=[0x0c]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x40]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave64 ULEB128(2560)=[0x80, 0x14]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x60]
+; WAVE64-NEXT: .cfi_escape 0x10, 0x11, 0x0c, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x40, 0x90, 0x80, 0x14, 0x9d, 0x20, 0x60
+
+; WAVE32: v_writelane_b32 v0, exec_lo, 2
+; DW_CFA_expression [0x10]
+;   EXEC_MASK_wave32 ULEB128(1)=[0x01]
+;   BLOCK_LENGTH ULEB128(6)=[0x06]
+;     DW_OP_regx [0x90]
+;       VGPR0_wave32 ULEB128(1536)=[0x80, 0x0c]
+;     DW_OP_bit_piece [0x9d]
+;       PIECE_SIZE [0x20]
+;       PIECE_OFFSET [0x40]
+; WAVE32-NEXT: .cfi_escape 0x10, 0x01, 0x06, 0x90, 0x80, 0x0c, 0x9d, 0x20, 0x40
+
+; CHECK-NOT: .cfi_{{.*}}
+; CHECK: .cfi_endproc
+define hidden void @func() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "filename", directory: "directory")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}