[AMDGPU] Implement CFI for non-kernel functions

[scott.linder]

Summary

This does not implement CSR spills other than those AMDGPU handles
during PEI. The remaining spills are handled in a subsequent patch.

[scott.linder]

git clang-format

[arsenm]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

630

Register

643

Double cast

656

MCPhysReg?

917

Register

llvm/lib/Target/AMDGPU/SIFrameLowering.h

87

s/unsigned/Register

[scott.linder]

Rebase and address feedback.

[scott.linder]

Rebase, generalize the previously hard-coded list of caller-saved registers
with help from @cdevadas, and emit CFI for the new case where we spill the FP
to VMEM.

[cdevadas]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

42

I think it is good to move getAllSGPRs and getAllVGPRs into SIRegisterInfo.cpp.
There is another function getAllVGPR32 which does the same thing as getAllVGPRs.
We can unify these instances.

[scott.linder]

Rebase over https://reviews.llvm.org/D79878

[cdevadas]

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

1291

Can you write a small description of why the FIXME?

[RamNalamothu]

Rebase and emit CFI for Base pointer

[RamNalamothu]

Rebase

[RamNalamothu]

Rebase onto LLVM master

[greened]

Not my area of expertise but looks fine if the comments from others are addressed.

[RamNalamothu]

Rebase onto LLVM master and address feedback

[RamNalamothu]

Rebase onto LLVM trunk.

[RamNalamothu]

Rebase onto LLVM top of the tree.

[RamNalamothu]

Incorporated all the downstream changes.

[arsenm]

LGTM. If there are remaining issues it's easier to fix later rather than continuing the routine pain of carrying this out of tree

[scott.linder]

Abandoned in favor of D147279 which avoids using most/all of the remaining user opcode

llvm/lib/Target/AMDGPU/SIFrameLowering.h

private:
   Register getEntryFunctionReservedScratchRsrcReg(MachineFunction &MF) const;
 
   void emitEntryFunctionScratchRsrcRegSetup(
       MachineFunction &MF, MachineBasicBlock &MBB,
       MachineBasicBlock::iterator I, const DebugLoc &DL,
       Register PreloadedPrivateBufferReg, Register ScratchRsrcReg,
       Register ScratchWaveOffsetReg) const;
 
+  void emitPrologueEntryCFI(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MBBI,
+                            const DebugLoc &DL) const;
+
 public:
   bool hasFP(const MachineFunction &MF) const override;
 
   /// Create a CFI index for CFIInst and build a MachineInstr around it.
   void buildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                 const DebugLoc &DL, const MCCFIInstruction &CFIInst) const;
+  /// Create a CFI index describing a spill of an SGPR to a single lane of
+  /// a VGPR and build a MachineInstr around it.
+  void buildCFIForSGPRToVGPRSpill(MachineBasicBlock &MBB,
+                                  MachineBasicBlock::iterator MBBI,
+                                  const DebugLoc &DL, const Register SGPR,
+                                  const Register VGPR, const int Lane) const;
 };
 
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H

llvm/lib/Target/AMDGPU/SIFrameLowering.cpp

 //===----------------------- SIFrameLowering.cpp --------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 
 #include "SIFrameLowering.h"
 #include "AMDGPUSubtarget.h"
 #include "SIInstrInfo.h"
 #include "SIMachineFunctionInfo.h"
 #include "SIRegisterInfo.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 
+#include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/MC/MCDwarf.h"
+#include "llvm/Support/LEB128.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "frame-info"
 
 
 static ArrayRef<MCPhysReg> getAllSGPR128(const GCNSubtarget &ST,
                                          const MachineFunction &MF) {
   return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(),
                       ST.getMaxNumSGPRs(MF) / 4);
 }
 
 static ArrayRef<MCPhysReg> getAllSGPRs(const GCNSubtarget &ST,
                                        const MachineFunction &MF) {
   return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF));
 }
 
+static ArrayRef<MCPhysReg> getAllVGPRs(const GCNSubtarget &ST,
+                                       const MachineFunction &MF) {
+  return makeArrayRef(AMDGPU::VGPR_32RegClass.begin(), ST.getMaxNumVGPRs(MF));
+}
+
 // Find a scratch register that we can use at the start of the prologue to
 // re-align the stack pointer. We avoid using callee-save registers since they
 // may appear to be free when this is called from canUseAsPrologue (during
 // shrink wrapping), but then no longer be free when this is called from
 // emitPrologue.
 //
 // FIXME: This is a bit conservative, since in the above case we could use one
 // of the callee-save registers as a scratch temp to re-align the stack pointer,

 }
 
 // Activate all lanes, returns saved exec.
 static Register buildScratchExecCopy(LivePhysRegs &LiveRegs,
                                      MachineFunction &MF,
                                      MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator MBBI,
                                      bool IsProlog) {
   Register ScratchExecCopy;
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const SIInstrInfo *TII = ST.getInstrInfo();
   const SIRegisterInfo &TRI = TII->getRegisterInfo();
   SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
   DebugLoc DL;
 
   if (LiveRegs.empty()) {
     if (IsProlog) {
       LiveRegs.init(TRI);
       LiveRegs.addLiveIns(MBB);
       if (FuncInfo->SGPRForFPSaveRestoreCopy)
         LiveRegs.removeReg(FuncInfo->SGPRForFPSaveRestoreCopy);
     } else {
       // In epilog.
       LiveRegs.init(*ST.getRegisterInfo());
       LiveRegs.addLiveOuts(MBB);
       LiveRegs.stepBackward(*MBBI);
     }
   }
 
   ScratchExecCopy = findScratchNonCalleeSaveRegister(
       MRI, LiveRegs, *TRI.getWaveMaskRegClass());
 
   if (!IsProlog)
     LiveRegs.removeReg(ScratchExecCopy);
 
   const unsigned OrSaveExec =
       ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64;
   BuildMI(MBB, MBBI, DL, TII->get(OrSaveExec), ScratchExecCopy).addImm(-1);
 
   return ScratchExecCopy;
 }
 
+void SIFrameLowering::emitPrologueEntryCFI(MachineBasicBlock &MBB,
+                                           MachineBasicBlock::iterator MBBI,
+                                           const DebugLoc &DL) const {
+  const MachineFunction &MF = *MBB.getParent();
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+  const MCRegisterInfo *MCRI = MF.getMMI().getContext().getRegisterInfo();
+  Register StackPtrReg =
+      MF.getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo();
+
+  // DW_ASPACE_AMDGPU_private_wave FIXME: should be defined elsewhere
+  buildCFI(MBB, MBBI, DL,
+           MCCFIInstruction::createLLVMDefAspaceCfa(
+               nullptr, MCRI->getDwarfRegNum(StackPtrReg, false), 0, 6));
+
+  static const char PCEncodedInst[] = {
+      dwarf::DW_CFA_expression,
+      16, // PC 64
+      8,  // length
+      static_cast<char>(dwarf::DW_OP_regx),
+      62, // SGPR30
+      static_cast<char>(dwarf::DW_OP_piece),
+      4, // 32 bits
+      static_cast<char>(dwarf::DW_OP_regx),
+      63, // SGPR31
+      static_cast<char>(dwarf::DW_OP_piece),
+      4 // 32 bits
+  };
+  buildCFI(MBB, MBBI, DL,
+           MCCFIInstruction::createEscape(
+               nullptr, StringRef(PCEncodedInst, sizeof(PCEncodedInst))));
+
+  BitVector IsCalleeSaved(TRI.getNumRegs());
+  const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
+  for (unsigned I = 0; CSRegs[I]; ++I) {
+    IsCalleeSaved.set(CSRegs[I]);
+  }
+  auto ProcessReg = [&](MCPhysReg Reg) {
+    if (IsCalleeSaved.test(Reg) || !MRI.isPhysRegModified(Reg))
+      return;
+    MCRegister DwarfReg = MCRI->getDwarfRegNum(Reg, false);
+    buildCFI(MBB, MBBI, DL,
+             MCCFIInstruction::createUndefined(nullptr, DwarfReg));
+  };
+  for_each(getAllVGPRs(ST, MF), ProcessReg);
+  for_each(getAllSGPRs(ST, MF), ProcessReg);
+};
+
 void SIFrameLowering::emitPrologue(MachineFunction &MF,
                                    MachineBasicBlock &MBB) const {
   SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
   if (FuncInfo->isEntryFunction()) {
     emitEntryFunctionPrologue(MF, MBB);
     return;
   }
 
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const SIInstrInfo *TII = ST.getInstrInfo();
   const SIRegisterInfo &TRI = TII->getRegisterInfo();
+  const MCRegisterInfo *MCRI = MF.getMMI().getContext().getRegisterInfo();
 
   Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
   Register FramePtrReg = FuncInfo->getFrameOffsetReg();
   LivePhysRegs LiveRegs;
 
   MachineBasicBlock::iterator MBBI = MBB.begin();
   DebugLoc DL;
 
   bool HasFP = false;
   uint32_t NumBytes = MFI.getStackSize();
   uint32_t RoundedSize = NumBytes;
   // To avoid clobbering VGPRs in lanes that weren't active on function entry,
   // turn on all lanes before doing the spill to memory.
   Register ScratchExecCopy;
 
+  emitPrologueEntryCFI(MBB, MBBI, DL);
+
   bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue();
   bool SpillFPToMemory = false;
   // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR.
   // Otherwise we are spilling the FP to memory.
   if (HasFPSaveIndex) {
     SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) !=
                       TargetStackID::SGPRSpill;
   }
 
   // Emit the copy if we need an FP, and are using a free SGPR to save it.
   if (FuncInfo->SGPRForFPSaveRestoreCopy) {
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FuncInfo->SGPRForFPSaveRestoreCopy)
       .addReg(FramePtrReg)
       .setMIFlag(MachineInstr::FrameSetup);
+    buildCFI(
+        MBB, MBBI, DL,
+        MCCFIInstruction::createRegister(
+            nullptr, MCRI->getDwarfRegNum(FramePtrReg, false),
+            MCRI->getDwarfRegNum(FuncInfo->SGPRForFPSaveRestoreCopy, false)));
     // Make the register live throughout the function.
     for (MachineBasicBlock &MBB : MF)
       MBB.addLiveIn(FuncInfo->SGPRForFPSaveRestoreCopy);
   }
 
   for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg
          : FuncInfo->getSGPRSpillVGPRs()) {
     if (!Reg.FI.hasValue())
       continue;
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
 
+    int FI = Reg.FI.getValue();
+
     buildPrologSpill(LiveRegs, MBB, MBBI, TII, Reg.VGPR,
-                     FuncInfo->getScratchRSrcReg(),
-                     StackPtrReg,
-                     Reg.FI.getValue());
+                     FuncInfo->getScratchRSrcReg(), StackPtrReg, FI);
+    // We spill the entire VGPR, so we can get away with just cfi_offset
+    buildCFI(MBB, MBBI, DL,
+             MCCFIInstruction::createOffset(
+                 nullptr, MCRI->getDwarfRegNum(Reg.VGPR, false),
+                 MFI.getObjectOffset(FI) * ST.getWavefrontSize()));
   }
 
   if (HasFPSaveIndex && SpillFPToMemory) {
     assert(!MFI.isDeadObjectIndex(FuncInfo->FramePointerSaveIndex.getValue()));
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
 
     MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
         MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
 
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
         .addReg(FramePtrReg);
 
+    int FI = FuncInfo->FramePointerSaveIndex.getValue();
+
     buildPrologSpill(LiveRegs, MBB, MBBI, TII, TmpVGPR,
-                     FuncInfo->getScratchRSrcReg(), StackPtrReg,
-                     FuncInfo->FramePointerSaveIndex.getValue());
+                     FuncInfo->getScratchRSrcReg(), StackPtrReg, FI);
+    buildCFI(MBB, MBBI, DL,
+             MCCFIInstruction::createOffset(
+                 nullptr, MCRI->getDwarfRegNum(FramePtrReg, false),
+                 MFI.getObjectOffset(FI) * ST.getWavefrontSize()));
   }
 
   if (ScratchExecCopy) {
     // FIXME: Split block and make terminator.
     unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
     MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
     BuildMI(MBB, MBBI, DL, TII->get(ExecMov), Exec)
         .addReg(ScratchExecCopy, RegState::Kill);
if (HasFPSaveIndex && !SpillFPToMemory) {
 
     // Save FP before setting it up.
     // FIXME: This should respect spillSGPRToVGPR;
     BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32),
             Spill[0].VGPR)
         .addReg(FramePtrReg)
         .addImm(Spill[0].Lane)
         .addReg(Spill[0].VGPR, RegState::Undef);
+    buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, FramePtrReg, Spill[0].VGPR,
+                               Spill[0].Lane);
   }
 
   if (TRI.needsStackRealignment(MF)) {
     HasFP = true;
     const unsigned Alignment = MFI.getMaxAlign().value();
 
     RoundedSize += Alignment;
     if (LiveRegs.empty()) {
if (TRI.needsStackRealignment(MF)) {
     // the stack pointer is at this point. Any variable size objects will be
     // allocated after this, so we can still use the base pointer to reference
     // locals.
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
         .addReg(StackPtrReg)
         .setMIFlag(MachineInstr::FrameSetup);
   }
 
+  if (HasFP) {
+    buildCFI(MBB, MBBI, DL,
+             MCCFIInstruction::createDefCfaRegister(
+                 nullptr, MCRI->getDwarfRegNum(FramePtrReg, false)));
+  }
+
   if (HasFP && RoundedSize != 0) {
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_U32), StackPtrReg)
         .addReg(StackPtrReg)
         .addImm(RoundedSize * ST.getWavefrontSize())
         .setMIFlag(MachineInstr::FrameSetup);
   }
 
   assert((!HasFP || (FuncInfo->SGPRForFPSaveRestoreCopy ||
void SIFrameLowering::emitEpilogue(MachineFunction &MF,
                                    MachineBasicBlock &MBB) const {
   const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
   if (FuncInfo->isEntryFunction())
     return;
 
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const SIInstrInfo *TII = ST.getInstrInfo();
   MachineRegisterInfo &MRI = MF.getRegInfo();
+  const MCRegisterInfo *MCRI = MF.getMMI().getContext().getRegisterInfo();
   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
   LivePhysRegs LiveRegs;
   DebugLoc DL;
 
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   uint32_t NumBytes = MFI.getStackSize();
   uint32_t RoundedSize = FuncInfo->isStackRealigned()
                              ? NumBytes + MFI.getMaxAlign().value()
                              : NumBytes;
   const Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
   const Register FramePtrReg = FuncInfo->getFrameOffsetReg();
 
   bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue();
   bool SpillFPToMemory = false;
   if (HasFPSaveIndex) {
     SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) !=
                       TargetStackID::SGPRSpill;
   }
if (SpillFPToMemory) {
       assert(Spill.size() == 1);
       BuildMI(MBB, MBBI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32),
               FramePtrReg)
           .addReg(Spill[0].VGPR)
           .addImm(Spill[0].Lane);
     }
   }
 
+  if (hasFP(MF)) {
+    buildCFI(MBB, MBBI, DL,
+             MCCFIInstruction::createDefCfaRegister(
+                 nullptr, MCRI->getDwarfRegNum(StackPtrReg, false)));
+  }
+
   for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg :
        FuncInfo->getSGPRSpillVGPRs()) {
     if (!Reg.FI.hasValue())
       continue;
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
 
void SIFrameLowering::buildCFI(MachineBasicBlock &MBB,
                                const DebugLoc &DL,
                                const MCCFIInstruction &CFIInst) const {
   MachineFunction &MF = *MBB.getParent();
   const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
   BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
       .addCFIIndex(MF.addFrameInst(CFIInst))
       .setMIFlag(MachineInstr::FrameSetup);
 }
+
+static void encodeDwarfRegisterLocation(int DwarfReg, raw_ostream &OS) {
+  if (DwarfReg < 32) {
+    OS << uint8_t(dwarf::DW_OP_reg0 + DwarfReg);
+  } else {
+    OS << uint8_t(dwarf::DW_OP_regx);
+    encodeULEB128(DwarfReg, OS);
+  }
+}
+
+void SIFrameLowering::buildCFIForSGPRToVGPRSpill(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    const DebugLoc &DL, const Register SGPR, const Register VGPR,
+    const int Lane) const {
+  MachineFunction &MF = *MBB.getParent();
+  const MCRegisterInfo &MCRI = *MF.getMMI().getContext().getRegisterInfo();
+  int DwarfSGPR = MCRI.getDwarfRegNum(SGPR, false);
+  int DwarfVGPR = MCRI.getDwarfRegNum(VGPR, false);
+
+  // CFI for an SGPR spilled to a single lane of a VGPR is implemented as an
+  // expression(E) rule where E is a register location description referencing
+  // a VGPR register location storage at a byte offset of the lane index
+  // multiplied by the size of an SGPR (4 bytes). In other words we generate
+  // the following DWARF:
+  //
+  // DW_CFA_expression: <SGPR>,
+  //    (DW_OP_regx <VGPR>) (DW_OP_LLVM_offset_uconst <Lane>*4)
+  //
+  // The memory location description for the current CFA is pushed on the
+  // stack before E is evaluated, but we choose not to drop it as it would
+  // require a longer expression E and DWARF defines the result of the
+  // evaulation to be the location description on the top of the stack (i.e. the
+  // implictly pushed one is just ignored.)
+  SmallString<20> CFIInst;
+  raw_svector_ostream OSCFIInst(CFIInst);
+  SmallString<20> Block;
+  raw_svector_ostream OSBlock(Block);
+
+  OSCFIInst << uint8_t(dwarf::DW_CFA_expression);
+  encodeULEB128(DwarfSGPR, OSCFIInst);
+
+  encodeDwarfRegisterLocation(DwarfVGPR, OSBlock);
+  OSBlock << uint8_t(dwarf::DW_OP_LLVM_offset_uconst);
+  // FIXME:
+  const unsigned SGPRByteSize = 4;
+  encodeULEB128(Lane * SGPRByteSize, OSBlock);
+
+  encodeULEB128(Block.size(), OSCFIInst);
+  OSCFIInst << Block;
+
+  buildCFI(MBB, MBBI, DL,
+           MCCFIInstruction::createEscape(nullptr, OSCFIInst.str()));
+}

llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll

 ; GCN-LABEL: {{^}}spill_only_csr_sgpr:
 ; GCN: s_waitcnt
 ; GCN-NEXT: v_writelane_b32 v0, s42, 0
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: ; clobber s42
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NEXT: v_readlane_b32 s42, v0, 0
 ; GCN-NEXT: s_setpc_b64
-define void @spill_only_csr_sgpr() {
+define void @spill_only_csr_sgpr() #0 {
   call void asm sideeffect "; clobber s42", "~{s42}"()
   ret void
 }
 
 ; TODO: Can the SP inc/deec be remvoed?
 ; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_csr_vgpr:
 ; GCN: s_waitcnt
 ; GCN-NEXT:s_mov_b32 [[FP_COPY:s[0-9]+]], s33
call void asm sideeffect "",
     ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
     ,~{s100},~{s101},~{s102}"() #1
 
   ret void
 }
 
 ; GCN-LABEL: {{^}}realign_stack_no_fp_elim:
 ; GCN: s_waitcnt
-; GCN-NEXT: s_add_u32 [[SCRATCH:s[0-9]+]], s32, 0x7ffc0
 ; GCN-NEXT: s_mov_b32 s4, s33
+; GCN-NEXT: s_add_u32 [[SCRATCH:s[0-9]+]], s32, 0x7ffc0
 ; GCN-NEXT: s_and_b32 s33, [[SCRATCH]], 0xfff80000
 ; GCN-NEXT: s_add_u32 s32, s32, 0x100000
 ; GCN-NEXT: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
 ; GCN-NEXT: buffer_store_dword [[ZERO]], off, s[0:3], s33
 ; GCN-NEXT: s_sub_u32 s32, s32, 0x100000
 ; GCN-NEXT: s_mov_b32 s33, s4
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @realign_stack_no_fp_elim() #1 {
   %alloca = alloca i32, align 8192, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   ret void
 }
 
 ; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp:
 ; GCN: s_waitcnt
 ; GCN-NEXT: v_writelane_b32 v1, s33, 2
-; GCN-NEXT: v_writelane_b32 v1, s30, 0
 ; GCN-NEXT: s_mov_b32 s33, s32
+; GCN-NEXT: v_writelane_b32 v1, s30, 0
 ; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
 ; GCN: v_writelane_b32 v1, s31, 1
 ; GCN: buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:4
 ; GCN: ;;#ASMSTART
-; GCN: v_readlane_b32 s4, v1, 0
-; GCN-NEXT: s_add_u32 s32, s32, 0x200
+; GCN: s_add_u32 s32, s32, 0x200
+; GCN-NEXT: v_readlane_b32 s4, v1, 0
 ; GCN-NEXT: v_readlane_b32 s5, v1, 1
 ; GCN-NEXT: s_sub_u32 s32, s32, 0x200
 ; GCN-NEXT: v_readlane_b32 s33, v1, 2
 ; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: s_setpc_b64 s[4:5]
 define void @no_unused_non_csr_sgpr_for_fp() #1 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca

 
 ; Need a new CSR VGPR to satisfy the FP spill.
 ; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2
-; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-NEXT: s_mov_b32 s33, s32
+; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s30, 0
 
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; GCN-DAG: buffer_store_dword
 ; GCN: s_add_u32 s32, s32, 0x300{{$}}
 
 ; GCN: ;;#ASMSTART
 
 ; GCN: v_readlane_b32 s4, [[CSR_VGPR]], 0

 ; register is needed to access the CSR VGPR slot.
 ; GCN-LABEL: {{^}}scratch_reg_needed_mubuf_offset:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
 ; GCN-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2
-; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-NEXT: s_mov_b32 s33, s32
+; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s30, 0
 ; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
 ; GCN-DAG: s_add_u32 s32, s32, 0x40300{{$}}
 ; GCN-DAG: buffer_store_dword
 
 ; GCN: ;;#ASMSTART
 
 ; GCN: v_readlane_b32 s4, [[CSR_VGPR]], 0
 ; GCN-NEXT: v_readlane_b32 s5, [[CSR_VGPR]], 1

llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll

 define float @call_split_type_used_outside_block_v2f32() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_v2f32:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
 ; GCN-NEXT:    v_writelane_b32 v40, s33, 2
-; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_getpc_b64 s[4:5]
 ; GCN-NEXT:    s_add_u32 s4, s4, func_v2f32@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s5, s5, func_v2f32@rel32@hi+4
 ; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GCN-NEXT:    v_readlane_b32 s4, v40, 0
 ; GCN-NEXT:    v_readlane_b32 s5, v40, 1

 define float @call_split_type_used_outside_block_v3f32() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_v3f32:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
 ; GCN-NEXT:    v_writelane_b32 v40, s33, 2
-; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_getpc_b64 s[4:5]
 ; GCN-NEXT:    s_add_u32 s4, s4, func_v3f32@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s5, s5, func_v3f32@rel32@hi+4
 ; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GCN-NEXT:    v_readlane_b32 s4, v40, 0
 ; GCN-NEXT:    v_readlane_b32 s5, v40, 1

 define half @call_split_type_used_outside_block_v4f16() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_v4f16:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
 ; GCN-NEXT:    v_writelane_b32 v40, s33, 2
-; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_getpc_b64 s[4:5]
 ; GCN-NEXT:    s_add_u32 s4, s4, func_v4f16@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s5, s5, func_v4f16@rel32@hi+4
 ; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GCN-NEXT:    v_readlane_b32 s4, v40, 0
 ; GCN-NEXT:    v_readlane_b32 s5, v40, 1

 define { i32, half } @call_split_type_used_outside_block_struct() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_struct:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
 ; GCN-NEXT:    v_writelane_b32 v40, s33, 2
-; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_mov_b32 s33, s32
+; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_getpc_b64 s[4:5]
 ; GCN-NEXT:    s_add_u32 s4, s4, func_struct@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s5, s5, func_struct@rel32@hi+4
 ; GCN-NEXT:    v_writelane_b32 v40, s31, 1
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GCN-NEXT:    v_readlane_b32 s4, v40, 0
 ; GCN-NEXT:    v_readlane_b32 s5, v40, 1

llvm/test/CodeGen/AMDGPU/debug-frame.ll

-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=asm -o - %s | FileCheck --check-prefixes=CHECK,WAVE64 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -filetype=asm -o - %s | FileCheck --check-prefixes=CHECK,WAVE32 %s
 
 ; CHECK-LABEL: kern1:
 ; CHECK: .cfi_startproc
 
 ; CHECK-NOT: .cfi_{{.*}}
 
 ; CHECK: %bb.0:
 ; DW_CFA_def_cfa_expression [0x0f]
 ;   BLOCK_LENGTH ULEB128(3)=[0x03]
 ;     DW_OP_lit0 [0x30]
 ;     DW_OP_lit6 [0x36]
 ;     DW_OP_LLVM_form_aspace_address [0xe1]
 ; CHECK-NEXT: .cfi_escape 0x0f, 0x03, 0x30, 0x36, 0xe1
 ; PC_64 = 16
 ; CHECK-NEXT: .cfi_undefined 16
 
 ; CHECK-NOT: .cfi_{{.*}}
 
 ; CHECK: .cfi_endproc
 define protected amdgpu_kernel void @kern1() #0 {
 entry:
   ret void
 }
 
+; CHECK-LABEL: func_no_clobber:
+; CHECK: .cfi_startproc
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; CHECK: %bb.0:
+; SGPR32 = 64
+; CHECK-NEXT: .cfi_llvm_def_aspace_cfa 64, 0, 6
+; DW_CFA_expression [0x10]
+;   PC_64 ULEB128(17)=[0x10]
+;   BLOCK_LENGTH ULEB128(8)=[0x08]
+;     DW_OP_regx [0x90]
+;       SGPR30 ULEB128(62)=[0x3e]
+;     DW_OP_piece [0x93]
+;       PIECE_SIZE [0x04]
+;     DW_OP_regx [0x90]
+;       SGPR31 ULEB128(63)=[0x3f]
+;     DW_OP_piece [0x93]
+;       PIECE_SIZE [0x04]
+; CHECK-NEXT: .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; CHECK: .cfi_endproc
+define hidden void @func_no_clobber() #0 {
+entry:
+  ret void
+}
+
+; CHECK-LABEL: {{^}}callee_need_to_spill_fp_to_memory:
+; CHECK: .cfi_startproc
+
+; SGPR33 = 65
+; CHECK: v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
+; CHECK: buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; WAVE64: .cfi_offset 65, 29184
+; WAVE32: .cfi_offset 65, 14592
+
+; CHECK: .cfi_endproc
+define void @callee_need_to_spill_fp_to_memory() #1 {
+  call void asm sideeffect "; clobber nonpreserved SGPRs",
+    "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
+    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
+    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
+    ,~{vcc}"()
+
+  call void asm sideeffect "; clobber all VGPRs",
+    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}
+    ,~{v40},~{v41},~{v42},~{v43},~{v44},~{v45},~{v46},~{v47},~{v48},~{v49}
+    ,~{v50},~{v51},~{v52},~{v53},~{v54},~{v55},~{v56},~{v57},~{v58},~{v59}
+    ,~{v60},~{v61},~{v62},~{v63},~{v64},~{v65},~{v66},~{v67},~{v68},~{v69}
+    ,~{v70},~{v71},~{v72},~{v73},~{v74},~{v75},~{v76},~{v77},~{v78},~{v79}
+    ,~{v80},~{v81},~{v82},~{v83},~{v84},~{v85},~{v86},~{v87},~{v88},~{v89}
+    ,~{v90},~{v91},~{v92},~{v93},~{v94},~{v95},~{v96},~{v97},~{v98},~{v99}
+    ,~{v100},~{v101},~{v102},~{v103},~{v104},~{v105},~{v106},~{v107},~{v108},~{v109}
+    ,~{v110},~{v111},~{v112},~{v113},~{v114},~{v115},~{v116},~{v117},~{v118},~{v119}
+    ,~{v120},~{v121},~{v122},~{v123},~{v124},~{v125},~{v126},~{v127},~{v128},~{v129}
+    ,~{v130},~{v131},~{v132},~{v133},~{v134},~{v135},~{v136},~{v137},~{v138},~{v139}
+    ,~{v140},~{v141},~{v142},~{v143},~{v144},~{v145},~{v146},~{v147},~{v148},~{v149}
+    ,~{v150},~{v151},~{v152},~{v153},~{v154},~{v155},~{v156},~{v157},~{v158},~{v159}
+    ,~{v160},~{v161},~{v162},~{v163},~{v164},~{v165},~{v166},~{v167},~{v168},~{v169}
+    ,~{v170},~{v171},~{v172},~{v173},~{v174},~{v175},~{v176},~{v177},~{v178},~{v179}
+    ,~{v180},~{v181},~{v182},~{v183},~{v184},~{v185},~{v186},~{v187},~{v188},~{v189}
+    ,~{v190},~{v191},~{v192},~{v193},~{v194},~{v195},~{v196},~{v197},~{v198},~{v199}
+    ,~{v200},~{v201},~{v202},~{v203},~{v204},~{v205},~{v206},~{v207},~{v208},~{v209}
+    ,~{v210},~{v211},~{v212},~{v213},~{v214},~{v215},~{v216},~{v217},~{v218},~{v219}
+    ,~{v220},~{v221},~{v222},~{v223},~{v224},~{v225},~{v226},~{v227},~{v228},~{v229}
+    ,~{v230},~{v231},~{v232},~{v233},~{v234},~{v235},~{v236},~{v237},~{v238},~{v239}
+    ,~{v240},~{v241},~{v242},~{v243},~{v244},~{v245},~{v246},~{v247},~{v248},~{v249}
+    ,~{v250},~{v251},~{v252},~{v253},~{v254},~{v255}"()
+  ret void
+}
+
+declare hidden void @ex() #0
+
+; CHECK-LABEL: func_call_clobber:
+; CHECK: .cfi_startproc
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; CHECK: %bb.0:
+; CHECK-NEXT: .cfi_llvm_def_aspace_cfa 64, 0, 6
+; CHECK-NEXT: .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
+
+; VGPR0_wave64 = 2560
+; WAVE64-NEXT: .cfi_undefined 2560
+; WAVE64-NEXT: .cfi_undefined 2561
+; WAVE64-NEXT: .cfi_undefined 2562
+; WAVE64-NEXT: .cfi_undefined 2563
+; WAVE64-NEXT: .cfi_undefined 2564
+; WAVE64-NEXT: .cfi_undefined 2565
+; WAVE64-NEXT: .cfi_undefined 2566
+; WAVE64-NEXT: .cfi_undefined 2567
+; WAVE64-NEXT: .cfi_undefined 2568
+; WAVE64-NEXT: .cfi_undefined 2569
+; WAVE64-NEXT: .cfi_undefined 2570
+; WAVE64-NEXT: .cfi_undefined 2571
+; WAVE64-NEXT: .cfi_undefined 2572
+; WAVE64-NEXT: .cfi_undefined 2573
+; WAVE64-NEXT: .cfi_undefined 2574
+; WAVE64-NEXT: .cfi_undefined 2575
+; WAVE64-NEXT: .cfi_undefined 2576
+; WAVE64-NEXT: .cfi_undefined 2577
+; WAVE64-NEXT: .cfi_undefined 2578
+; WAVE64-NEXT: .cfi_undefined 2579
+; WAVE64-NEXT: .cfi_undefined 2580
+; WAVE64-NEXT: .cfi_undefined 2581
+; WAVE64-NEXT: .cfi_undefined 2582
+; WAVE64-NEXT: .cfi_undefined 2583
+; WAVE64-NEXT: .cfi_undefined 2584
+; WAVE64-NEXT: .cfi_undefined 2585
+; WAVE64-NEXT: .cfi_undefined 2586
+; WAVE64-NEXT: .cfi_undefined 2587
+; WAVE64-NEXT: .cfi_undefined 2588
+; WAVE64-NEXT: .cfi_undefined 2589
+; WAVE64-NEXT: .cfi_undefined 2590
+; WAVE64-NEXT: .cfi_undefined 2591
+; WAVE64-NEXT: .cfi_undefined 2592
+; WAVE64-NEXT: .cfi_undefined 2593
+; WAVE64-NEXT: .cfi_undefined 2594
+; WAVE64-NEXT: .cfi_undefined 2595
+; WAVE64-NEXT: .cfi_undefined 2596
+; WAVE64-NEXT: .cfi_undefined 2597
+; WAVE64-NEXT: .cfi_undefined 2598
+; WAVE64-NEXT: .cfi_undefined 2599
+
+; VPGR48_wave64 = 2608
+; WAVE64-NEXT: .cfi_undefined 2608
+; WAVE64-NEXT: .cfi_undefined 2609
+; WAVE64-NEXT: .cfi_undefined 2610
+; WAVE64-NEXT: .cfi_undefined 2611
+; WAVE64-NEXT: .cfi_undefined 2612
+; WAVE64-NEXT: .cfi_undefined 2613
+; WAVE64-NEXT: .cfi_undefined 2614
+; WAVE64-NEXT: .cfi_undefined 2615
+
+; WAVE64-NEXT: .cfi_undefined 2624
+; WAVE64-NEXT: .cfi_undefined 2625
+; WAVE64-NEXT: .cfi_undefined 2626
+; WAVE64-NEXT: .cfi_undefined 2627
+; WAVE64-NEXT: .cfi_undefined 2628
+; WAVE64-NEXT: .cfi_undefined 2629
+; WAVE64-NEXT: .cfi_undefined 2630
+; WAVE64-NEXT: .cfi_undefined 2631
+
+; WAVE64-NEXT: .cfi_undefined 2640
+; WAVE64-NEXT: .cfi_undefined 2641
+; WAVE64-NEXT: .cfi_undefined 2642
+; WAVE64-NEXT: .cfi_undefined 2643
+; WAVE64-NEXT: .cfi_undefined 2644
+; WAVE64-NEXT: .cfi_undefined 2645
+; WAVE64-NEXT: .cfi_undefined 2646
+; WAVE64-NEXT: .cfi_undefined 2647
+
+; WAVE64-NEXT: .cfi_undefined 2656
+; WAVE64-NEXT: .cfi_undefined 2657
+; WAVE64-NEXT: .cfi_undefined 2658
+; WAVE64-NEXT: .cfi_undefined 2659
+; WAVE64-NEXT: .cfi_undefined 2660
+; WAVE64-NEXT: .cfi_undefined 2661
+; WAVE64-NEXT: .cfi_undefined 2662
+; WAVE64-NEXT: .cfi_undefined 2663
+
+; WAVE64-NEXT: .cfi_undefined 2672
+; WAVE64-NEXT: .cfi_undefined 2673
+; WAVE64-NEXT: .cfi_undefined 2674
+; WAVE64-NEXT: .cfi_undefined 2675
+; WAVE64-NEXT: .cfi_undefined 2676
+; WAVE64-NEXT: .cfi_undefined 2677
+; WAVE64-NEXT: .cfi_undefined 2678
+; WAVE64-NEXT: .cfi_undefined 2679
+
+; WAVE64-NEXT: .cfi_undefined 2688
+; WAVE64-NEXT: .cfi_undefined 2689
+; WAVE64-NEXT: .cfi_undefined 2690
+; WAVE64-NEXT: .cfi_undefined 2691
+; WAVE64-NEXT: .cfi_undefined 2692
+; WAVE64-NEXT: .cfi_undefined 2693
+; WAVE64-NEXT: .cfi_undefined 2694
+; WAVE64-NEXT: .cfi_undefined 2695
+
+; WAVE64-NEXT: .cfi_undefined 2704
+; WAVE64-NEXT: .cfi_undefined 2705
+; WAVE64-NEXT: .cfi_undefined 2706
+; WAVE64-NEXT: .cfi_undefined 2707
+; WAVE64-NEXT: .cfi_undefined 2708
+; WAVE64-NEXT: .cfi_undefined 2709
+; WAVE64-NEXT: .cfi_undefined 2710
+; WAVE64-NEXT: .cfi_undefined 2711
+
+; WAVE64-NEXT: .cfi_undefined 2720
+; WAVE64-NEXT: .cfi_undefined 2721
+; WAVE64-NEXT: .cfi_undefined 2722
+; WAVE64-NEXT: .cfi_undefined 2723
+; WAVE64-NEXT: .cfi_undefined 2724
+; WAVE64-NEXT: .cfi_undefined 2725
+; WAVE64-NEXT: .cfi_undefined 2726
+; WAVE64-NEXT: .cfi_undefined 2727
+
+; WAVE64-NEXT: .cfi_undefined 2736
+; WAVE64-NEXT: .cfi_undefined 2737
+; WAVE64-NEXT: .cfi_undefined 2738
+; WAVE64-NEXT: .cfi_undefined 2739
+; WAVE64-NEXT: .cfi_undefined 2740
+; WAVE64-NEXT: .cfi_undefined 2741
+; WAVE64-NEXT: .cfi_undefined 2742
+; WAVE64-NEXT: .cfi_undefined 2743
+
+; WAVE64-NEXT: .cfi_undefined 2752
+; WAVE64-NEXT: .cfi_undefined 2753
+; WAVE64-NEXT: .cfi_undefined 2754
+; WAVE64-NEXT: .cfi_undefined 2755
+; WAVE64-NEXT: .cfi_undefined 2756
+; WAVE64-NEXT: .cfi_undefined 2757
+; WAVE64-NEXT: .cfi_undefined 2758
+; WAVE64-NEXT: .cfi_undefined 2759
+
+; WAVE64-NEXT: .cfi_undefined 2768
+; WAVE64-NEXT: .cfi_undefined 2769
+; WAVE64-NEXT: .cfi_undefined 2770
+; WAVE64-NEXT: .cfi_undefined 2771
+; WAVE64-NEXT: .cfi_undefined 2772
+; WAVE64-NEXT: .cfi_undefined 2773
+; WAVE64-NEXT: .cfi_undefined 2774
+; WAVE64-NEXT: .cfi_undefined 2775
+
+; WAVE64-NEXT: .cfi_undefined 2784
+; WAVE64-NEXT: .cfi_undefined 2785
+; WAVE64-NEXT: .cfi_undefined 2786
+; WAVE64-NEXT: .cfi_undefined 2787
+; WAVE64-NEXT: .cfi_undefined 2788
+; WAVE64-NEXT: .cfi_undefined 2789
+; WAVE64-NEXT: .cfi_undefined 2790
+; WAVE64-NEXT: .cfi_undefined 2791
+
+; WAVE64-NEXT: .cfi_undefined 2800
+; WAVE64-NEXT: .cfi_undefined 2801
+; WAVE64-NEXT: .cfi_undefined 2802
+; WAVE64-NEXT: .cfi_undefined 2803
+; WAVE64-NEXT: .cfi_undefined 2804
+; WAVE64-NEXT: .cfi_undefined 2805
+; WAVE64-NEXT: .cfi_undefined 2806
+; WAVE64-NEXT: .cfi_undefined 2807
+
+
+; VGPR0_wave32 = 1536
+; WAVE32-NEXT: .cfi_undefined 1536
+; WAVE32-NEXT: .cfi_undefined 1537
+; WAVE32-NEXT: .cfi_undefined 1538
+; WAVE32-NEXT: .cfi_undefined 1539
+; WAVE32-NEXT: .cfi_undefined 1540
+; WAVE32-NEXT: .cfi_undefined 1541
+; WAVE32-NEXT: .cfi_undefined 1542
+; WAVE32-NEXT: .cfi_undefined 1543
+; WAVE32-NEXT: .cfi_undefined 1544
+; WAVE32-NEXT: .cfi_undefined 1545
+; WAVE32-NEXT: .cfi_undefined 1546
+; WAVE32-NEXT: .cfi_undefined 1547
+; WAVE32-NEXT: .cfi_undefined 1548
+; WAVE32-NEXT: .cfi_undefined 1549
+; WAVE32-NEXT: .cfi_undefined 1550
+; WAVE32-NEXT: .cfi_undefined 1551
+; WAVE32-NEXT: .cfi_undefined 1552
+; WAVE32-NEXT: .cfi_undefined 1553
+; WAVE32-NEXT: .cfi_undefined 1554
+; WAVE32-NEXT: .cfi_undefined 1555
+; WAVE32-NEXT: .cfi_undefined 1556
+; WAVE32-NEXT: .cfi_undefined 1557
+; WAVE32-NEXT: .cfi_undefined 1558
+; WAVE32-NEXT: .cfi_undefined 1559
+; WAVE32-NEXT: .cfi_undefined 1560
+; WAVE32-NEXT: .cfi_undefined 1561
+; WAVE32-NEXT: .cfi_undefined 1562
+; WAVE32-NEXT: .cfi_undefined 1563
+; WAVE32-NEXT: .cfi_undefined 1564
+; WAVE32-NEXT: .cfi_undefined 1565
+; WAVE32-NEXT: .cfi_undefined 1566
+; WAVE32-NEXT: .cfi_undefined 1567
+; WAVE32-NEXT: .cfi_undefined 1568
+; WAVE32-NEXT: .cfi_undefined 1569
+; WAVE32-NEXT: .cfi_undefined 1570
+; WAVE32-NEXT: .cfi_undefined 1571
+; WAVE32-NEXT: .cfi_undefined 1572
+; WAVE32-NEXT: .cfi_undefined 1573
+; WAVE32-NEXT: .cfi_undefined 1574
+; WAVE32-NEXT: .cfi_undefined 1575
+
+; VPGR48_wave64 = 1584
+; WAVE32-NEXT: .cfi_undefined 1584
+; WAVE32-NEXT: .cfi_undefined 1585
+; WAVE32-NEXT: .cfi_undefined 1586
+; WAVE32-NEXT: .cfi_undefined 1587
+; WAVE32-NEXT: .cfi_undefined 1588
+; WAVE32-NEXT: .cfi_undefined 1589
+; WAVE32-NEXT: .cfi_undefined 1590
+; WAVE32-NEXT: .cfi_undefined 1591
+
+; WAVE32-NEXT: .cfi_undefined 1600
+; WAVE32-NEXT: .cfi_undefined 1601
+; WAVE32-NEXT: .cfi_undefined 1602
+; WAVE32-NEXT: .cfi_undefined 1603
+; WAVE32-NEXT: .cfi_undefined 1604
+; WAVE32-NEXT: .cfi_undefined 1605
+; WAVE32-NEXT: .cfi_undefined 1606
+; WAVE32-NEXT: .cfi_undefined 1607
+
+; WAVE32-NEXT: .cfi_undefined 1616
+; WAVE32-NEXT: .cfi_undefined 1617
+; WAVE32-NEXT: .cfi_undefined 1618
+; WAVE32-NEXT: .cfi_undefined 1619
+; WAVE32-NEXT: .cfi_undefined 1620
+; WAVE32-NEXT: .cfi_undefined 1621
+; WAVE32-NEXT: .cfi_undefined 1622
+; WAVE32-NEXT: .cfi_undefined 1623
+
+; WAVE32-NEXT: .cfi_undefined 1632
+; WAVE32-NEXT: .cfi_undefined 1633
+; WAVE32-NEXT: .cfi_undefined 1634
+; WAVE32-NEXT: .cfi_undefined 1635
+; WAVE32-NEXT: .cfi_undefined 1636
+; WAVE32-NEXT: .cfi_undefined 1637
+; WAVE32-NEXT: .cfi_undefined 1638
+; WAVE32-NEXT: .cfi_undefined 1639
+
+; WAVE32-NEXT: .cfi_undefined 1648
+; WAVE32-NEXT: .cfi_undefined 1649
+; WAVE32-NEXT: .cfi_undefined 1650
+; WAVE32-NEXT: .cfi_undefined 1651
+; WAVE32-NEXT: .cfi_undefined 1652
+; WAVE32-NEXT: .cfi_undefined 1653
+; WAVE32-NEXT: .cfi_undefined 1654
+; WAVE32-NEXT: .cfi_undefined 1655
+
+; WAVE32-NEXT: .cfi_undefined 1664
+; WAVE32-NEXT: .cfi_undefined 1665
+; WAVE32-NEXT: .cfi_undefined 1666
+; WAVE32-NEXT: .cfi_undefined 1667
+; WAVE32-NEXT: .cfi_undefined 1668
+; WAVE32-NEXT: .cfi_undefined 1669
+; WAVE32-NEXT: .cfi_undefined 1670
+; WAVE32-NEXT: .cfi_undefined 1671
+
+; WAVE32-NEXT: .cfi_undefined 1680
+; WAVE32-NEXT: .cfi_undefined 1681
+; WAVE32-NEXT: .cfi_undefined 1682
+; WAVE32-NEXT: .cfi_undefined 1683
+; WAVE32-NEXT: .cfi_undefined 1684
+; WAVE32-NEXT: .cfi_undefined 1685
+; WAVE32-NEXT: .cfi_undefined 1686
+; WAVE32-NEXT: .cfi_undefined 1687
+
+; WAVE32-NEXT: .cfi_undefined 1696
+; WAVE32-NEXT: .cfi_undefined 1697
+; WAVE32-NEXT: .cfi_undefined 1698
+; WAVE32-NEXT: .cfi_undefined 1699
+; WAVE32-NEXT: .cfi_undefined 1700
+; WAVE32-NEXT: .cfi_undefined 1701
+; WAVE32-NEXT: .cfi_undefined 1702
+; WAVE32-NEXT: .cfi_undefined 1703
+
+; WAVE32-NEXT: .cfi_undefined 1712
+; WAVE32-NEXT: .cfi_undefined 1713
+; WAVE32-NEXT: .cfi_undefined 1714
+; WAVE32-NEXT: .cfi_undefined 1715
+; WAVE32-NEXT: .cfi_undefined 1716
+; WAVE32-NEXT: .cfi_undefined 1717
+; WAVE32-NEXT: .cfi_undefined 1718
+; WAVE32-NEXT: .cfi_undefined 1719
+
+; WAVE32-NEXT: .cfi_undefined 1728
+; WAVE32-NEXT: .cfi_undefined 1729
+; WAVE32-NEXT: .cfi_undefined 1730
+; WAVE32-NEXT: .cfi_undefined 1731
+; WAVE32-NEXT: .cfi_undefined 1732
+; WAVE32-NEXT: .cfi_undefined 1733
+; WAVE32-NEXT: .cfi_undefined 1734
+; WAVE32-NEXT: .cfi_undefined 1735
+
+; WAVE32-NEXT: .cfi_undefined 1744
+; WAVE32-NEXT: .cfi_undefined 1745
+; WAVE32-NEXT: .cfi_undefined 1746
+; WAVE32-NEXT: .cfi_undefined 1747
+; WAVE32-NEXT: .cfi_undefined 1748
+; WAVE32-NEXT: .cfi_undefined 1749
+; WAVE32-NEXT: .cfi_undefined 1750
+; WAVE32-NEXT: .cfi_undefined 1751
+
+; WAVE32-NEXT: .cfi_undefined 1760
+; WAVE32-NEXT: .cfi_undefined 1761
+; WAVE32-NEXT: .cfi_undefined 1762
+; WAVE32-NEXT: .cfi_undefined 1763
+; WAVE32-NEXT: .cfi_undefined 1764
+; WAVE32-NEXT: .cfi_undefined 1765
+; WAVE32-NEXT: .cfi_undefined 1766
+; WAVE32-NEXT: .cfi_undefined 1767
+
+; WAVE32-NEXT: .cfi_undefined 1776
+; WAVE32-NEXT: .cfi_undefined 1777
+; WAVE32-NEXT: .cfi_undefined 1778
+; WAVE32-NEXT: .cfi_undefined 1779
+; WAVE32-NEXT: .cfi_undefined 1780
+; WAVE32-NEXT: .cfi_undefined 1781
+; WAVE32-NEXT: .cfi_undefined 1782
+; WAVE32-NEXT: .cfi_undefined 1783
+
+
+; SGPR0 = 32
+; CHECK-NEXT: .cfi_undefined 32
+; CHECK-NEXT: .cfi_undefined 33
+; CHECK-NEXT: .cfi_undefined 34
+; CHECK-NEXT: .cfi_undefined 35
+; CHECK-NEXT: .cfi_undefined 36
+; CHECK-NEXT: .cfi_undefined 37
+; CHECK-NEXT: .cfi_undefined 38
+; CHECK-NEXT: .cfi_undefined 39
+; CHECK-NEXT: .cfi_undefined 40
+; CHECK-NEXT: .cfi_undefined 41
+; CHECK-NEXT: .cfi_undefined 42
+; CHECK-NEXT: .cfi_undefined 43
+; CHECK-NEXT: .cfi_undefined 44
+; CHECK-NEXT: .cfi_undefined 45
+; CHECK-NEXT: .cfi_undefined 46
+; CHECK-NEXT: .cfi_undefined 47
+; CHECK-NEXT: .cfi_undefined 48
+; CHECK-NEXT: .cfi_undefined 49
+; CHECK-NEXT: .cfi_undefined 50
+; CHECK-NEXT: .cfi_undefined 51
+; CHECK-NEXT: .cfi_undefined 52
+; CHECK-NEXT: .cfi_undefined 53
+; CHECK-NEXT: .cfi_undefined 54
+; CHECK-NEXT: .cfi_undefined 55
+; CHECK-NEXT: .cfi_undefined 56
+; CHECK-NEXT: .cfi_undefined 57
+; CHECK-NEXT: .cfi_undefined 58
+; CHECK-NEXT: .cfi_undefined 59
+; CHECK-NEXT: .cfi_undefined 60
+; CHECK-NEXT: .cfi_undefined 61
+; CHECK-NEXT: .cfi_undefined 62
+; CHECK-NEXT: .cfi_undefined 63
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; WAVE64: s_or_saveexec_b64 s[4:5], -1
+; WAVE32: s_or_saveexec_b32 s4, -1
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
+; VGPR40_wave64 = 2600
+; WAVE64-NEXT: .cfi_offset 2600, 0
+; VGPR40_wave32 = 1576
+; WAVE32-NEXT: .cfi_offset 1576, 0
+; CHECK-NOT: .cfi_{{.*}}
+; WAVE64: s_mov_b64 exec, s[4:5]
+; WAVE32: s_mov_b32 exec_lo, s4
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; CHECK: v_writelane_b32 v40, s33, 2
+
+; DW_CFA_expression [0x10] SGPR33 ULEB128(65)=[0x41]
+;   BLOCK_LENGTH ULEB128(5)=[0x05]
+;     DW_OP_regx [0x90]
+;       VGPR40_wave64 ULEB128(2600)=[0xa8, 0x14]
+;     DW_OP_LLVM_offset_uconst [0xe4]
+;       OFFSET ULEB128(0x08) [0x08]
+; WAVE64-NEXT: .cfi_escape 0x10, 0x41, 0x05, 0x90, 0xa8, 0x14, 0xe4, 0x08
+
+; DW_CFA_expression [0x10] SGPR33 ULEB128(65)=[0x41]
+;   BLOCK_LENGTH ULEB128(5)=[0x05]
+;     DW_OP_regx [0x90]
+;       VGPR40_wave32 ULEB128(1576)=[0xa8, 0x0c]
+;     DW_OP_LLVM_offset_uconst [0xe4]
+;       OFFSET ULEB128(0x08) [0x08]
+; WAVE32-NEXT: .cfi_escape 0x10, 0x41, 0x05, 0x90, 0xa8, 0x0c, 0xe4, 0x08
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; CHECK: s_mov_b32 s33, s32
+; SGPR33 = 65
+; CHECK-NEXT: .cfi_def_cfa_register 65
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; CHECK: s_sub_u32 s32, s32,
+; CHECK-NEXT: v_readlane_b32 s33, v40, 2
+; SGPR32 = 64
+; CHECK-NEXT: .cfi_def_cfa_register 64
+
+; CHECK-NOT: .cfi_{{.*}}
+
+; CHECK: .cfi_endproc
+define hidden void @func_call_clobber() #0 {
+entry:
+  call void @ex() #0
+  ret void
+}
+
 attributes #0 = { nounwind }
+attributes #1 = { nounwind "frame-pointer"="all" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2, !3}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug)
 !1 = !DIFile(filename: "filename", directory: "directory")
 !2 = !{i32 7, !"Dwarf Version", i32 4}
 !3 = !{i32 2, !"Debug Info Version", i32 3}

llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll

 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_writelane_b32 v43, s33, 4
 ; GFX9-NEXT:    s_mov_b32 s33, s32
 ; GFX9-NEXT:    s_add_u32 s32, s32, 0x800
+; GFX9-NEXT:    s_nop 0
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; GFX9-NEXT:    buffer_store_dword v42, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    v_writelane_b32 v43, s34, 0
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, foo@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, foo@gotpcrel32@hi+4
 ; GFX9-NEXT:    v_writelane_b32 v43, s35, 1

llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll

 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN %s
 ; Make sure dbg_value reports something for argument registers when they are split into multiple registers
 
 define hidden <4 x float> @split_v4f32_arg(<4 x float> returned %arg) local_unnamed_addr #0 !dbg !7 {
 ; GCN-LABEL: split_v4f32_arg:
 ; GCN:       .Lfunc_begin0:
 ; GCN-NEXT:    .file 0
 ; GCN-NEXT:    .loc 0 3 0 ; /tmp/dbg.cl:3:0
 ; GCN-NEXT:    .cfi_sections .debug_frame
 ; GCN-NEXT:    .cfi_startproc
 ; GCN-NEXT:  ; %bb.0:
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 96 32] $vgpr3
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 64 32] $vgpr2
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .cfi_llvm_def_aspace_cfa 64, 0, 6
+; GCN-NEXT:    .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:  .Ltmp0:
 ; GCN-NEXT:    .loc 0 4 5 prologue_end ; /tmp/dbg.cl:4:5
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .Ltmp1:
 ; GCN:    .cfi_endproc
   call void @llvm.dbg.value(metadata <4 x float> %arg, metadata !18, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !19
   ret <4 x float> %arg, !dbg !20
 }
 
 define hidden <4 x float> @split_v4f32_multi_arg(<4 x float> %arg0, <2 x float> %arg1) local_unnamed_addr #0 !dbg !21 {
 ; GCN-LABEL: split_v4f32_multi_arg:
 ; GCN:       .Lfunc_begin1:
 ; GCN-NEXT:    .loc 0 7 0 ; /tmp/dbg.cl:7:0
 ; GCN-NEXT:    .cfi_startproc
 ; GCN-NEXT:  ; %bb.0:
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg1 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr5
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg1 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr4
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 96 32] $vgpr3
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 64 32] $vgpr2
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f32_multi_arg:arg0 <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .cfi_llvm_def_aspace_cfa 64, 0, 6
+; GCN-NEXT:    .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
+; GCN-NEXT:    .cfi_undefined 2560
+; GCN-NEXT:    .cfi_undefined 2561
+; GCN-NEXT:    .cfi_undefined 2562
+; GCN-NEXT:    .cfi_undefined 2563
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:  .Ltmp2:
 ; GCN-NEXT:    .loc 0 8 17 prologue_end ; /tmp/dbg.cl:8:17
 ; GCN-NEXT:    v_add_f32_e32 v0, v4, v0
 ; GCN-NEXT:  .Ltmp3:
 ; GCN-NEXT:    v_add_f32_e32 v1, v5, v1
 ; GCN-NEXT:  .Ltmp4:
 ; GCN-NEXT:    v_add_f32_e32 v2, v4, v2

 define hidden <4 x half> @split_v4f16_arg(<4 x half> returned %arg) local_unnamed_addr #0 !dbg !35 {
 ; GCN-LABEL: split_v4f16_arg:
 ; GCN:       .Lfunc_begin2:
 ; GCN-NEXT:    .loc 0 11 0 is_stmt 1 ; /tmp/dbg.cl:11:0
 ; GCN-NEXT:    .cfi_startproc
 ; GCN-NEXT:  ; %bb.0:
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f16_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v4f16_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .cfi_llvm_def_aspace_cfa 64, 0, 6
+; GCN-NEXT:    .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:  .Ltmp8:
 ; GCN-NEXT:    .loc 0 12 5 prologue_end ; /tmp/dbg.cl:12:5
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .Ltmp9:
   call void @llvm.dbg.value(metadata <4 x half> %arg, metadata !42, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !43
   ret <4 x half> %arg, !dbg !44
 }
 
 define hidden double @split_f64_arg(double returned %arg) local_unnamed_addr #0 !dbg !45 {
 ; GCN-LABEL: split_f64_arg:
 ; GCN:       .Lfunc_begin3:
 ; GCN-NEXT:    .loc 0 15 0 ; /tmp/dbg.cl:15:0
 ; GCN-NEXT:    .cfi_startproc
 ; GCN-NEXT:  ; %bb.0:
 ; GCN-NEXT:    ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
 ; GCN-NEXT:    ;DEBUG_VALUE: split_f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .cfi_llvm_def_aspace_cfa 64, 0, 6
+; GCN-NEXT:    .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:  .Ltmp10:
 ; GCN-NEXT:    .loc 0 16 5 prologue_end ; /tmp/dbg.cl:16:5
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .Ltmp11:
 ; GCN:    .cfi_endproc
   call void @llvm.dbg.value(metadata double %arg, metadata !50, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !51
   ret double %arg, !dbg !52
 }
 
 define hidden <2 x double> @split_v2f64_arg(<2 x double> returned %arg) local_unnamed_addr #0 !dbg !53 {
 ; GCN-LABEL: split_v2f64_arg:
 ; GCN:       .Lfunc_begin4:
 ; GCN-NEXT:    .loc 0 19 0 ; /tmp/dbg.cl:19:0
 ; GCN-NEXT:    .cfi_startproc
 ; GCN-NEXT:  ; %bb.0:
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 96 32] $vgpr3
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 64 32] $vgpr2
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
 ; GCN-NEXT:    ;DEBUG_VALUE: split_v2f64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .cfi_llvm_def_aspace_cfa 64, 0, 6
+; GCN-NEXT:    .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:  .Ltmp12:
 ; GCN-NEXT:    .loc 0 20 5 prologue_end ; /tmp/dbg.cl:20:5
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .Ltmp13:
   call void @llvm.dbg.value(metadata <2 x double> %arg, metadata !59, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !60
   ret <2 x double> %arg, !dbg !61
 }
 
 define hidden i64 @split_i64_arg(i64 returned %arg) local_unnamed_addr #0 !dbg !62 {
 ; GCN-LABEL: split_i64_arg:
 ; GCN:       .Lfunc_begin5:
 ; GCN-NEXT:    .loc 0 23 0 ; /tmp/dbg.cl:23:0
 ; GCN-NEXT:    .cfi_startproc
 ; GCN-NEXT:  ; %bb.0:
 ; GCN-NEXT:    ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
 ; GCN-NEXT:    ;DEBUG_VALUE: split_i64_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .cfi_llvm_def_aspace_cfa 64, 0, 6
+; GCN-NEXT:    .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:  .Ltmp14:
 ; GCN-NEXT:    .loc 0 24 5 prologue_end ; /tmp/dbg.cl:24:5
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .Ltmp15:
 ; GCN:    .cfi_endproc
   call void @llvm.dbg.value(metadata i64 %arg, metadata !67, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !68
   ret i64 %arg, !dbg !69
 }
 
 define hidden i8 addrspace(1)* @split_ptr_arg(i8 addrspace(1)* readnone returned %arg) local_unnamed_addr #0 !dbg !70 {
 ; GCN-LABEL: split_ptr_arg:
 ; GCN:       .Lfunc_begin6:
 ; GCN-NEXT:    .loc 0 27 0 ; /tmp/dbg.cl:27:0
 ; GCN-NEXT:    .cfi_startproc
 ; GCN-NEXT:  ; %bb.0:
 ; GCN-NEXT:    ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 32 32] $vgpr1
 ; GCN-NEXT:    ;DEBUG_VALUE: split_ptr_arg:arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef, DW_OP_LLVM_fragment 0 32] $vgpr0
+; GCN-NEXT:    .cfi_llvm_def_aspace_cfa 64, 0, 6
+; GCN-NEXT:    .cfi_escape 0x10, 0x10, 0x08, 0x90, 0x3e, 0x93, 0x04, 0x90, 0x3f, 0x93, 0x04
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:  .Ltmp16:
 ; GCN-NEXT:    .loc 0 28 5 prologue_end ; /tmp/dbg.cl:28:5
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .Ltmp17:
 ; GCN:    .cfi_endproc
   call void @llvm.dbg.value(metadata i8 addrspace(1)* %arg, metadata !76, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !77
   ret i8 addrspace(1)* %arg, !dbg !78

llvm/test/CodeGen/AMDGPU/stack-realign.ll

define amdgpu_kernel void @kernel_call_align4_from_5() {
   %alloca0 = alloca i8, align 1, addrspace(5)
   store volatile i8 2, i8  addrspace(5)* %alloca0
 
   call void @needs_align16_stack_align4(i32 1)
   ret void
 }
 
 ; GCN-LABEL: {{^}}default_realign_align128:
-; GCN: s_add_u32 [[TMP:s[0-9]+]], s32, 0x1fc0
-; GCN-NEXT: s_mov_b32 [[FP_COPY:s[0-9]+]], s33
+; GCN-DAG: s_add_u32 [[TMP:s[0-9]+]], s32, 0x1fc0
+; GCN-DAG: s_mov_b32 [[FP_COPY:s[0-9]+]], s33
 ; GCN-NEXT: s_and_b32 s33, [[TMP]], 0xffffe000
 ; GCN-NEXT: s_add_u32 s32, s32, 0x4000
 ; GCN-NOT: s33
 ; GCN: buffer_store_dword v0, off, s[0:3], s33{{$}}
 ; GCN: s_sub_u32 s32, s32, 0x4000
 ; GCN: s_mov_b32 s33, [[FP_COPY]]
 define void @default_realign_align128(i32 %idx) #0 {
   %alloca.align = alloca i32, align 128, addrspace(5)