[AArch64] Async unwind - function epilogues

[chill]

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

1869

Note to self: remove.

[danielkiss]

LGTM

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

1965–1967

looks it can be removed.

[MaskRay]

LGTM.

[chill]

Committing this change caused failures in the asan/hwasan regression tests. The reason was that emitting unwind info
in epilogues caused the unwind information in case like below:

foo:
  ...
  CSR saves
  ...
  CSR restores
  ret
L:
  ...
  b <somewhere>

to no longer be accidentally correct for the region starting at L. That could be a miscompilation
if a throwing call somehow ends up in that region.

Thus, I've changed the order of the patches to bring CFIFixup before the this patch.

[chill]

Updated on top of latest main, will commit it the following days, based on previous
acceptance, as it contains no non-trivial changes.

[MaskRay]

Note: arm64-shrink-wrapping.ll and merge-store-dependency.ll need rebase.

llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp

120–144

if (!NeedsAsyncDwarfUnwindInfo) {
  const Function &F = MF.getFunction();
  // TODO hasMinSize() is to ...
  NeedsAsyncDwarfUnwindInfo = 
}
return *NeedsAsyncDwarfUnwindInfo;

operator* is more common than getValue. The comment is too long to use the in-line style.

llvm/test/CodeGen/AArch64/framelayout-sve.mir

266

Specify the operands of CFI_INSTRUCTION

343

Specify the operands of CFI_INSTRUCTION

499

Specify the operands of CFI_INSTRUCTION

[chill]

llvm/test/CodeGen/AArch64/framelayout-sve.mir

266

Looks redundant to me. The operands are evident from the ASM and UNWINDINFO checks.

[MaskRay]

llvm/test/CodeGen/AArch64/framelayout-sve.mir

266

SVE has quite complex code paths in the implementation, so it's good to test at least the CFI instruction name to have some coverage. Other operands may be a bit redundant.

llvm/lib/Target/AArch64/AArch64FrameLowering.h

int64_t assignSVEStackObjectOffsets(MachineFrameInfo &MF,
                                       int &MinCSFrameIndex,
                                       int &MaxCSFrameIndex) const;
   bool shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock &MBB,
                                                 unsigned StackBumpBytes) const;
   void emitCalleeSavedGPRLocations(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MBBI) const;
   void emitCalleeSavedSVELocations(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MBBI) const;
+  void emitCalleeSavedGPRRestores(MachineBasicBlock &MBB,
+                                  MachineBasicBlock::iterator MBBI) const;
+  void emitCalleeSavedSVERestores(MachineBasicBlock &MBB,
+                                  MachineBasicBlock::iterator MBBI) const;
 };
 
 } // End llvm namespace
 
 #endif

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

 }
 
 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
   emitCalleeSavedGPRLocations(MBB, MBBI);
   emitCalleeSavedSVELocations(MBB, MBBI);
 }
 
+static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator MBBI,
+                                    bool SVE) {
+  MachineFunction &MF = *MBB.getParent();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+
+  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
+  if (CSI.empty())
+    return;
+
+  const TargetSubtargetInfo &STI = MF.getSubtarget();
+  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
+  const TargetInstrInfo &TII = *STI.getInstrInfo();
+  DebugLoc DL = MBB.findDebugLoc(MBBI);
+
+  for (const auto &Info : CSI) {
+    if (SVE !=
+        (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
+      continue;
+
+    unsigned Reg = Info.getReg();
+    if (SVE &&
+        !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
+      continue;
+
+    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
+        nullptr, TRI.getDwarfRegNum(Info.getReg(), true)));
+    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameDestroy);
+  }
+}
+
+void AArch64FrameLowering::emitCalleeSavedGPRRestores(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
+  emitCalleeSavedRestores(MBB, MBBI, false);
+}
+
+void AArch64FrameLowering::emitCalleeSavedSVERestores(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
+  emitCalleeSavedRestores(MBB, MBBI, true);
+}
+
 // Find a scratch register that we can use at the start of the prologue to
 // re-align the stack pointer.  We avoid using callee-save registers since they
 // may appear to be free when this is called from canUseAsPrologue (during
 // shrink wrapping), but then no longer be free when this is called from
 // emitPrologue.
 //
 // FIXME: This is a bit conservative, since in the above case we could use one
 // of the callee-save registers as a scratch temp to re-align the stack pointer,

 }
 
 // Convert callee-save register save/restore instruction to do stack pointer
 // decrement/increment to allocate/deallocate the callee-save stack area by
 // converting store/load to use pre/post increment version.
 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
-    bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, bool InProlog = true) {
+    bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI,
+    MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup,
+    int CFAOffset = 0) {
   unsigned NewOpc;
   switch (MBBI->getOpcode()) {
   default:
     llvm_unreachable("Unexpected callee-save save/restore opcode!");
   case AArch64::STPXi:
     NewOpc = AArch64::STPXpre;
     break;
   case AArch64::STPDi:
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
   assert(Success && "unknown load/store opcode");
 
   // If the first store isn't right where we want SP then we can't fold the
   // update in so create a normal arithmetic instruction instead.
   MachineFunction  &MF = *MBB.getParent();
   if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 ||
       CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
     emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
-                    StackOffset::getFixed(CSStackSizeInc), TII,
-                    InProlog ? MachineInstr::FrameSetup
-                             : MachineInstr::FrameDestroy,
-                    false, false, nullptr, EmitCFI && InProlog);
+                    StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag,
+                    false, false, nullptr, EmitCFI,
+                    StackOffset::getFixed(CFAOffset));
 
     return std::prev(MBBI);
   }
 
   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
   MIB.addReg(AArch64::SP, RegState::Define);
 
   // Copy all operands other than the immediate offset.
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
   MIB.addImm(CSStackSizeInc / (int)Scale);
 
   MIB.setMIFlags(MBBI->getFlags());
   MIB.setMemRefs(MBBI->memoperands());
 
   // Generate a new SEH code that corresponds to the new instruction.
   if (NeedsWinCFI) {
     *HasWinCFI = true;
-    InsertSEH(*MIB, *TII,
-              InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy);
+    InsertSEH(*MIB, *TII, FrameFlag);
   }
 
-  if (EmitCFI && InProlog) {
+  if (EmitCFI) {
     unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfaOffset(nullptr, -CSStackSizeInc));
+        MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset - CSStackSizeInc));
     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
         .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+        .setMIFlags(FrameFlag);
   }
 
   return std::prev(MBB.erase(MBBI));
 }
 
 // Fixup callee-save register save/restore instructions to take into account
 // combined SP bump by adding the local stack size to the stack offsets.
 static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII,
                                         const DebugLoc &DL) {
   // Shadow call stack epilog: ldr x30, [x18, #-8]!
   BuildMI(MBB, MBBI, DL, TII.get(AArch64::LDRXpre))
       .addReg(AArch64::X18, RegState::Define)
       .addReg(AArch64::LR, RegState::Define)
       .addReg(AArch64::X18)
       .addImm(-8)
       .setMIFlag(MachineInstr::FrameDestroy);
+
+  if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncUnwindInfo()) {
+    unsigned CFIIndex =
+        MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, 18));
+    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameDestroy);
+  }
 }
 
 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
                                         MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   const Function &F = MF.getFunction();
   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   DebugLoc DL;
 
   const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
   if (needsShadowCallStackPrologueEpilogue(MF))
     emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI,
                                 MFnI.needsUnwindInfo());
 
   if (MFnI.shouldSignReturnAddress()) {
-
     unsigned PACI;
     if (MFnI.shouldSignWithBKey()) {
       BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
           .setMIFlag(MachineInstr::FrameSetup);
       PACI = Subtarget.hasPAuth() ? AArch64::PACIB : AArch64::PACIBSP;
     } else {
       PACI = Subtarget.hasPAuth() ? AArch64::PACIA : AArch64::PACIASP;
     }

 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
                                         MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   DebugLoc DL;
   bool NeedsWinCFI = needsWinCFI(MF);
+  bool EmitCFI = MF.getInfo<AArch64FunctionInfo>()->needsAsyncUnwindInfo();
   bool HasWinCFI = false;
   bool IsFunclet = false;
   auto WinCFI = make_scope_exit([&]() { assert(HasWinCFI == MF.hasWinCFI()); });
 
   if (MBB.end() != MBBI) {
     DL = MBBI->getDebugLoc();
     IsFunclet = isFuncletReturnInstr(*MBBI);
   }
 
-  auto ShadowStackEpilogue = make_scope_exit([&]() {
+  auto ShadowStackEpilogueAndCSRRestores = make_scope_exit([&]() {
     if (needsShadowCallStackPrologueEpilogue(MF))
       emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL);
+    if (EmitCFI)
+      emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
   });
 
   int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
                                : MFI.getStackSize();
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
 
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.
   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     return;
 
   // How much of the stack used by incoming arguments this function is expected
   // to restore in this particular epilogue.
   int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
 
-  // The stack frame should be like below,
-  //
-  //      ----------------------                     ---
-  //      |                    |                      |
-  //      | BytesInStackArgArea|              CalleeArgStackSize
-  //      | (NumReusableBytes) |                (of tail call)
-  //      |                    |                     ---
-  //      |                    |                      |
-  //      ---------------------|        ---           |
-  //      |                    |         |            |
-  //      |   CalleeSavedReg   |         |            |
-  //      | (CalleeSavedStackSize)|      |            |
-  //      |                    |         |            |
-  //      ---------------------|         |         NumBytes
-  //      |                    |     StackSize  (StackAdjustUp)
-  //      |   LocalStackSize   |         |            |
-  //      | (covering callee   |         |            |
-  //      |       args)        |         |            |
-  //      |                    |         |            |
-  //      ----------------------        ---          ---
-  //
-  // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
-  //             = StackSize + ArgumentPopSize
-  //
-  // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
-  // it as the 2nd argument of AArch64ISD::TC_RETURN.
-
   auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
 
   bool IsWin64 =
       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
   unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
 
   int64_t AfterCSRPopSize = ArgumentStackToRestore;
   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
if (homogeneousPrologEpilog(MF, &MBB)) {
     // SP has been already adjusted while restoring callee save regs.
     // We've bailed-out the case with adjusting SP for arguments.
     assert(AfterCSRPopSize == 0);
     return;
   }
   bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
   // Assume we can't combine the last pop with the sp restore.
 
+  bool CombineAfterCSRBump = false;
   if (!CombineSPBump && PrologueSaveSize != 0) {
     MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
-    while (AArch64InstrInfo::isSEHInstruction(*Pop))
+    while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
+           AArch64InstrInfo::isSEHInstruction(*Pop))
       Pop = std::prev(Pop);
     // Converting the last ldp to a post-index ldp is valid only if the last
     // ldp's offset is 0.
     const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
     // If the offset is 0 and the AfterCSR pop is not actually trying to
     // allocate more stack for arguments (in space that an untimely interrupt
     // may clobber), convert it to a post-index ldp.
-    if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0)
+    if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
       convertCalleeSaveRestoreToSPPrePostIncDec(
           MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
-          false, false);
-    else {
+          EmitCFI, MachineInstr::FrameDestroy, PrologueSaveSize);
+    } else {
       // If not, make sure to emit an add after the last ldp.
       // We're doing this by transfering the size to be restored from the
       // adjustment *before* the CSR pops to the adjustment *after* the CSR
       // pops.
       AfterCSRPopSize += PrologueSaveSize;
+      CombineAfterCSRBump = true;
     }
   }
 
   // Move past the restores of the callee-saved registers.
   // If we plan on combining the sp bump of the local stack size and the callee
   // save stack size, we might need to adjust the CSR save and restore offsets.
   MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
   MachineBasicBlock::iterator Begin = MBB.begin();
BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
         .setMIFlag(MachineInstr::FrameDestroy);
   }
 
   const StackOffset &SVEStackSize = getSVEStackSize(MF);
 
   // If there is a single SP update, insert it before the ret and we're done.
   if (CombineSPBump) {
     assert(!SVEStackSize && "Cannot combine SP bump with SVE");
+
+    // When we are about to restore the CSRs, the CFA register is SP again.
+    if (EmitCFI && hasFP(MF)) {
+      const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
+      unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
+      unsigned CFIIndex = MF.addFrameInst(
+          MCCFIInstruction::cfiDefCfa(nullptr, Reg, NumBytes));
+      BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+          .addCFIIndex(CFIIndex)
+          .setMIFlags(MachineInstr::FrameDestroy);
+    }
+
+
     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
                     TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
-                    &HasWinCFI);
+                    &HasWinCFI, EmitCFI,
+                    StackOffset::getFixed(NumBytes));
     if (HasWinCFI)
       BuildMI(MBB, MBB.getFirstTerminator(), DL,
               TII->get(AArch64::SEH_EpilogEnd))
           .setMIFlag(MachineInstr::FrameDestroy);
+    // if (EmitCFI)
+    //   emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
     return;
   }
 
   NumBytes -= PrologueSaveSize;
   assert(NumBytes >= 0 && "Negative stack allocation size!?");
 
   // Process the SVE callee-saves to determine what space needs to be
   // deallocated.
StackOffset CalleeSavedSizeAsOffset =
         StackOffset::getScalable(CalleeSavedSize);
     DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
     DeallocateAfter = CalleeSavedSizeAsOffset;
   }
 
   // Deallocate the SVE area.
   if (SVEStackSize) {
     if (AFI->isStackRealigned()) {
-      if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize())
+      if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
         // Set SP to start of SVE callee-save area from which they can
         // be reloaded. The code below will deallocate the stack space
         // space by moving FP -> SP.
         emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
                         StackOffset::getScalable(-CalleeSavedSize), TII,
                         MachineInstr::FrameDestroy);
+      }
     } else {
       if (AFI->getSVECalleeSavedStackSize()) {
         // Deallocate the non-SVE locals first before we can deallocate (and
         // restore callee saves) from the SVE area.
-        emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
-                        StackOffset::getFixed(NumBytes), TII,
-                        MachineInstr::FrameDestroy);
+        emitFrameOffset(
+            MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
+            StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy,
+            false, false, nullptr, EmitCFI && !hasFP(MF),
+            SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
         NumBytes = 0;
       }
 
       emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
-                      DeallocateBefore, TII, MachineInstr::FrameDestroy);
+                      DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
+                      false, nullptr, EmitCFI && !hasFP(MF),
+                      SVEStackSize +
+                          StackOffset::getFixed(NumBytes + PrologueSaveSize));
 
       emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
-                      DeallocateAfter, TII, MachineInstr::FrameDestroy);
+                      DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
+                      false, nullptr, EmitCFI && !hasFP(MF),
+                      DeallocateAfter +
+                          StackOffset::getFixed(NumBytes + PrologueSaveSize));
     }
+    if (EmitCFI)
+      emitCalleeSavedSVERestores(MBB, RestoreEnd);
   }
 
   if (!hasFP(MF)) {
     bool RedZone = canUseRedZone(MF);
     // If this was a redzone leaf function, we don't need to restore the
     // stack pointer (but we may need to pop stack args for fastcc).
     if (RedZone && AfterCSRPopSize == 0)
       return;
 
+    // Pop the local variables off the stack. If there are no callee-saved
+    // registers, it means we are actually positioned at the terminator and can
+    // combine stack increment for the locals and the stack increment for
+    // callee-popped arguments into (possibly) a single instruction and be done.
     bool NoCalleeSaveRestore = PrologueSaveSize == 0;
     int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
     if (NoCalleeSaveRestore)
       StackRestoreBytes += AfterCSRPopSize;
 
-    emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
-                    StackOffset::getFixed(StackRestoreBytes), TII,
-                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
+    emitFrameOffset(
+        MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
+        MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI,
+        StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize));
+
     // If we were able to combine the local stack pop with the argument pop,
     // then we're done.
     if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
       if (HasWinCFI) {
         BuildMI(MBB, MBB.getFirstTerminator(), DL,
                 TII->get(AArch64::SEH_EpilogEnd))
             .setMIFlag(MachineInstr::FrameDestroy);
       }
+
+      // if (EmitCFI)
+      //   emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
+
       return;
     }
 
     NumBytes = 0;
   }
 
   // Restore the original stack pointer.
   // FIXME: Rather than doing the math here, we should instead just use
   // non-post-indexed loads for the restores if we aren't actually going to
   // be able to save any instructions.
   if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
     emitFrameOffset(
         MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
         StackOffset::getFixed(-AFI->getCalleeSaveBaseToFrameRecordOffset()),
         TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
   } else if (NumBytes)
     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(NumBytes), TII,
                     MachineInstr::FrameDestroy, false, NeedsWinCFI);
 
+  // When we are about to restore the CSRs, the CFA register is SP again.
+  if (EmitCFI && hasFP(MF)) {
+    const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
+    unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
+    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
+        nullptr, Reg, PrologueSaveSize));
+    BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameDestroy);
+  }
+
   // This must be placed after the callee-save restore code because that code
   // assumes the SP is at the same location as it was after the callee-save save
   // code in the prologue.
   if (AfterCSRPopSize) {
     assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
                                   "interrupt may have clobbered");
 
     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(AfterCSRPopSize), TII,
-                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
+                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI,
+                    EmitCFI, StackOffset::getFixed(CombineAfterCSRBump ? PrologueSaveSize : 0));
   }
   if (HasWinCFI)
     BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
         .setMIFlag(MachineInstr::FrameDestroy);
+  // if (EmitCFI)
+  //   emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
 }
 
 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
 /// debug info.  It's the same as what we use for resolving the code-gen
 /// references for now.  FIXME: This can go wrong when references are
 /// SP-relative and simple call frames aren't used.
 StackOffset
 AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
MIB.addReg(Reg1, getDefRegState(true))
         .addImm(RPI.Offset) // [sp, #offset*scale]
                             // where factor*scale is implicit
         .setMIFlag(MachineInstr::FrameDestroy);
     MIB.addMemOperand(MF.getMachineMemOperand(
         MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
         MachineMemOperand::MOLoad, Size, Alignment));
     if (NeedsWinCFI)
       InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
+
     return MIB->getIterator();
   };
 
   // SVE objects are always restored in reverse order.
   for (const RegPairInfo &RPI : reverse(RegPairs))
     if (RPI.isScalable())
       EmitMI(RPI);
 

llvm/lib/Target/AArch64/AArch64InstrInfo.h

 
 /// Return true if there is an instruction /after/ \p DefMI and before \p UseMI
 /// which either reads or clobbers NZCV.
 bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
                                      const MachineInstr &UseMI,
                                      const TargetRegisterInfo *TRI);
 
 MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg,
-                              unsigned Reg, const StackOffset &Offset);
+                              unsigned Reg, const StackOffset &Offset,
+                              bool LastAdjustmentWasScalable = true);
 MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,
                                  const StackOffset &OffsetFromDefCFA);
 
 /// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg
 /// plus Offset.  This is intended to be used from within the prolog/epilog
 /// insertion (PEI) pass, where a virtual scratch register may be allocated
 /// if necessary, to be replaced by the scavenger at the end of PEI.
 void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
   DefCfaExpr.append(buffer, buffer + encodeULEB128(Expr.size(), buffer));
   DefCfaExpr.append(Expr.str());
   return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(),
                                         Comment.str());
 }
 
 MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
                                     unsigned FrameReg, unsigned Reg,
-                                    const StackOffset &Offset) {
+                                    const StackOffset &Offset,
+                                    bool LastAdjustmentWasScalable) {
   if (Offset.getScalable())
     return createDefCFAExpression(TRI, Reg, Offset);
 
-  if (FrameReg == Reg)
+  if (FrameReg == Reg && !LastAdjustmentWasScalable)
     return MCCFIInstruction::cfiDefCfaOffset(nullptr, int(Offset.getFixed()));
 
   unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
   return MCCFIInstruction::cfiDefCfa(nullptr, DwarfReg, (int)Offset.getFixed());
 }
 
 MCCFIInstruction llvm::createCFAOffset(const TargetRegisterInfo &TRI,
                                        unsigned Reg,
do {
     else
       CFAOffset -= Change;
     if (EmitCFAOffset && DestReg == TmpReg) {
       MachineFunction &MF = *MBB.getParent();
       const TargetSubtargetInfo &STI = MF.getSubtarget();
       const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
 
       unsigned CFIIndex =
-          MF.addFrameInst(createDefCFA(TRI, FrameReg, DestReg, CFAOffset));
+          MF.addFrameInst(createDefCFA(TRI, FrameReg, DestReg, CFAOffset, VScale != 1));
       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
           .addCFIIndex(CFIIndex)
           .setMIFlags(Flag);
     }
 
     if (NeedsWinCFI) {
       assert(Sign == 1 && "SEH directives should always have a positive sign");
       int Imm = (int)(ThisVal << LocalShiftSize);

llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp

 }
 
 bool AArch64FunctionInfo::shouldSignReturnAddress() const {
   return shouldSignReturnAddress(llvm::any_of(
       MF.getFrameInfo().getCalleeSavedInfo(),
       [](const auto &Info) { return Info.getReg() == AArch64::LR; }));
 }
 
 bool AArch64FunctionInfo::needsUnwindInfo() const {
   if (!NeedsUnwindInfo.hasValue())
     NeedsUnwindInfo = MF.needsFrameMoves() &&
                       !MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
 
   return NeedsUnwindInfo.getValue();
 }
 
 bool AArch64FunctionInfo::needsAsyncUnwindInfo() const {
   if (!NeedsAsyncUnwindInfo.hasValue())
-    NeedsAsyncUnwindInfo = MF.needsFrameMoves() &&
-                           !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
-                           !MF.getSubtarget<AArch64Subtarget>().isTargetMachO();
+    NeedsAsyncUnwindInfo =
+        MF.needsFrameMoves() &&
+        !MF.getSubtarget<AArch64Subtarget>().isTargetMachO() &&
+        !MF.getFunction()
+             .hasMinSize(); // TODO: this is to prevent epilogue unwind info
+                            // from being emitted for homogeneous epilogues,
+                            // outlined functions, and functions outlined from.
+                            // Alternatively, we could disable those
+                            // optimisations. Or eve better, add async unwind
+                            // support to them!
 
   assert((!NeedsAsyncUnwindInfo.getValue() || needsUnwindInfo()) &&
          "Async unwind info must imply unwind info");
   return NeedsAsyncUnwindInfo.getValue();
 }

llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll

 ; CHECK-CAS-O0-NEXT:    caspa x2, x3, x4, x5, [x0]
 ; CHECK-CAS-O0-NEXT:    mov x9, x2
 ; CHECK-CAS-O0-NEXT:    mov x8, x3
 ; CHECK-CAS-O0-NEXT:    // implicit-def: $q0
 ; CHECK-CAS-O0-NEXT:    mov v0.d[0], x9
 ; CHECK-CAS-O0-NEXT:    mov v0.d[1], x8
 ; CHECK-CAS-O0-NEXT:    str q0, [x0]
 ; CHECK-CAS-O0-NEXT:    add sp, sp, #16
+; CHECK-CAS-O0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-CAS-O0-NEXT:    ret
 
 %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire
   %val = extractvalue { i128, i1 } %pair, 0
   store i128 %val, i128* %p
   ret void
 }
 

 ; CHECK-CAS-O0-NEXT:    caspal x2, x3, x4, x5, [x0]
 ; CHECK-CAS-O0-NEXT:    mov x9, x2
 ; CHECK-CAS-O0-NEXT:    mov x8, x3
 ; CHECK-CAS-O0-NEXT:    // implicit-def: $q0
 ; CHECK-CAS-O0-NEXT:    mov v0.d[0], x9
 ; CHECK-CAS-O0-NEXT:    mov v0.d[1], x8
 ; CHECK-CAS-O0-NEXT:    str q0, [x0]
 ; CHECK-CAS-O0-NEXT:    add sp, sp, #16
+; CHECK-CAS-O0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-CAS-O0-NEXT:    ret
 
   %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval monotonic seq_cst
   %val = extractvalue { i128, i1 } %pair, 0
   store i128 %val, i128* %p
   ret void
 }
 

 ; CHECK-CAS-O0-NEXT:    caspal x2, x3, x4, x5, [x0]
 ; CHECK-CAS-O0-NEXT:    mov x9, x2
 ; CHECK-CAS-O0-NEXT:    mov x8, x3
 ; CHECK-CAS-O0-NEXT:    // implicit-def: $q0
 ; CHECK-CAS-O0-NEXT:    mov v0.d[0], x9
 ; CHECK-CAS-O0-NEXT:    mov v0.d[1], x8
 ; CHECK-CAS-O0-NEXT:    str q0, [x0]
 ; CHECK-CAS-O0-NEXT:    add sp, sp, #16
+; CHECK-CAS-O0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-CAS-O0-NEXT:    ret
 
   %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release acquire
   %val = extractvalue { i128, i1 } %pair, 0
   store i128 %val, i128* %p
   ret void
 }
 

 ; CHECK-CAS-O0-NEXT:    caspal x2, x3, x4, x5, [x0]
 ; CHECK-CAS-O0-NEXT:    mov x9, x2
 ; CHECK-CAS-O0-NEXT:    mov x8, x3
 ; CHECK-CAS-O0-NEXT:    // implicit-def: $q0
 ; CHECK-CAS-O0-NEXT:    mov v0.d[0], x9
 ; CHECK-CAS-O0-NEXT:    mov v0.d[1], x8
 ; CHECK-CAS-O0-NEXT:    str q0, [x0]
 ; CHECK-CAS-O0-NEXT:    add sp, sp, #16
+; CHECK-CAS-O0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-CAS-O0-NEXT:    ret
   %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release acquire
   %val = extractvalue { i128, i1 } %pair, 0
   store i128 %val, i128* %p
   ret void
 }
 
 define void @atomic_load_relaxed(i64, i64, i128* %p, i128* %p2) {

llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    str w8, [sp]
 ; CHECK-NEXT:    bl byval_i32
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call void @byval_i32(i32* byval(i32) %incoming)
   ret void
 }
 
 declare void @byval_a64i32([64 x i32]* byval([64 x i32]) %ptr)
 
 define void @call_byval_a64i32([64 x i32]* %incoming) {

 ; CHECK-NEXT:    str q0, [sp, #192]
 ; CHECK-NEXT:    ldr q0, [x0, #208]
 ; CHECK-NEXT:    str q0, [sp, #208]
 ; CHECK-NEXT:    ldr q0, [x0, #224]
 ; CHECK-NEXT:    str q0, [sp, #224]
 ; CHECK-NEXT:    ldr q0, [x0, #240]
 ; CHECK-NEXT:    str q0, [sp, #240]
 ; CHECK-NEXT:    bl byval_a64i32
+; CHECK-NEXT:    .cfi_def_cfa wsp, 288
 ; CHECK-NEXT:    ldr x28, [sp, #272] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #288
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   call void @byval_a64i32([64 x i32]* byval([64 x i32]) %incoming)
   ret void
 }

llvm/test/CodeGen/AArch64/aarch64-be-bv.ll

 ; CHECK-NEXT:    movi v0.4s, #3
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.4s, #2
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 8, i8 0, i8 0, i8 0, i8 8, i8 0, i8 0, i8 0>)
   call i16 @f_v4i16(<4 x i16> <i16 7, i16 0, i16 7, i16 0>)
   call i32 @f_v2i32(<2 x i32> <i32 6, i32 6>)
   call i64 @f_v1i64(<1 x i64> <i64 21474836485>)
   call i8 @f_v16i8(<16 x i8> <i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0>)
   call i16 @f_v8i16(<8 x i16> <i16 4, i16 0, i16 4, i16 0, i16 4, i16 0, i16 4, i16 0>)
   call i32 @f_v4i32(<4 x i32> <i32 3, i32 3, i32 3, i32 3>)

 ; CHECK-NEXT:    movi v0.4s, #3, lsl #8
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.4s, #2, lsl #8
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 0, i8 8, i8 0, i8 0, i8 0, i8 8, i8 0, i8 0>)
   call i16 @f_v4i16(<4 x i16> <i16 1792, i16 0, i16 1792, i16 0>)
   call i32 @f_v2i32(<2 x i32> <i32 1536, i32 1536>)
   call i64 @f_v1i64(<1 x i64> <i64 5497558140160>)
   call i8 @f_v16i8(<16 x i8> <i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0>)
   call i16 @f_v8i16(<8 x i16> <i16 1024, i16 0, i16 1024, i16 0, i16 1024, i16 0, i16 1024, i16 0>)
   call i32 @f_v4i32(<4 x i32> <i32 768, i32 768, i32 768, i32 768>)

 ; CHECK-NEXT:    movi v0.4s, #3, lsl #16
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.4s, #2, lsl #16
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 0, i8 0, i8 8, i8 0, i8 0, i8 0, i8 8, i8 0>)
   call i16 @f_v4i16(<4 x i16> <i16 0, i16 7, i16 0, i16 7>)
   call i32 @f_v2i32(<2 x i32> <i32 393216, i32 393216>)
   call i64 @f_v1i64(<1 x i64> <i64 1407374883880960>)
   call i8 @f_v16i8(<16 x i8> <i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0>)
   call i16 @f_v8i16(<8 x i16> <i16 0, i16 4, i16 0, i16 4, i16 0, i16 4, i16 0, i16 4>)
   call i32 @f_v4i32(<4 x i32> <i32 196608, i32 196608, i32 196608, i32 196608>)

 ; CHECK-NEXT:    movi v0.4s, #3, lsl #24
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.4s, #2, lsl #24
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 0, i8 0, i8 0, i8 8, i8 0, i8 0, i8 0, i8 8>)
   call i16 @f_v4i16(<4 x i16> <i16 0, i16 1792, i16 0, i16 1792>)
   call i32 @f_v2i32(<2 x i32> <i32 100663296, i32 100663296>)
   call i64 @f_v1i64(<1 x i64> <i64 360287970273525760>)
   call i8 @f_v16i8(<16 x i8> <i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5, i8 0, i8 0, i8 0, i8 5>)
   call i16 @f_v8i16(<8 x i16> <i16 0, i16 1024, i16 0, i16 1024, i16 0, i16 1024, i16 0, i16 1024>)
   call i32 @f_v4i32(<4 x i32> <i32 50331648, i32 50331648, i32 50331648, i32 50331648>)

 ; CHECK-NEXT:    movi v0.8h, #3
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.8h, #2
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 8, i8 0, i8 8, i8 0, i8 8, i8 0, i8 8, i8 0>)
   call i16 @f_v4i16(<4 x i16> <i16 7, i16 7, i16 7, i16 7>)
   call i32 @f_v2i32(<2 x i32> <i32 393222, i32 393222>)
   call i64 @f_v1i64(<1 x i64> <i64 1407396358717445>)
   call i8 @f_v16i8(<16 x i8> <i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0>)
   call i16 @f_v8i16(<8 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4>)
   call i32 @f_v4i32(<4 x i32> <i32 196611, i32 196611, i32 196611, i32 196611>)

 ; CHECK-NEXT:    movi v0.8h, #3, lsl #8
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.8h, #2, lsl #8
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 0, i8 8, i8 0, i8 8, i8 0, i8 8, i8 0, i8 8>)
   call i16 @f_v4i16(<4 x i16> <i16 1792, i16 1792, i16 1792, i16 1792>)
   call i32 @f_v2i32(<2 x i32> <i32 100664832, i32 100664832>)
   call i64 @f_v1i64(<1 x i64> <i64 360293467831665920>)
   call i8 @f_v16i8(<16 x i8> <i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5, i8 0, i8 5>)
   call i16 @f_v8i16(<8 x i16> <i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024, i16 1024>)
   call i32 @f_v4i32(<4 x i32> <i32 50332416, i32 50332416, i32 50332416, i32 50332416>)

 ; CHECK-NEXT:    movi v0.4s, #3, msl #8
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.4s, #2, msl #8
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 255, i8 8, i8 0, i8 0, i8 255, i8 8, i8 0, i8 0>)
   call i16 @f_v4i16(<4 x i16> <i16 2047, i16 0, i16 2047, i16 0>)
   call i32 @f_v2i32(<2 x i32> <i32 1791, i32 1791>)
   call i64 @f_v1i64(<1 x i64> <i64 6592774800895>)
   call i8 @f_v16i8(<16 x i8> <i8 255, i8 5, i8 0, i8 0, i8 255, i8 5, i8 0, i8 0, i8 255, i8 5, i8 0, i8 0, i8 255, i8 5, i8 0, i8 0>)
   call i16 @f_v8i16(<8 x i16> <i16 1279, i16 0, i16 1279, i16 0, i16 1279, i16 0, i16 1279, i16 0>)
   call i32 @f_v4i32(<4 x i32> <i32 1023, i32 1023, i32 1023, i32 1023>)

 ; CHECK-NEXT:    movi v0.4s, #3, msl #16
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    movi v0.4s, #2, msl #16
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 255, i8 255, i8 8, i8 0, i8 255, i8 255, i8 8, i8 0>)
   call i16 @f_v4i16(<4 x i16> <i16 65535, i16 7, i16 65535, i16 7>)
   call i32 @f_v2i32(<2 x i32> <i32 458751, i32 458751>)
   call i64 @f_v1i64(<1 x i64> <i64 1688845565689855>)
   call i8 @f_v16i8(<16 x i8> <i8 255, i8 255, i8 5, i8 0, i8 255, i8 255, i8 5, i8 0, i8 255, i8 255, i8 5, i8 0, i8 255, i8 255, i8 5, i8 0>)
   call i16 @f_v8i16(<8 x i16> <i16 65535, i16 4, i16 65535, i16 4, i16 65535, i16 4, i16 65535, i16 4>)
   call i32 @f_v4i32(<4 x i32> <i32 262143, i32 262143, i32 262143, i32 262143>)

 ; CHECK-NEXT:    rev64 v0.8h, v0.8h
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v8i16
 ; CHECK-NEXT:    movi v0.16b, #3
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>)
   call i16 @f_v4i16(<4 x i16> <i16 1799, i16 1799, i16 1799, i16 1799>)
   call i32 @f_v2i32(<2 x i32> <i32 101058054, i32 101058054>)
   call i8 @f_v16i8(<16 x i8> <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>)
   call i16 @f_v8i16(<8 x i16> <i16 1028, i16 1028, i16 1028, i16 1028, i16 1028, i16 1028, i16 1028, i16 1028>)
   call i32 @f_v4i32(<4 x i32> <i32 50529027, i32 50529027, i32 50529027, i32 50529027>)
 

 ; CHECK-NEXT:    rev64 v0.8h, v0.8h
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v8i16
 ; CHECK-NEXT:    movi v0.2d, #0xffffffff00000000
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 -1, i8 0, i8 0, i8 0, i8 -1, i8 0, i8 0, i8 0>)
   call i16 @f_v4i16(<4 x i16> <i16 -1, i16 0, i16 -1, i16 0>)
   call i32 @f_v2i32(<2 x i32> <i32 -1, i32 -1>)
   call i8 @f_v16i8(<16 x i8> <i8 -1, i8 -1, i8 -1, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 -1, i8 -1, i8 -1, i8 0>)
   call i16 @f_v8i16(<8 x i16> <i16 0, i16 -1, i16 -1, i16 -1, i16 0, i16 -1, i16 -1, i16 -1>)
   call i32 @f_v4i32(<4 x i32> <i32 0, i32 -1, i32 0, i32 -1>)
 

 ; CHECK-NEXT:    fmov v0.4s, #2.75000000
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    fmov v0.4s, #2.50000000
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v2i64
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v8i8(<8 x i8> <i8 0, i8 0, i8 128, i8 64, i8 0, i8 0, i8 128, i8 64>)
   call i16 @f_v4i16(<4 x i16> <i16 0, i16 16496, i16 0, i16 16496>)
   call i32 @f_v2i32(<2 x i32> <i32 1080033280, i32 1080033280>)
   call i64 @f_v1i64(<1 x i64> <i64 4523865826746957824>)
   call i8 @f_v16i8(<16 x i8> <i8 0, i8 0, i8 80, i8 64, i8 0, i8 0, i8 80, i8 64, i8 0, i8 0, i8 80, i8 64, i8 0, i8 0, i8 80, i8 64>)
   call i16 @f_v8i16(<8 x i16> <i16 0, i16 16448, i16 0, i16 16448, i16 0, i16 16448, i16 0, i16 16448>)
   call i32 @f_v4i32(<4 x i32> <i32 1076887552, i32 1076887552, i32 1076887552, i32 1076887552>)

 ; CHECK-NEXT:    rev64 v0.8h, v0.8h
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v8i16
 ; CHECK-NEXT:    fmov v0.2d, #0.17187500
 ; CHECK-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    bl f_v4i32
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call i8 @f_v16i8(<16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 200, i8 63, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 200, i8 63>)
   call i16 @f_v8i16(<8 x i16> <i16 0, i16 0, i16 0, i16 16327, i16 0, i16 0, i16 0, i16 16327>)
   call i32 @f_v4i32(<4 x i32> <i32 0, i32 1069940736, i32 0, i32 1069940736>)
 
   ret void
 }

llvm/test/CodeGen/AArch64/aarch64-load-ext.ll

 ; CHECK-LE:       // %bb.0:
 ; CHECK-LE-NEXT:    sub sp, sp, #16
 ; CHECK-LE-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-LE-NEXT:    str w0, [sp, #12]
 ; CHECK-LE-NEXT:    ldr s0, [sp, #12]
 ; CHECK-LE-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-LE-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-LE-NEXT:    add sp, sp, #16
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-LE-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: bitcast:
 ; CHECK-BE:       // %bb.0:
 ; CHECK-BE-NEXT:    sub sp, sp, #16
 ; CHECK-BE-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-BE-NEXT:    str w0, [sp, #12]
 ; CHECK-BE-NEXT:    ldr s0, [sp, #12]
 ; CHECK-BE-NEXT:    rev32 v0.8b, v0.8b
 ; CHECK-BE-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    rev64 v0.4h, v0.4h
 ; CHECK-BE-NEXT:    add sp, sp, #16
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-BE-NEXT:    ret
   %2 = bitcast i32 %0 to <4 x i8>
   ret <4 x i8> %2
 }

llvm/test/CodeGen/AArch64/addsub-constant-folding.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    add w0, w19, #10
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = add i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_add_const_add_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    movi v0.4s, #10
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = add <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = add <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_add_const_add_const_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    add w0, w19, #6
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_add_const_sub_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    movi v0.4s, #6
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = add <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_add_const_sub_const_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #-6
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 2, %t0
   ret i32 %t1
 }
 
 define <4 x i32> @vec_add_const_const_sub(<4 x i32> %arg) {

 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    mvni v0.4s, #5
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = add <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> <i32 2, i32 2, i32 2, i32 2>, %t0
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_add_const_const_sub_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    sub w0, w19, #6
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = add i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_sub_const_add_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    mvni v0.4s, #5
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = add <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_sub_const_add_const_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    sub w0, w19, #10
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_sub_const_sub_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    movi v0.4s, #10
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_sub_const_sub_const_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #10
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 2, %t0
   ret i32 %t1
 }
 
 define <4 x i32> @vec_sub_const_const_sub(<4 x i32> %arg) {

 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    movi v0.4s, #2
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> <i32 2, i32 2, i32 2, i32 2>, %t0
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_sub_const_const_sub_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w8, w0
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #10
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
   %t1 = add i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_const_sub_add_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    movi v0.4s, #10
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> <i32 8, i32 8, i32 8, i32 8>, %arg
   call void @vec_use(<4 x i32> %t0)
   %t1 = add <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_const_sub_add_const_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w8, w0
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #6
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
   %t1 = sub i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_const_sub_sub_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    movi v0.4s, #6
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> <i32 8, i32 8, i32 8, i32 8>, %arg
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_const_sub_sub_const_nonsplat(<4 x i32> %arg) {

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    sub w19, w8, w0
 ; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #2
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
   %t1 = sub i32 2, %t0
   ret i32 %t1
 }
 
 define <4 x i32> @vec_const_sub_const_sub(<4 x i32> %arg) {

 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    movi v0.4s, #2
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> <i32 8, i32 8, i32 8, i32 8>, %arg
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> <i32 2, i32 2, i32 2, i32 2>, %t0
   ret <4 x i32> %t1
 }
 
 define <4 x i32> @vec_const_sub_const_sub_nonsplat(<4 x i32> %arg) {

llvm/test/CodeGen/AArch64/argument-blocks-array-of-struct.ll

 ; CHECK-NEXT:    add x8, x8, :lo12:in_block_store
 ; CHECK-NEXT:    str d0, [x8]
 ; CHECK-NEXT:    str d1, [x8, #8]
 ; CHECK-NEXT:    str d2, [x8, #16]
 ; CHECK-NEXT:    str d3, [x8, #24]
 ; CHECK-NEXT:    str d4, [x8, #32]
 ; CHECK-NEXT:    str d5, [x8, #40]
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = call %T_IN_BLOCK @return_in_block()
   store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store
   ret void
 }
 
 define void @callee_in_block(%T_IN_BLOCK %a) {
 ; CHECK-LABEL: callee_in_block:

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x8, in_block_store
 ; CHECK-NEXT:    add x8, x8, :lo12:in_block_store
 ; CHECK-NEXT:    ldp d4, d5, [x8, #32]
 ; CHECK-NEXT:    ldp d2, d3, [x8, #16]
 ; CHECK-NEXT:    ldp d0, d1, [x8]
 ; CHECK-NEXT:    bl callee_in_block
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = load %T_IN_BLOCK, %T_IN_BLOCK* @in_block_store
   call void @callee_in_block(%T_IN_BLOCK %1)
   ret void
 }
 
 %T_IN_MEMORY = type [ 3 x { double, { double, double } } ]
 

 ; CHECK-NEXT:    ldur q2, [sp, #56]
 ; CHECK-NEXT:    ldur q3, [sp, #40]
 ; CHECK-NEXT:    ldr d4, [sp, #72]
 ; CHECK-NEXT:    stp q1, q0, [x8]
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    stp q3, q2, [x8, #32]
 ; CHECK-NEXT:    str d4, [x8, #64]
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = call %T_IN_MEMORY @return_in_memory()
   store %T_IN_MEMORY %1, %T_IN_MEMORY* @in_memory_store
   ret void
 }
 
 define void @callee_in_memory(%T_IN_MEMORY %a) {
 ; CHECK-LABEL: callee_in_memory:

 ; CHECK-NEXT:    ldp q2, q3, [x8, #32]
 ; CHECK-NEXT:    stp q0, q1, [sp]
 ; CHECK-NEXT:    ldr d4, [x8, #64]
 ; CHECK-NEXT:    stp q2, q3, [sp, #32]
 ; CHECK-NEXT:    str d4, [sp, #64]
 ; CHECK-NEXT:    bl callee_in_memory
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = load %T_IN_MEMORY, %T_IN_MEMORY* @in_memory_store
   call void @callee_in_memory(%T_IN_MEMORY %1)
   ret void
 }
 
 %T_NO_BLOCK = type [ 2 x { double, { i32 } } ]
 

 ; CHECK-NEXT:    bl return_no_block
 ; CHECK-NEXT:    adrp x8, no_block_store
 ; CHECK-NEXT:    add x8, x8, :lo12:no_block_store
 ; CHECK-NEXT:    str d0, [x8]
 ; CHECK-NEXT:    str w0, [x8, #8]
 ; CHECK-NEXT:    str d1, [x8, #16]
 ; CHECK-NEXT:    str w1, [x8, #24]
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = call %T_NO_BLOCK @return_no_block()
   store %T_NO_BLOCK %1, %T_NO_BLOCK* @no_block_store
   ret void
 }
 
 define void @callee_no_block(%T_NO_BLOCK %a) {
 ; CHECK-LABEL: callee_no_block:

 ; CHECK-NEXT:    adrp x8, no_block_store
 ; CHECK-NEXT:    add x8, x8, :lo12:no_block_store
 ; CHECK-NEXT:    ldr w1, [x8, #24]
 ; CHECK-NEXT:    ldr d1, [x8, #16]
 ; CHECK-NEXT:    ldr w0, [x8, #8]
 ; CHECK-NEXT:    ldr d0, [x8]
 ; CHECK-NEXT:    bl callee_no_block
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = load %T_NO_BLOCK, %T_NO_BLOCK* @no_block_store
   call void @callee_no_block(%T_NO_BLOCK %1)
   ret void
 }

llvm/test/CodeGen/AArch64/arm64-atomic-128.ll

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_acq
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_def_cfa_offset 0
+; OUTLINE-NEXT:    .cfi_restore w30
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_acq_rel
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_def_cfa_offset 0
+; OUTLINE-NEXT:    .cfi_restore w30
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap_seqcst:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_rel
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_def_cfa_offset 0
+; OUTLINE-NEXT:    .cfi_restore w30
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap_release:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_relax
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_def_cfa_offset 0
+; OUTLINE-NEXT:    .cfi_restore w30
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap_monotonic:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

llvm/test/CodeGen/AArch64/arm64-fp128.ll

 ; CHECK-NEXT:    adrp x8, var32
 ; CHECK-NEXT:    str w0, [x8, :lo12:var32]
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __fixtfdi
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    str x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %val = load fp128, fp128* @lhs, align 16
 
   %val32 = fptosi fp128 %val to i32
   store i32 %val32, i32* @var32
 
   %val64 = fptosi fp128 %val to i64
   store i64 %val64, i64* @var64

 ; CHECK-NEXT:    adrp x8, var32
 ; CHECK-NEXT:    str w0, [x8, :lo12:var32]
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __fixunstfdi
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    str x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %val = load fp128, fp128* @lhs, align 16
 
   %val32 = fptoui fp128 %val to i32
   store i32 %val32, i32* @var32
 
   %val64 = fptoui fp128 %val to i64
   store i64 %val64, i64* @var64

 ; CHECK-NEXT:    bl __floatsitf
 ; CHECK-NEXT:    adrp x19, lhs
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    ldr x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    bl __floatditf
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %src32 = load i32, i32* @var32
   %val32 = sitofp i32 %src32 to fp128
   store volatile fp128 %val32, fp128* @lhs
 
   %src64 = load i64, i64* @var64
   %val64 = sitofp i64 %src64 to fp128

 ; CHECK-NEXT:    bl __floatunsitf
 ; CHECK-NEXT:    adrp x19, lhs
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    ldr x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    bl __floatunditf
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %src32 = load i32, i32* @var32
   %val32 = uitofp i32 %src32 to fp128
   store volatile fp128 %val32, fp128* @lhs
 
   %src64 = load i64, i64* @var64
   %val64 = uitofp i64 %src64 to fp128

 ; CHECK-NEXT:    adrp x8, lhs
 ; CHECK-NEXT:    ldr q0, [x8, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, rhs
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:rhs]
 ; CHECK-NEXT:    bl __letf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w0, le
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
 ; Technically, everything after the call to __letf2 is redundant, but we'll let
 ; LLVM have its fun for now.
   %val = fcmp ole fp128 %lhs, %rhs

 ; CHECK-NEXT:    adrp x8, lhs
 ; CHECK-NEXT:    ldr q0, [x8, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, rhs
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:rhs]
 ; CHECK-NEXT:    bl __letf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w0, gt
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
   %val = fcmp ugt fp128 %lhs, %rhs
 
   ret i1 %val

 ; CHECK-NEXT:    cset w19, eq
 ; CHECK-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
 ; CHECK-NEXT:    bl __unordtf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w8, ne
 ; CHECK-NEXT:    orr w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
   %val = fcmp ueq fp128 %lhs, %rhs
 
   ret i1 %val

 ; CHECK-NEXT:    adrp x8, rhs
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:rhs]
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    b.ge .LBB11_2
 ; CHECK-NEXT:  // %bb.1: // %iftrue
 ; CHECK-NEXT:    mov w0, #42
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB11_2: // %iffalse
 ; CHECK-NEXT:    mov w0, #29
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
   ; olt == !uge, which LLVM optimizes this to.
   %cond = fcmp olt fp128 %lhs, %rhs
   br i1 %cond, label %iftrue, label %iffalse

 ; CHECK-NEXT:    adrp x8, varfloat
 ; CHECK-NEXT:    str s0, [x8, :lo12:varfloat]
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
 ; CHECK-NEXT:    adrp x8, vardouble
 ; CHECK-NEXT:    str d0, [x8, :lo12:vardouble]
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %val = load fp128, fp128* @lhs, align 16
 
   %half = fptrunc fp128 %val to half
   store half %half, half* @varhalf, align 2
 
   %float = fptrunc fp128 %val to float

 ; CHECK-NEXT:    ldr s0, [x8, :lo12:varfloat]
 ; CHECK-NEXT:    bl __extendsftf2
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, vardouble
 ; CHECK-NEXT:    ldr d0, [x8, :lo12:vardouble]
 ; CHECK-NEXT:    bl __extenddftf2
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %val = load fp128, fp128* @lhs, align 16
 
   %half = load half, half* @varhalf
   %fromhalf = fpext half %half to fp128
   store volatile fp128 %fromhalf, fp128* @lhs, align 16
 

 ; CHECK-LABEL: test_neg:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    str q0, [sp, #-16]!
 ; CHECK-NEXT:    ldrb w8, [sp, #15]
 ; CHECK-NEXT:    eor w8, w8, #0x80
 ; CHECK-NEXT:    strb w8, [sp, #15]
 ; CHECK-NEXT:    ldr q0, [sp], #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 
 ;; We convert this to fneg, and target-independent code expands it with
 ;; integer operations.
   %ret = fsub fp128 0xL00000000000000008000000000000000, %in
   ret fp128 %ret
 
 }

llvm/test/CodeGen/AArch64/arm64-memset-inline.ll

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    add x0, sp, #12
 ; CHECK-NEXT:    str wzr, [sp, #12]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [4 x i8], align 1
   %cast = bitcast [4 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 4, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_8_stack() {
 ; CHECK-LABEL: bzero_8_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp x30, xzr, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    add x0, sp, #8
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [8 x i8], align 1
   %cast = bitcast [8 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 8, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_12_stack() {
 ; CHECK-LABEL: bzero_12_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str wzr, [sp, #8]
 ; CHECK-NEXT:    str xzr, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [12 x i8], align 1
   %cast = bitcast [12 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 12, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_16_stack() {
 ; CHECK-LABEL: bzero_16_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    stp xzr, x30, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str xzr, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [16 x i8], align 1
   %cast = bitcast [16 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 16, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_20_stack() {
 ; CHECK-LABEL: bzero_20_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    add x0, sp, #8
 ; CHECK-NEXT:    stp xzr, xzr, [sp, #8]
 ; CHECK-NEXT:    str wzr, [sp, #24]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [20 x i8], align 1
   %cast = bitcast [20 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 20, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_26_stack() {
 ; CHECK-LABEL: bzero_26_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp xzr, xzr, [sp]
 ; CHECK-NEXT:    strh wzr, [sp, #24]
 ; CHECK-NEXT:    str xzr, [sp, #16]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [26 x i8], align 1
   %cast = bitcast [26 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 26, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_32_stack() {
 ; CHECK-LABEL: bzero_32_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [32 x i8], align 1
   %cast = bitcast [32 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 32, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_40_stack() {
 ; CHECK-LABEL: bzero_40_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #64
 ; CHECK-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str xzr, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [40 x i8], align 1
   %cast = bitcast [40 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 40, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_64_stack() {
 ; CHECK-LABEL: bzero_64_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #80
 ; CHECK-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [64 x i8], align 1
   %cast = bitcast [64 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 64, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_72_stack() {
 ; CHECK-LABEL: bzero_72_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str xzr, [sp, #64]
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [72 x i8], align 1
   %cast = bitcast [72 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 72, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @bzero_128_stack() {
 ; CHECK-LABEL: bzero_128_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #144
 ; CHECK-NEXT:    .cfi_def_cfa_offset 144
 ; CHECK-NEXT:    str x30, [sp, #128] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp q0, q0, [sp, #96]
 ; CHECK-NEXT:    stp q0, q0, [sp, #64]
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #144
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [128 x i8], align 1
   %cast = bitcast [128 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 128, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 

 ; CHECK-NEXT:    stp q0, q0, [sp, #128]
 ; CHECK-NEXT:    stp q0, q0, [sp, #96]
 ; CHECK-NEXT:    stp q0, q0, [sp, #64]
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #272
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %buf = alloca [256 x i8], align 1
   %cast = bitcast [256 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 0, i32 256, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_4_stack() {
 ; CHECK-LABEL: memset_4_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #-1431655766
 ; CHECK-NEXT:    add x0, sp, #12
 ; CHECK-NEXT:    str w8, [sp, #12]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [4 x i8], align 1
   %cast = bitcast [4 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 4, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_8_stack() {
 ; CHECK-LABEL: memset_8_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x8, #-6148914691236517206
 ; CHECK-NEXT:    stp x30, x8, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    add x0, sp, #8
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [8 x i8], align 1
   %cast = bitcast [8 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 8, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_12_stack() {
 ; CHECK-LABEL: memset_12_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x8, #-6148914691236517206
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str x8, [sp]
 ; CHECK-NEXT:    str w8, [sp, #8]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [12 x i8], align 1
   %cast = bitcast [12 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 12, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_16_stack() {
 ; CHECK-LABEL: memset_16_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x8, #-6148914691236517206
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp x8, x30, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    str x8, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [16 x i8], align 1
   %cast = bitcast [16 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 16, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_20_stack() {
 ; CHECK-LABEL: memset_20_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x8, #-6148914691236517206
 ; CHECK-NEXT:    add x0, sp, #8
 ; CHECK-NEXT:    stp x8, x8, [sp, #8]
 ; CHECK-NEXT:    str w8, [sp, #24]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [20 x i8], align 1
   %cast = bitcast [20 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 20, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_26_stack() {
 ; CHECK-LABEL: memset_26_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x8, #-6148914691236517206
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp x8, x8, [sp, #8]
 ; CHECK-NEXT:    str x8, [sp]
 ; CHECK-NEXT:    strh w8, [sp, #24]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [26 x i8], align 1
   %cast = bitcast [26 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 26, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_32_stack() {
 ; CHECK-LABEL: memset_32_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.16b, #170
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [32 x i8], align 1
   %cast = bitcast [32 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 32, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_40_stack() {
 ; CHECK-LABEL: memset_40_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #64
 ; CHECK-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.16b, #170
 ; CHECK-NEXT:    mov x8, #-6148914691236517206
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str x8, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [40 x i8], align 1
   %cast = bitcast [40 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 40, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_64_stack() {
 ; CHECK-LABEL: memset_64_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #80
 ; CHECK-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.16b, #170
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [64 x i8], align 1
   %cast = bitcast [64 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 64, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_72_stack() {
 ; CHECK-LABEL: memset_72_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.16b, #170
 ; CHECK-NEXT:    mov x8, #-6148914691236517206
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    str x8, [sp, #64]
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [72 x i8], align 1
   %cast = bitcast [72 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 72, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 define void @memset_128_stack() {
 ; CHECK-LABEL: memset_128_stack:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #144
 ; CHECK-NEXT:    .cfi_def_cfa_offset 144
 ; CHECK-NEXT:    str x30, [sp, #128] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    movi v0.16b, #170
 ; CHECK-NEXT:    mov x0, sp
 ; CHECK-NEXT:    stp q0, q0, [sp, #96]
 ; CHECK-NEXT:    stp q0, q0, [sp, #64]
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #144
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %buf = alloca [128 x i8], align 1
   %cast = bitcast [128 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 128, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 

 ; CHECK-NEXT:    stp q0, q0, [sp, #128]
 ; CHECK-NEXT:    stp q0, q0, [sp, #96]
 ; CHECK-NEXT:    stp q0, q0, [sp, #64]
 ; CHECK-NEXT:    stp q0, q0, [sp, #32]
 ; CHECK-NEXT:    stp q0, q0, [sp]
 ; CHECK-NEXT:    bl something
 ; CHECK-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #272
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %buf = alloca [256 x i8], align 1
   %cast = bitcast [256 x i8]* %buf to i8*
   call void @llvm.memset.p0i8.i32(i8* %cast, i8 -86, i32 256, i1 false)
   call void @something(i8* %cast)
   ret void
 }
 
 declare void @something(i8*)
 declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1) nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind

llvm/test/CodeGen/AArch64/arm64-neon-copy.ll

 ; CHECK-NEXT:    str q0, [sp]
 ; CHECK-NEXT:    bfi x9, x8, #1, #3
 ; CHECK-NEXT:    ldr h1, [x9]
 ; CHECK-NEXT:    mov v1.h[1], v0.h[1]
 ; CHECK-NEXT:    mov v1.h[2], v0.h[2]
 ; CHECK-NEXT:    mov v1.h[3], v0.h[3]
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
   %tmp = extractelement <8 x i16> %x, i32 %idx
   %tmp2 = insertelement <4 x i16> undef, i16 %tmp, i32 0
   %tmp3 = extractelement <8 x i16> %x, i32 1
   %tmp4 = insertelement <4 x i16> %tmp2, i16 %tmp3, i32 1
   %tmp5 = extractelement <8 x i16> %x, i32 2
   %tmp6 = insertelement <4 x i16> %tmp4, i16 %tmp5, i32 2
   %tmp7 = extractelement <8 x i16> %x, i32 3

 ; CHECK-NEXT:    bfi x9, x8, #1, #2
 ; CHECK-NEXT:    str h0, [x9]
 ; CHECK-NEXT:    ldr d1, [sp, #8]
 ; CHECK-NEXT:    mov v1.h[1], v0.h[1]
 ; CHECK-NEXT:    mov v1.h[2], v0.h[2]
 ; CHECK-NEXT:    mov v1.h[3], v0.h[3]
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
   %tmp = extractelement <8 x i16> %x, i32 0
   %tmp2 = insertelement <4 x i16> undef, i16 %tmp, i32 %idx
   %tmp3 = extractelement <8 x i16> %x, i32 1
   %tmp4 = insertelement <4 x i16> %tmp2, i16 %tmp3, i32 1
   %tmp5 = extractelement <8 x i16> %x, i32 2
   %tmp6 = insertelement <4 x i16> %tmp4, i16 %tmp5, i32 2
   %tmp7 = extractelement <8 x i16> %x, i32 3

llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll

 ; CHECK-NEXT:    mov v2.b[13], w12
 ; CHECK-NEXT:    msub w8, w8, w9, w10
 ; CHECK-NEXT:    sdiv w11, w14, w13
 ; CHECK-NEXT:    mov v2.b[14], w8
 ; CHECK-NEXT:    msub w8, w11, w13, w14
 ; CHECK-NEXT:    mov v2.b[15], w8
 ; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ldp x26, x25, [sp], #64 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
 ; CHECK-NEXT:    ret
   %tmp3 = srem <16 x i8> %A, %B;
   ret <16 x i8> %tmp3
 }
 
 define <1 x i16> @srem1x16(<1 x i16> %A, <1 x i16> %B) {
 ; CHECK-LABEL: srem1x16:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    mov v2.b[13], w12
 ; CHECK-NEXT:    msub w8, w8, w9, w10
 ; CHECK-NEXT:    udiv w11, w14, w13
 ; CHECK-NEXT:    mov v2.b[14], w8
 ; CHECK-NEXT:    msub w8, w11, w13, w14
 ; CHECK-NEXT:    mov v2.b[15], w8
 ; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    ldp x26, x25, [sp], #64 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
 ; CHECK-NEXT:    ret
   %tmp3 = urem <16 x i8> %A, %B;
   ret <16 x i8> %tmp3
 }
 
 define <1 x i16> @urem1x16(<1 x i16> %A, <1 x i16> %B) {
 ; CHECK-LABEL: urem1x16:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
 ; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[1], v1.s[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %tmp3 = frem <2 x float> %A, %B;
   ret <2 x float> %tmp3
 }
 
 define <4 x float> @frem4f32(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: frem4f32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    mov s1, v1.s[3]
 ; CHECK-NEXT:    bl fmodf
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
 ; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov v1.s[3], v0.s[0]
 ; CHECK-NEXT:    mov v0.16b, v1.16b
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %tmp3 = frem <4 x float> %A, %B;
   ret <4 x float> %tmp3
 }
 
 define <1 x double> @frem1d64(<1 x double> %A, <1 x double> %B) {
 ; CHECK-LABEL: frem1d64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl fmod
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %tmp3 = frem <1 x double> %A, %B;
   ret <1 x double> %tmp3
 }
 
 define <2 x double> @frem2d64(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: frem2d64:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
 ; CHECK-NEXT:    bl fmod
 ; CHECK-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %tmp3 = frem <2 x double> %A, %B;
   ret <2 x double> %tmp3
 }
 
 declare <8 x i8> @llvm.aarch64.neon.pmul.v8i8(<8 x i8>, <8 x i8>)
 declare <16 x i8> @llvm.aarch64.neon.pmul.v16i8(<16 x i8>, <16 x i8>)
 

llvm/test/CodeGen/AArch64/atomicrmw-O0.ll

 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB0_1
 ; NOLSE-NEXT:    b .LBB0_5
 ; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_add_8:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    mov w8, #1
 ; LSE-NEXT:    ldaddalb w8, w0, [x0]
 ; LSE-NEXT:    ret
 entry:

 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB1_1
 ; NOLSE-NEXT:    b .LBB1_5
 ; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_add_16:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    mov w8, #1
 ; LSE-NEXT:    ldaddalh w8, w0, [x0]
 ; LSE-NEXT:    ret
 entry:

 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB2_1
 ; NOLSE-NEXT:    b .LBB2_5
 ; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_add_32:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    mov w8, #1
 ; LSE-NEXT:    ldaddal w8, w0, [x0]
 ; LSE-NEXT:    ret
 entry:

 ; NOLSE-NEXT:    str x8, [sp, #8] // 8-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str x8, [sp, #24] // 8-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB3_1
 ; NOLSE-NEXT:    b .LBB3_5
 ; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr x0, [sp, #8] // 8-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_add_64:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    mov w8, #1
 ; LSE-NEXT:    // kill: def $x8 killed $w8
 ; LSE-NEXT:    ldaddal x8, x0, [x0]
 ; LSE-NEXT:    ret

 ; NOLSE-NEXT:    str x10, [sp, #32] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x9, [sp, #40] // 8-byte Folded Spill
 ; NOLSE-NEXT:    cbnz x8, .LBB4_1
 ; NOLSE-NEXT:    b .LBB4_6
 ; NOLSE-NEXT:  .LBB4_6: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr x1, [sp, #8] // 8-byte Folded Reload
 ; NOLSE-NEXT:    ldr x0, [sp, #16] // 8-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #48
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_add_128:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #80
 ; LSE-NEXT:    .cfi_def_cfa_offset 80
 ; LSE-NEXT:    str x0, [sp, #56] // 8-byte Folded Spill
 ; LSE-NEXT:    ldr x8, [x0, #8]

 ; LSE-NEXT:    str x10, [sp, #64] // 8-byte Folded Spill
 ; LSE-NEXT:    str x9, [sp, #72] // 8-byte Folded Spill
 ; LSE-NEXT:    cbnz x8, .LBB4_1
 ; LSE-NEXT:    b .LBB4_2
 ; LSE-NEXT:  .LBB4_2: // %atomicrmw.end
 ; LSE-NEXT:    ldr x1, [sp, #40] // 8-byte Folded Reload
 ; LSE-NEXT:    ldr x0, [sp, #48] // 8-byte Folded Reload
 ; LSE-NEXT:    add sp, sp, #80
+; LSE-NEXT:    .cfi_def_cfa_offset 0
 ; LSE-NEXT:    ret
 entry:
   %res = atomicrmw add i128* %dst, i128 1 seq_cst
   ret i128 %res
 }
 define i8 @test_rmw_nand_8(i8* %dst)   {
 ; NOLSE-LABEL: test_rmw_nand_8:
 ; NOLSE:       // %bb.0: // %entry

 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB5_1
 ; NOLSE-NEXT:    b .LBB5_5
 ; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_nand_8:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #32
 ; LSE-NEXT:    .cfi_def_cfa_offset 32
 ; LSE-NEXT:    str x0, [sp, #16] // 8-byte Folded Spill
 ; LSE-NEXT:    ldrb w8, [x0]

 ; LSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB5_1
 ; LSE-NEXT:    b .LBB5_2
 ; LSE-NEXT:  .LBB5_2: // %atomicrmw.end
 ; LSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; LSE-NEXT:    add sp, sp, #32
+; LSE-NEXT:    .cfi_def_cfa_offset 0
 ; LSE-NEXT:    ret
 entry:
   %res = atomicrmw nand i8* %dst, i8 1 seq_cst
   ret i8 %res
 }
 
 define i16 @test_rmw_nand_16(i16* %dst)   {
 ; NOLSE-LABEL: test_rmw_nand_16:

 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB6_1
 ; NOLSE-NEXT:    b .LBB6_5
 ; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_nand_16:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #32
 ; LSE-NEXT:    .cfi_def_cfa_offset 32
 ; LSE-NEXT:    str x0, [sp, #16] // 8-byte Folded Spill
 ; LSE-NEXT:    ldrh w8, [x0]

 ; LSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB6_1
 ; LSE-NEXT:    b .LBB6_2
 ; LSE-NEXT:  .LBB6_2: // %atomicrmw.end
 ; LSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; LSE-NEXT:    add sp, sp, #32
+; LSE-NEXT:    .cfi_def_cfa_offset 0
 ; LSE-NEXT:    ret
 entry:
   %res = atomicrmw nand i16* %dst, i16 1 seq_cst
   ret i16 %res
 }
 
 define i32 @test_rmw_nand_32(i32* %dst)   {
 ; NOLSE-LABEL: test_rmw_nand_32:

 ; NOLSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB7_1
 ; NOLSE-NEXT:    b .LBB7_5
 ; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_nand_32:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #32
 ; LSE-NEXT:    .cfi_def_cfa_offset 32
 ; LSE-NEXT:    str x0, [sp, #16] // 8-byte Folded Spill
 ; LSE-NEXT:    ldr w8, [x0]

 ; LSE-NEXT:    str w8, [sp, #12] // 4-byte Folded Spill
 ; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB7_1
 ; LSE-NEXT:    b .LBB7_2
 ; LSE-NEXT:  .LBB7_2: // %atomicrmw.end
 ; LSE-NEXT:    ldr w0, [sp, #12] // 4-byte Folded Reload
 ; LSE-NEXT:    add sp, sp, #32
+; LSE-NEXT:    .cfi_def_cfa_offset 0
 ; LSE-NEXT:    ret
 entry:
   %res = atomicrmw nand i32* %dst, i32 1 seq_cst
   ret i32 %res
 }
 
 define i64 @test_rmw_nand_64(i64* %dst)   {
 ; NOLSE-LABEL: test_rmw_nand_64:

 ; NOLSE-NEXT:    str x8, [sp, #8] // 8-byte Folded Spill
 ; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    str x8, [sp, #24] // 8-byte Folded Spill
 ; NOLSE-NEXT:    b.ne .LBB8_1
 ; NOLSE-NEXT:    b .LBB8_5
 ; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr x0, [sp, #8] // 8-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_nand_64:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #32
 ; LSE-NEXT:    .cfi_def_cfa_offset 32
 ; LSE-NEXT:    str x0, [sp, #16] // 8-byte Folded Spill
 ; LSE-NEXT:    ldr x8, [x0]

 ; LSE-NEXT:    str x8, [sp, #8] // 8-byte Folded Spill
 ; LSE-NEXT:    subs w9, w9, #1
 ; LSE-NEXT:    str x8, [sp, #24] // 8-byte Folded Spill
 ; LSE-NEXT:    b.ne .LBB8_1
 ; LSE-NEXT:    b .LBB8_2
 ; LSE-NEXT:  .LBB8_2: // %atomicrmw.end
 ; LSE-NEXT:    ldr x0, [sp, #8] // 8-byte Folded Reload
 ; LSE-NEXT:    add sp, sp, #32
+; LSE-NEXT:    .cfi_def_cfa_offset 0
 ; LSE-NEXT:    ret
 entry:
   %res = atomicrmw nand i64* %dst, i64 1 seq_cst
   ret i64 %res
 }
 
 define i128 @test_rmw_nand_128(i128* %dst)   {
 ; NOLSE-LABEL: test_rmw_nand_128:

 ; NOLSE-NEXT:    str x10, [sp, #32] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x9, [sp, #40] // 8-byte Folded Spill
 ; NOLSE-NEXT:    cbnz x8, .LBB9_1
 ; NOLSE-NEXT:    b .LBB9_6
 ; NOLSE-NEXT:  .LBB9_6: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr x1, [sp, #8] // 8-byte Folded Reload
 ; NOLSE-NEXT:    ldr x0, [sp, #16] // 8-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #48
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_nand_128:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #80
 ; LSE-NEXT:    .cfi_def_cfa_offset 80
 ; LSE-NEXT:    str x0, [sp, #56] // 8-byte Folded Spill
 ; LSE-NEXT:    ldr x8, [x0, #8]

 ; LSE-NEXT:    str x10, [sp, #64] // 8-byte Folded Spill
 ; LSE-NEXT:    str x9, [sp, #72] // 8-byte Folded Spill
 ; LSE-NEXT:    cbnz x8, .LBB9_1
 ; LSE-NEXT:    b .LBB9_2
 ; LSE-NEXT:  .LBB9_2: // %atomicrmw.end
 ; LSE-NEXT:    ldr x1, [sp, #40] // 8-byte Folded Reload
 ; LSE-NEXT:    ldr x0, [sp, #48] // 8-byte Folded Reload
 ; LSE-NEXT:    add sp, sp, #80
+; LSE-NEXT:    .cfi_def_cfa_offset 0
 ; LSE-NEXT:    ret
 entry:
   %res = atomicrmw nand i128* %dst, i128 1 seq_cst
   ret i128 %res
 }

llvm/test/CodeGen/AArch64/atomicrmw-xchg-fp.ll

 ; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
 ; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
 ; NOLSE-NEXT:    ldaxp x11, x10, [x0]
 ; NOLSE-NEXT:    stlxp w12, x9, x8, [x0]
 ; NOLSE-NEXT:    cbnz w12, .LBB3_1
 ; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    stp x11, x10, [sp]
 ; NOLSE-NEXT:    ldr q0, [sp], #32
+; NOLSE-NEXT:    .cfi_def_cfa_offset 0
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_xchg_f128:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    sub sp, sp, #32
 ; LSE-NEXT:    .cfi_def_cfa_offset 32
 ; LSE-NEXT:    str q0, [sp, #16]
 ; LSE-NEXT:    ldp x9, x8, [sp, #16]
 ; LSE-NEXT:  .LBB3_1: // %atomicrmw.start
 ; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
 ; LSE-NEXT:    ldaxp x11, x10, [x0]
 ; LSE-NEXT:    stlxp w12, x9, x8, [x0]
 ; LSE-NEXT:    cbnz w12, .LBB3_1
 ; LSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; LSE-NEXT:    stp x11, x10, [sp]
 ; LSE-NEXT:    ldr q0, [sp], #32
+; LSE-NEXT:    .cfi_def_cfa_offset 0
 ; LSE-NEXT:    ret
   %res = atomicrmw xchg fp128* %dst, fp128 %new seq_cst
   ret fp128 %res
 }

llvm/test/CodeGen/AArch64/bcmp-inline-small.ll

 ; CHECKS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECKS-NEXT:    .cfi_def_cfa_offset 16
 ; CHECKS-NEXT:    .cfi_offset w30, -16
 ; CHECKS-NEXT:    mov w2, #15
 ; CHECKS-NEXT:    bl bcmp
 ; CHECKS-NEXT:    cmp w0, #0
 ; CHECKS-NEXT:    cset w0, eq
 ; CHECKS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECKS-NEXT:    .cfi_def_cfa_offset 0
+; CHECKS-NEXT:    .cfi_restore w30
 ; CHECKS-NEXT:    ret
 entry:
   %bcmp = call i32 @bcmp(i8* %s1, i8* %s2, i64 15)
   %ret = icmp eq i32 %bcmp, 0
   ret i1 %ret
 }
 
 ; TODO: Four loads should be within the limit, but the heuristic isn't implemented.

 ; CHECKS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECKS-NEXT:    .cfi_def_cfa_offset 16
 ; CHECKS-NEXT:    .cfi_offset w30, -16
 ; CHECKS-NEXT:    mov w2, #15
 ; CHECKS-NEXT:    bl bcmp
 ; CHECKS-NEXT:    cmp w0, #0
 ; CHECKS-NEXT:    cset w0, eq
 ; CHECKS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECKS-NEXT:    .cfi_def_cfa_offset 0
+; CHECKS-NEXT:    .cfi_restore w30
 ; CHECKS-NEXT:    ret
 entry:
   %bcmp = call i32 @bcmp(i8* %s1, i8* %s2, i64 15)
   %ret = icmp eq i32 %bcmp, 0
   ret i1 %ret
 }
 
 define i1 @test_bs(i8* %s1, i8* %s2) optsize {

 ; CHECKS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECKS-NEXT:    .cfi_def_cfa_offset 16
 ; CHECKS-NEXT:    .cfi_offset w30, -16
 ; CHECKS-NEXT:    mov w2, #31
 ; CHECKS-NEXT:    bl memcmp
 ; CHECKS-NEXT:    cmp w0, #0
 ; CHECKS-NEXT:    cset w0, eq
 ; CHECKS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECKS-NEXT:    .cfi_def_cfa_offset 0
+; CHECKS-NEXT:    .cfi_restore w30
 ; CHECKS-NEXT:    ret
 entry:
   %memcmp = call i32 @memcmp(i8* %s1, i8* %s2, i64 31)
   %ret = icmp eq i32 %memcmp, 0
   ret i1 %ret
 }

llvm/test/CodeGen/AArch64/build-one-lane.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    and x9, x0, #0x1f
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    mov w10, #30
 ; CHECK-NEXT:    stp q0, q1, [sp]
 ; CHECK-NEXT:    strb w10, [x8, x9]
 ; CHECK-NEXT:    ldp q0, q1, [sp], #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
   %b = insertelement <32 x i8> %a, i8 30, i32 %x
   ret <32 x i8> %b
 }
 

llvm/test/CodeGen/AArch64/call-rv-marker.ll

 
 define dso_local void @rv_marker_2_select(i32 %c) {
 ; CHECK-LABEL: rv_marker_2_select:
 ; SELDAG:        cinc  w0, w8, eq
 ; GISEL:         csinc w0, w8, wzr, eq
 ; CHECK-NEXT:    bl  foo0
 ; SELDAG-NEXT:   mov x29, x29
 ; CHECK-NEXT:    ldr x30, [sp], #16
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w30
 ; CHECK-NEXT:    b  foo2
 ;
 entry:
   %tobool.not = icmp eq i32 %c, 0
   %.sink = select i1 %tobool.not, i32 2, i32 1
   %call1 = call i8* @foo0(i32 %.sink) [ "clang.arc.attachedcall"() ]
   tail call void @foo2(i8* %call1)
   ret void

llvm/test/CodeGen/AArch64/cmp-select-sign.ll

 ; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    orr v2.4s, #1
 ; CHECK-NEXT:    xtn v0.4h, v0.4s
 ; CHECK-NEXT:    str q2, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl use_4xi1
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %c = icmp sgt <4 x i32> %a, <i32 -1, i32 -1, i32 -1, i32 -1>
   %res = select <4 x i1> %c, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
   call void @use_4xi1(<4 x i1> %c)
   ret <4 x i32> %res
 }
 
 ; Second icmp operand breaks sign pattern.

llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll

 ; CHECK-NEXT:    bl Update
 ; CHECK-NEXT:    sub x21, x21, #2
 ; CHECK-NEXT:    cmp x20, x21
 ; CHECK-NEXT:    b.lt .LBB6_1
 ; CHECK-NEXT:  .LBB6_3: // %while.end
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %size = getelementptr inbounds %struct.Struct, %struct.Struct* %hdCall, i64 0, i32 0
   %0 = load i64, i64* %size, align 8
   br label %land.rhs
 
 land.rhs:
   %rp.06 = phi i64 [ %0, %entry ], [ %sub, %while.body ]

 ; CHECK-NEXT:  // %bb.6:
 ; CHECK-NEXT:    mov w0, #123
 ; CHECK-NEXT:    b .LBB7_8
 ; CHECK-NEXT:  .LBB7_7: // %if.end
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:  .LBB7_8: // %return
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, i32* @a, align 4
   %cmp4 = icmp slt i32 %0, -1
   br i1 %cmp4, label %while.body.preheader, label %while.end
 
 while.body.preheader:                             ; preds = %entry
   br label %while.body

 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
 ; CHECK-NEXT:    b.ne .LBB8_6
 ; CHECK-NEXT:  // %bb.5:
 ; CHECK-NEXT:    mov w0, #123
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB8_6: // %if.end
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, i32* @a, align 4
   %cmp4 = icmp slt i32 %0, 1
   br i1 %cmp4, label %while.body.preheader, label %while.end
 
 while.body.preheader:                             ; preds = %entry
   br label %while.body

 ; CHECK-NEXT:    fmov d1, #-2.00000000
 ; CHECK-NEXT:    fadd d0, d8, d0
 ; CHECK-NEXT:    fcsel d0, d8, d0, gt
 ; CHECK-NEXT:    bl woo
 ; CHECK-NEXT:    mov w0, #4
 ; CHECK-NEXT:  .LBB9_4: // %return
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    ret
 
 ; CHECK-LABEL-DAG: .LBB9_3
 
 entry:
   %cmp = icmp sgt i32 %argc, 1
   br i1 %cmp, label %land.lhs.true, label %if.end
 

 ; CHECK-NEXT:    mov w8, #42
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csinc w8, w8, wzr, gt
 ; CHECK-NEXT:    cmp w0, #2, lsl #12 // =8192
 ; CHECK-NEXT:    mov w9, #128
 ; CHECK-NEXT:    csel w0, w9, w8, ge
 ; CHECK-NEXT:    bl zoo
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; [...]
 
   %tst_low = icmp sgt i32 %in, 8191
   br i1 %tst_low, label %true, label %false
 
 true:
   call i32 @zoo(i32 128)

llvm/test/CodeGen/AArch64/cond-br-tuning.ll

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    and w19, w1, #0x6
 ; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    cbnz w19, .LBB9_2
 ; CHECK-NEXT:  // %bb.1: // %if.end
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB9_2: // %if.then
 ; CHECK-NEXT:    bl foo
 entry:
   %c = and i32 %a, 6
   call void @bar(i32 %c)
   %tobool = icmp eq i32 %c, 0
   br i1 %tobool, label %if.end, label %if.then
 
 if.then:
   tail call void @foo()
   unreachable
 
 if.end:
   ret void
 }

llvm/test/CodeGen/AArch64/csr-split.ll

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x8, a
 ; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
 ; CHECK-NEXT:    cmp x8, x0
 ; CHECK-NEXT:    b.eq .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %if.end
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %if.then
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl callVoid
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    b callNonVoid
 ;
 ; CHECK-APPLE-LABEL: test1:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8

 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
 ; CHECK-NEXT:    cmp x8, x0
 ; CHECK-NEXT:    b.ne .LBB1_3
 ; CHECK-NEXT:  // %bb.2: // %if.then2
 ; CHECK-NEXT:    bl callVoid
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    b callNonVoid
 ; CHECK-NEXT:  .LBB1_3: // %return
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-APPLE-LABEL: test2:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8

 ; CHECK-NEXT:    mov x20, x0
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    str x0, [x20]
 ; CHECK-NEXT:  .LBB2_2: // %land.end
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-APPLE-LABEL: test3:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8

llvm/test/CodeGen/AArch64/emutls.ll

 declare i8* @my_emutls_get_address(i8*)
 
 define i32 @my_get_xyz() {
 ; ARM64-LABEL: my_get_xyz:
 ; ARM64:        adrp x0, :got:my_emutls_v_xyz
 ; ARM64-NEXT:   ldr x0, [x0, :got_lo12:my_emutls_v_xyz]
 ; ARM64-NEXT:   bl my_emutls_get_address
 ; ARM64-NEXT:   ldr  w0, [x0]
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
   %0 = bitcast i8* %call to i32*
   %1 = load i32, i32* %0, align 4
   ret i32 %1
 }
 
 @i1 = thread_local global i32 15
 @i2 = external thread_local global i32
 @i3 = internal thread_local global i32 15
 @i4 = hidden thread_local global i32 15
 @i5 = external hidden thread_local global i32
 @s1 = thread_local global i16 15
 @b1 = thread_local global i8 0
 
 define i32 @f1() {
 ; ARM64-LABEL: f1:
 ; ARM64:        adrp x0, :got:__emutls_v.i1
 ; ARM64-NEXT:   ldr x0, [x0, :got_lo12:__emutls_v.i1]
 ; ARM64-NEXT:   bl __emutls_get_address
 ; ARM64-NEXT:   ldr  w0, [x0]
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   %tmp1 = load i32, i32* @i1
   ret i32 %tmp1
 }
 
 define i32* @f2() {
 ; ARM64-LABEL: f2:
 ; ARM64:        adrp x0, :got:__emutls_v.i1
 ; ARM64-NEXT:   ldr x0, [x0, :got_lo12:__emutls_v.i1]
 ; ARM64-NEXT:   bl __emutls_get_address
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   ret i32* @i1
 }
 
 define i32 @f5() nounwind {
 ; ARM64-LABEL: f5:
 ; ARM64:        adrp x0, __emutls_v.i3
 ; ARM64:        add x0, x0, :lo12:__emutls_v.i3
 ; ARM64:        bl __emutls_get_address
 ; ARM64-NEXT:   ldr w0, [x0]
 
 entry:
   %tmp1 = load i32, i32* @i3
   ret i32 %tmp1
 }
 
 define i32* @f6() {
 ; ARM64-LABEL: f6:
 ; ARM64:        adrp x0, __emutls_v.i3
 ; ARM64:        add x0, x0, :lo12:__emutls_v.i3
 ; ARM64-NEXT:   bl __emutls_get_address
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   ret i32* @i3
 }
 
 ; Simple test of comdat __thread variables.
 ; template <class T> struct A { static __thread T x; };

llvm/test/CodeGen/AArch64/fastcc.ll

 
 
 ; CHECK-TAIL: bl func_stack0
 ; CHECK-TAIL-NOT: sub sp, sp
 
   ret void
 ; CHECK: ldp     x29, x30, [sp, #32]
 ; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
 ; CHECK-NEXT: ret
 
 
 ; CHECK-TAIL: ldp     x29, x30, [sp, #32]
 ; CHECK-TAIL-NEXT: add sp, sp, #48
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-NEXT: .cfi_restore w30
+; CHECK-TAIL-NEXT: .cfi_restore w29
 ; CHECK-TAIL-NEXT: ret
 }
 
 define fastcc void @func_stack8([8 x i64], i32 %stacked) {
 ; CHECK-LABEL: func_stack8:
 ; CHECK: sub sp, sp, #48
 ; CHECK: stp x29, x30, [sp, #32]
 ; CHECK: add x29, sp, #32
; CHECK-TAIL: sub sp, sp, #32
   call fastcc void @func_stack0()
 ; CHECK: bl func_stack0
 ; CHECK-NOT: sub sp, sp
 
 ; CHECK-TAIL: bl func_stack0
 ; CHECK-TAIL-NOT: sub sp, sp
 
   ret void
+; CHECK-NEXT: .cfi_def_cfa wsp, 48
 ; CHECK-NEXT: ldp     x29, x30, [sp, #32]
-; CHECK: add sp, sp, #48
+; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
 ; CHECK-NEXT: ret
 
 
 ; CHECK-TAIL: ldp     x29, x30, [sp, #32]
 ; CHECK-TAIL-NEXT: add     sp, sp, #64
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -16
+; CHECK-TAIL-NEXT: .cfi_restore w30
+; CHECK-TAIL-NEXT: .cfi_restore w29
 ; CHECK-TAIL-NEXT: ret
 }
 
 define fastcc void @func_stack32([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32:
 ; CHECK: add x29, sp, #32
 
 ; CHECK-TAIL-LABEL: func_stack32:

 ; CHECK: bl func_stack0
 ; CHECK-NOT: sub sp, sp
 
 
 ; CHECK-TAIL: bl func_stack0
 ; CHECK-TAIL-NOT: sub sp, sp
 
   ret void
-; CHECK: ldp     x29, x30, [sp, #32]
+; CHECK:      .cfi_def_cfa wsp, 48
+; CHECK-NEXT: ldp x29, x30, [sp, #32]
 ; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL: ldp     x29, x30, [sp, #32]
 ; CHECK-TAIL-NEXT: add     sp, sp, #80
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-NEXT: .cfi_restore w30
+; CHECK-TAIL-NEXT: .cfi_restore w29
 ; CHECK-TAIL-NEXT: ret
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
 define fastcc void @func_stack32_leaf([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32_leaf:
 ; CHECK: str     x20, [sp, #-16]!
 ; CHECK: nop
 ; CHECK-NEXT: //NO_APP
 ; CHECK-NEXT: ldr     x20, [sp], #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w20
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL-LABEL: func_stack32_leaf:
 ; CHECK-TAIL: str     x20, [sp, #-16]!
 ; CHECK-TAIL: nop
 ; CHECK-TAIL-NEXT: //NO_APP
 ; CHECK-TAIL-NEXT: ldr     x20, [sp], #16
-; CHECK-TAIL-NEXT: add     sp, sp, #32
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-NEXT: add  sp, sp, #32
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-NEXT: .cfi_restore w20
 ; CHECK-TAIL-NEXT: ret
 
 ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf:
 ; CHECK-TAIL-RZ: str     x20, [sp, #-16]!
 ; CHECK-TAIL-RZ-NOT: sub     sp, sp
 ; CHECK-TAIL-RZ: nop
 ; CHECK-TAIL-RZ-NEXT: //NO_APP
 ; CHECK-TAIL-RZ-NEXT: ldr     x20, [sp], #16
-; CHECK-TAIL-RZ-NEXT: add     sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-RZ-NEXT: add sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-RZ-NEXT: .cfi_restore w20
 ; CHECK-TAIL-RZ-NEXT: ret
 
   ; Make sure there is a callee-save register to save/restore.
   call void asm sideeffect "nop", "~{x20}"() nounwind
   ret void
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
 define fastcc void @func_stack32_leaf_local([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32_leaf_local:
 ; CHECK: sub     sp, sp, #32
 ; CHECK-NEXT:  .cfi_def_cfa_offset 32
 ; CHECK-NEXT: str     x20, [sp, #16]
 ; CHECK: nop
 ; CHECK-NEXT: //NO_APP
 ; CHECK-NEXT: ldr     x20, [sp, #16]
 ; CHECK-NEXT: add     sp, sp, #32
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w20
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL-LABEL: func_stack32_leaf_local:
 ; CHECK-TAIL: sub     sp, sp, #32
 ; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 32
 ; CHECK-TAIL-NEXT: str     x20, [sp, #16]
 ; CHECK-TAIL: nop
 ; CHECK-TAIL-NEXT: //NO_APP
 ; CHECK-TAIL-NEXT: ldr     x20, [sp, #16]
 ; CHECK-TAIL-NEXT: add     sp, sp, #64
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-NEXT: .cfi_restore w20
 ; CHECK-TAIL-NEXT: ret
 
 ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf_local:
 ; CHECK-TAIL-RZ: str     x20, [sp, #-16]!
 ; CHECK-TAIL-RZ-NOT: sub     sp, sp
 ; CHECK-TAIL-RZ: nop
 ; CHECK-TAIL-RZ-NEXT: //NO_APP
 ; CHECK-TAIL-RZ-NEXT: ldr     x20, [sp], #16
-; CHECK-TAIL-RZ-NEXT: add     sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-RZ-NEXT: add sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-RZ-NEXT: .cfi_restore w20
 ; CHECK-TAIL-RZ-NEXT: ret
 
   %val0 = alloca [2 x i64], align 8
 
   ; Make sure there is a callee-save register to save/restore.
   call void asm sideeffect "nop", "~{x20}"() nounwind
   ret void
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
 define fastcc void @func_stack32_leaf_local_nocs([8 x i64], i128 %stacked0, i128 %stacked1) {
 ; CHECK-LABEL: func_stack32_leaf_local_nocs:
 ; CHECK: sub     sp, sp, #16
-; CHECK: add     sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL-LABEL: func_stack32_leaf_local_nocs:
 ; CHECK-TAIL: sub     sp, sp, #16
-; CHECK-TAIL: add     sp, sp, #48
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 16
+; CHECK-TAIL-NEXT: add  sp, sp, #48
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
 ; CHECK-TAIL-NEXT: ret
 
 ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf_local_nocs:
 ; CHECK-TAIL-RZ: add     sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32
 ; CHECK-TAIL-RZ-NEXT: ret
 
   %val0 = alloca [2 x i64], align 8
 
   ret void
 }

llvm/test/CodeGen/AArch64/fcopysign.ll

 ; CHECK-NEXT:    str q0, [sp, #-16]!
 ; CHECK-NEXT:    ldr x8, [x8, :lo12:val_double]
 ; CHECK-NEXT:    ldrb w9, [sp, #15]
 ; CHECK-NEXT:    and x8, x8, #0x8000000000000000
 ; CHECK-NEXT:    lsr x8, x8, #56
 ; CHECK-NEXT:    bfxil w8, w9, #0, #7
 ; CHECK-NEXT:    strb w8, [sp, #15]
 ; CHECK-NEXT:    ldr q0, [sp], #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
   %v = load double, double* @val_double, align 8
   %conv = fpext double %v to fp128
   %call = tail call fp128 @llvm.copysign.f128(fp128 0xL00000000000000007FFF000000000000, fp128 %conv) #2
   ret fp128 %call
 }
 
 define fp128@copysign1() {
 ; CHECK-LABEL: copysign1:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    adrp x8, val_fp128
 ; CHECK-NEXT:    ldr q0, [x8, :lo12:val_fp128]
 ; CHECK-NEXT:    adrp x8, val_float
 ; CHECK-NEXT:    str q0, [sp, #-16]!
 ; CHECK-NEXT:    ldr w8, [x8, :lo12:val_float]
 ; CHECK-NEXT:    ldrb w9, [sp, #15]
 ; CHECK-NEXT:    and w8, w8, #0x80000000
 ; CHECK-NEXT:    lsr w8, w8, #24
 ; CHECK-NEXT:    bfxil w8, w9, #0, #7
 ; CHECK-NEXT:    strb w8, [sp, #15]
 ; CHECK-NEXT:    ldr q0, [sp], #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
   %v0 = load fp128, fp128* @val_fp128, align 16
   %v1 = load float, float* @val_float, align 4
   %conv = fpext float %v1 to fp128
   %call = tail call fp128 @llvm.copysign.f128(fp128 %v0, fp128 %conv)
   ret fp128 %call
 }

llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll

 ; CHECK-NEXT:    csel w19, w8, w19, gt
 ; CHECK-NEXT:    mov v1.16b, v0.16b
 ; CHECK-NEXT:    bl __unordtf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <1 x i32> @llvm.fptosi.sat.v1f128.v1i32(<1 x fp128> %f)
     ret <1 x i32> %x
 }
 
 define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-LABEL: test_signed_v2f128_v2i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    mov v0.s[1], w22
 ; CHECK-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <2 x i32> @llvm.fptosi.sat.v2f128.v2i32(<2 x fp128> %f)
     ret <2 x i32> %x
 }
 
 define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-LABEL: test_signed_v3f128_v3i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[2], w8
 ; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <3 x i32> @llvm.fptosi.sat.v3f128.v3i32(<3 x fp128> %f)
     ret <3 x i32> %x
 }
 
 define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-LABEL: test_signed_v4f128_v4i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x20, x19, [sp, #128] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[3], w8
 ; CHECK-NEXT:    add sp, sp, #144
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <4 x i32> @llvm.fptosi.sat.v4f128.v4i32(<4 x fp128> %f)
     ret <4 x i32> %x
 }
 
 ;
 ; FP16 to signed 32-bit -- Vector size variation
 ;

 ; CHECK-NEXT:    fcmp s0, s0
 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel x9, xzr, x9, vs
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptosi.sat.v2f32.v2i100(<2 x float> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_signed_v2f32_v2i128(<2 x float> %f) {
 ; CHECK-LABEL: test_signed_v2f32_v2i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    fcmp s0, s0
 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel x9, xzr, x9, vs
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptosi.sat.v2f32.v2i128(<2 x float> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 4-Vector float to signed integer -- result size variation
 ;

 ; CHECK-NEXT:    csel x9, xzr, x9, vs
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    ldp x26, x25, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    ldp d9, d8, [sp, #40] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <4 x i100> @llvm.fptosi.sat.v4f32.v4i100(<4 x float> %f)
     ret <4 x i100> %x
 }
 
 define <4 x i128> @test_signed_v4f32_v4i128(<4 x float> %f) {
 ; CHECK-LABEL: test_signed_v4f32_v4i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    csel x9, xzr, x9, vs
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    ldp x26, x25, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    ldp d9, d8, [sp, #40] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <4 x i128> @llvm.fptosi.sat.v4f32.v4i128(<4 x float> %f)
     ret <4 x i128> %x
 }
 
 ;
 ; 2-Vector double to signed integer -- result size variation
 ;

 ; CHECK-NEXT:    fcmp d0, d0
 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel x9, xzr, x9, vs
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptosi.sat.v2f64.v2i100(<2 x double> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_signed_v2f64_v2i128(<2 x double> %f) {
 ; CHECK-LABEL: test_signed_v2f64_v2i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    fcmp d0, d0
 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel x9, xzr, x9, vs
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptosi.sat.v2f64.v2i128(<2 x double> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 4-Vector half to signed integer -- result size variation
 ;

 ; CHECK-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <4 x i100> @llvm.fptosi.sat.v4f16.v4i100(<4 x half> %f)
     ret <4 x i100> %x
 }
 
 define <4 x i128> @test_signed_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-LABEL: test_signed_v4f16_v4i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <4 x i128> @llvm.fptosi.sat.v4f16.v4i128(<4 x half> %f)
     ret <4 x i128> %x
 }
 
 ;
 ; 8-Vector half to signed integer -- result size variation
 ;

 ; CHECK-NEXT:    ldp x22, x21, [sp, #160] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #144] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x26, x25, [sp, #128] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x28, x27, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #192
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w27
+; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <8 x i100> @llvm.fptosi.sat.v8f16.v8i100(<8 x half> %f)
     ret <8 x i100> %x
 }
 
 define <8 x i128> @test_signed_v8f16_v8i128(<8 x half> %f) {
 ; CHECK-LABEL: test_signed_v8f16_v8i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ldp x22, x21, [sp, #160] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #144] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x26, x25, [sp, #128] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x28, x27, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #192
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w27
+; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
 ; CHECK-NEXT:    ret
     %x = call <8 x i128> @llvm.fptosi.sat.v8f16.v8i128(<8 x half> %f)
     ret <8 x i128> %x
 }
 
 

llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll

 ; CHECK-NEXT:    csel w19, wzr, w0, lt
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI14_1]
 ; CHECK-NEXT:    bl __gttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <1 x i32> @llvm.fptoui.sat.v1f128.v1i32(<1 x fp128> %f)
     ret <1 x i32> %x
 }
 
 define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-LABEL: test_unsigned_v2f128_v2i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    mov v0.s[1], w20
 ; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <2 x i32> @llvm.fptoui.sat.v2f128.v2i32(<2 x fp128> %f)
     ret <2 x i32> %x
 }
 
 define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-LABEL: test_unsigned_v3f128_v3i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    bl __gttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[2], w8
 ; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <3 x i32> @llvm.fptoui.sat.v3f128.v3i32(<3 x fp128> %f)
     ret <3 x i32> %x
 }
 
 define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-LABEL: test_unsigned_v4f128_v4i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    bl __gttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[3], w8
 ; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
     %x = call <4 x i32> @llvm.fptoui.sat.v4f128.v4i32(<4 x fp128> %f)
     ret <4 x i32> %x
 }
 
 ;
 ; FP16 to unsigned 32-bit -- Vector size variation
 ;

 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    csel x1, x21, x9, gt
 ; CHECK-NEXT:    ldp x30, x21, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptoui.sat.v2f32.v2i100(<2 x float> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_unsigned_v2f32_v2i128(<2 x float> %f) {
 ; CHECK-LABEL: test_unsigned_v2f32_v2i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    fcmp s0, s9
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    csinv x1, x9, xzr, le
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptoui.sat.v2f32.v2i128(<2 x float> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 4-Vector float to unsigned integer -- result size variation
 ;

 ; CHECK-NEXT:    csel x1, x25, x9, gt
 ; CHECK-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ldp x30, x25, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <4 x i100> @llvm.fptoui.sat.v4f32.v4i100(<4 x float> %f)
     ret <4 x i100> %x
 }
 
 define <4 x i128> @test_unsigned_v4f32_v4i128(<4 x float> %f) {
 ; CHECK-LABEL: test_unsigned_v4f32_v4i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    csinv x1, x9, xzr, le
 ; CHECK-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <4 x i128> @llvm.fptoui.sat.v4f32.v4i128(<4 x float> %f)
     ret <4 x i128> %x
 }
 
 ;
 ; 2-Vector double to unsigned integer -- result size variation
 ;

 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    csel x1, x21, x9, gt
 ; CHECK-NEXT:    ldp x30, x21, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptoui.sat.v2f64.v2i100(<2 x double> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_unsigned_v2f64_v2i128(<2 x double> %f) {
 ; CHECK-LABEL: test_unsigned_v2f64_v2i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    fcmp d0, d9
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    csinv x1, x9, xzr, le
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptoui.sat.v2f64.v2i128(<2 x double> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 4-Vector half to unsigned integer -- result size variation
 ;

 ; CHECK-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    ldp x30, x25, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <4 x i100> @llvm.fptoui.sat.v4f16.v4i100(<4 x half> %f)
     ret <4 x i100> %x
 }
 
 define <4 x i128> @test_unsigned_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-LABEL: test_unsigned_v4f16_v4i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <4 x i128> @llvm.fptoui.sat.v4f16.v4i128(<4 x half> %f)
     ret <4 x i128> %x
 }
 
 ;
 ; 8-Vector half to unsigned integer -- result size variation
 ;

 ; CHECK-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #128] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x26, x25, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x28, x27, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #176
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w27
+; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <8 x i100> @llvm.fptoui.sat.v8f16.v8i100(<8 x half> %f)
     ret <8 x i100> %x
 }
 
 define <8 x i128> @test_unsigned_v8f16_v8i128(<8 x half> %f) {
 ; CHECK-LABEL: test_unsigned_v8f16_v8i128:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #128] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x26, x25, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x28, x27, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #176
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w23
+; CHECK-NEXT:    .cfi_restore w24
+; CHECK-NEXT:    .cfi_restore w25
+; CHECK-NEXT:    .cfi_restore w26
+; CHECK-NEXT:    .cfi_restore w27
+; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    ret
     %x = call <8 x i128> @llvm.fptoui.sat.v8f16.v8i128(<8 x half> %f)
     ret <8 x i128> %x
 }

llvm/test/CodeGen/AArch64/framelayout-sve-calleesaves-fix.mir

--- |
   ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
   ; CHECK-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
   ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
   ; CHECK-NEXT:    addvl sp, sp, #-1
   ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
   ; CHECK-NEXT:    // implicit-def: $z8
   ; CHECK-NEXT:    // implicit-def: $p4
   ; CHECK-NEXT:    addvl sp, sp, #1
+  ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
   ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
   ; CHECK-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
   ; CHECK-NEXT:    addvl sp, sp, #2
+  ; CHECK-NEXT:    .cfi_def_cfa wsp, 16
+  ; CHECK-NEXT:    .cfi_restore z8
   ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+  ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+  ; CHECK-NEXT:    .cfi_restore w29
   ; CHECK-NEXT:    ret
 ...
 name: fix_restorepoint_p4
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
 body:             |
   bb.0.entry:
     $z8 = IMPLICIT_DEF
     $p4 = IMPLICIT_DEF
     B %bb.1
 
   bb.1.entry:
     RET_ReallyLR
 ---

llvm/test/CodeGen/AArch64/framelayout-sve.mir

 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 
 # ASM-LABEL: test_allocate_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 16 * VG
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 #
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +16
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 
 name:            test_allocate_sve
 stack:
   - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2 }
   - { id: 1, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     RET_ReallyLR

 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 #
 # CHECK-NEXT: $x20 = IMPLICIT_DEF
 # CHECK-NEXT: $x21 = IMPLICIT_DEF
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $x21, $x20 = frame-destroy LDPXi $sp, 2
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 32
+# CHECK-COUNT-4: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_allocate_sve_gpr_callee_saves:
 # ASM:       .cfi_def_cfa_offset 32
 # ASM:       .cfi_offset w20, -8
 # ASM-NEXT:  .cfi_offset w21, -16
 # ASM-NEXT:  .cfi_offset w29, -32
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 16 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 16 * VG
 #
+# ASM:       .cfi_def_cfa wsp, 48
+# ASM:       .cfi_def_cfa_offset 32
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w20
+# ASM-NEXT:  .cfi_restore w21
+# ASM-NEXT:  .cfi_restore w29
+
+#
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +32
 # UNWINDINFO:       DW_CFA_offset: reg20 -8
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg21 -16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -32
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +48
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +32
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg20
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg21
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 name:            test_allocate_sve_gpr_callee_saves
 stack:
   - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2 }
   - { id: 1, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     $x20 = IMPLICIT_DEF
     $x21 = IMPLICIT_DEF

 # CHECK-NEXT: $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
 # CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
 # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
+# CHECK-COUNT-3: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_allocate_sve_gpr_realigned:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM:       .cfi_def_cfa w29, 16
 # ASM-NEXT:  .cfi_offset w30, -8
 # ASM-NEXT:  .cfi_offset w29, -16
 #
+# ASM:        .cfi_def_cfa wsp, 16
+# ASM:        .cfi_def_cfa_offset 0
+# ASM-NEXT:   .cfi_restore w30
+# ASM-NEXT:   .cfi_restore w29
+
+#
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:       DW_CFA_def_cfa: reg29 +16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg30 -8
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
+#
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg30
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 
 name:            test_allocate_sve_gpr_realigned
 stack:
   - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2  }
   - { id: 1, stack-id: default, size: 16, alignment: 32 }
 body:             |
   bb.0.entry:
     RET_ReallyLR

 # CHECK-NEXT: $[[TMP:x[0-9]+]] = ADDXri $sp, 16
 # CHECK-NEXT: STR_ZXI $z0, killed $[[TMP]], 2
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = ADDXri $sp, 16
 # CHECK-NEXT: STR_ZXI $z1, killed $[[TMP]], 1
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = ADDXri $sp, 16
 # CHECK-NEXT: STR_PXI $p0, killed $[[TMP]], 7
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL:  test_address_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 24 * VG
 #
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+#
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +16
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 
 name:            test_address_sve
 frameInfo:
   maxAlignment:  16
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 1, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 2, stack-id: scalable-vector, size:  2, alignment: 2 }

 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
 
 # CHECK-NEXT: STR_ZXI $z0, $fp, -1
 # CHECK-NEXT: STR_ZXI $z1, $fp, -2
 # CHECK-NEXT: STR_PXI $p0, $fp, -17
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3
 # CHECK:      $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
+# CHECK-COUNT-3: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_address_sve_fp:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM:       .cfi_def_cfa w29, 16
 # ASM-NEXT:  .cfi_offset w30, -8
 # ASM-NEXT:  .cfi_offset w29, -16
 #
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w30
+# ASM-NEXT:  .cfi_restore w29
+#
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:      DW_CFA_def_cfa: reg29 +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg30
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name:            test_address_sve_fp
 frameInfo:
   maxAlignment:  16
   isFrameAddressTaken: true
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 1, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 2, stack-id: scalable-vector, size:  2, alignment: 2 }

 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
 
 # CHECK:      $[[TMP:x[0-9]+]] = ADDVL_XXI $sp, 1
 # CHECK-NEXT: $x0 = LDRXui killed $[[TMP]], 4
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_stack_arg_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 8 * VG
 #
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
 
 name:             test_stack_arg_sve
 fixedStack:
   - { id: 0, stack-id: default, size: 16, alignment: 16, offset: 0 }
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
   - { id: 1, stack-id: default, size: 16, alignment: 16 }
 body:             |

 
 # CHECK-NEXT: $[[TMP2:x[0-9]+]] = ADDVL_XXI $sp, 1
 # CHECK-NEXT: STR_ZXI $z0, killed $[[TMP2]], 255
 
 # CHECK-NEXT: $[[TMP2:x[0-9]+]] = ADDPL_XXI $sp, 1
 # CHECK-NEXT: STR_PXI $p0, killed $[[TMP2]], 255
 
 # CHECK:      $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 9
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_address_sve_out_of_range:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 256 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x04, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 512 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x06, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 768 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1024 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0a, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1280 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0c, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1536 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1792 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 2048 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 2056 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1808 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x98, 0x0c, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1560 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa0, 0x0a, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1312 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa8, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1064 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb0, 0x06, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 816 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb8, 0x04, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 568 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc0, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 320 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
+# ASM:        .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +256, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +512, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +768, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1024, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1280, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1536, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1792, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +2048, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +2056, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1808, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1560, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1312, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1064, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +816, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +568, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +320, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +72, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name:            test_address_sve_out_of_range
 frameInfo:
   maxAlignment:  16
 stack:
   - { id: 0, stack-id: scalable-vector, size:   16, alignment: 16 }
   - { id: 1, stack-id: scalable-vector, size: 3584, alignment: 16 }
   - { id: 2, stack-id: scalable-vector, size:  512, alignment: 16 }
 

 #
 # ASM-LABEL: test_address_gpr_vla:
 # ASM:       .cfi_def_cfa_offset 32
 # ASM:       .cfi_def_cfa w29, 32
 # ASM-NEXT:  .cfi_offset w19, -16
 # ASM-NEXT:  .cfi_offset w30, -24
 # ASM-NEXT:  .cfi_offset w29, -32
 #
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w19
+# ASM-NEXT:  .cfi_restore w30
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +32
 # UNWINDINFO:      DW_CFA_def_cfa: reg29 +32
 # UNWINDINFO-NEXT: DW_CFA_offset: reg19 -16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -24
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg19
+# UNWINDINFO-NEXT: DW_CFA_restore: reg30
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name:            test_address_gpr_vla
 frameInfo:
   maxAlignment:  16
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 1, stack-id: default, size: 16, alignment: 8 }
   - { id: 2, stack-id: default, type: variable-sized }
 body:             |

 # CHECK: RET_ReallyLR
 #
 # ASM-LABEL: save_restore_pregs_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 8 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:         DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:         DW_CFA_offset: reg29 -16
 # UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:         DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:         DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:    DW_CFA_restore: reg29
+
 name: save_restore_pregs_sve
 stack:
   - { id: 0, stack-id: default, size: 32, alignment: 16 }
 body:             |
   bb.0.entry:
 
     $p4 = IMPLICIT_DEF
     $p5 = IMPLICIT_DEF

 # CHECK-NEXT: frame-setup STR_ZXI killed $z10, $sp, 0
 # CHECK-NEXT: frame-setup STR_ZXI killed $z9, $sp, 1
 # CHECK-NEXT: frame-setup STR_ZXI killed $z8, $sp, 2
 # CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 32, 0
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 
 # CHECK:      $sp  = frame-destroy ADDXri $sp, 32, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $z10 = frame-destroy LDR_ZXI $sp, 0
 # CHECK-NEXT: $z9  = frame-destroy LDR_ZXI $sp, 1
 # CHECK-NEXT: $z8  = frame-destroy LDR_ZXI $sp, 2
 # CHECK-NEXT: $sp  = frame-destroy ADDVL_XXI $sp, 3
+# CHECK-COUNT-4: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $fp = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: save_restore_zregs_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9  @ cfa - 16 - 16 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10  @ cfa - 16 - 24 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 24 * VG
+#
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM-NEXT:  .cfi_restore z8
+# ASM-NEXT:  .cfi_restore z9
+# ASM-NEXT:  .cfi_restore z10
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
 
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
 
 name: save_restore_zregs_sve
 stack:
   - { id: 0, stack-id: default, size: 32, alignment: 16 }
 body:             |
   bb.0.entry:
 
     $z8 = IMPLICIT_DEF

 # CHECK: frame-setup STR_ZXI killed $z8, $sp, 17
 # CHECK-COUNT-8: frame-setup CFI_INSTRUCTION
 # CHECK: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK: frame-setup CFI_INSTRUCTION
 # CHECK: $sp = frame-setup SUBXri $sp, 32, 0
 # CHECK: frame-setup CFI_INSTRUCTION
 
 # CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK: $p15 = frame-destroy LDR_PXI $sp, 4
 # CHECK: $p14 = frame-destroy LDR_PXI $sp, 5
 # CHECK: $p5 = frame-destroy LDR_PXI $sp, 14
 # CHECK: $p4 = frame-destroy LDR_PXI $sp, 15
 # CHECK: $z23 = frame-destroy LDR_ZXI $sp, 2
 # CHECK: $z22 = frame-destroy LDR_ZXI $sp, 3
 # CHECK: $z9 = frame-destroy LDR_ZXI $sp, 16
 # CHECK: $z8 = frame-destroy LDR_ZXI $sp, 17
 # CHECK: $sp = frame-destroy ADDVL_XXI $sp, 18
+# CHECK-COUNT-9: frame-destroy CFI_INSTRUCTION
 # CHECK: $x20, $x19 = frame-destroy LDPXi $sp, 2
 # CHECK: $sp, ${{[a-z0-9]+}}, $x21 = frame-destroy LDPXpost $sp, 4
+# CHECK-COUNT-5: frame-destroy CFI_INSTRUCTION
 # CHECK: RET_ReallyLR
 #
 # ASM-LABEL: save_restore_sve:
 # ASM:       .cfi_def_cfa_offset 32
 # ASM:       .cfi_offset w19, -8
 # ASM-NEXT:  .cfi_offset w20, -16
 # ASM-NEXT:  .cfi_offset w21, -24
 # ASM-NEXT:  .cfi_offset w29, -32
-# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG 
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 32 - 8 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 32 - 16 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 32 - 24 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 32 - 32 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 32 - 40 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 32 - 48 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 32 - 56 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 32 - 64 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 152 * VG
 # ASM:       .cfi_escape 0x0f, 0x0e, 0x8f, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 64 + 152 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 152 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM-NEXT:  .cfi_restore z8
+# ASM-NEXT:  .cfi_restore z9
+# ASM-NEXT:  .cfi_restore z10
+# ASM-NEXT:  .cfi_restore z11
+# ASM-NEXT:  .cfi_restore z12
+# ASM-NEXT:  .cfi_restore z13
+# ASM-NEXT:  .cfi_restore z14
+# ASM-NEXT:  .cfi_restore z15
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w19
+# ASM-NEXT:  .cfi_restore w20
+# ASM-NEXT:  .cfi_restore w21
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +32
 # UNWINDINFO:      DW_CFA_offset: reg19 -8
 # UNWINDINFO-NEXT: DW_CFA_offset: reg20 -16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg21 -24
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +144, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -32, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -40, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -48, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -56, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +64, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +144, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg107
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg108
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg109
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg110
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg111
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg19
+# UNWINDINFO-NEXT: DW_CFA_restore: reg20
+# UNWINDINFO-NEXT: DW_CFA_restore: reg21
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
 
 name: save_restore_sve
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
   - { id: 1, stack-id: default, size: 32, alignment: 16 }
 body:             |
   bb.0.entry:
 

 # CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 4
 # CHECK-NEXT: $p14 = frame-destroy LDR_PXI $sp, 5
 # CHECK:      $p5 = frame-destroy LDR_PXI $sp, 14
 # CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 15
 # CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 2
 # CHECK-NEXT: $z22 = frame-destroy LDR_ZXI $sp, 3
 # CHECK:      $z9 = frame-destroy LDR_ZXI $sp, 16
 # CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 17
+# CHECK-COUNT-8: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
+# CHECK-COUNT-3: frame-destroy CFI_INSTRUCTION
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: save_restore_sve_realign:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM:       .cfi_def_cfa w29, 16
 # ASM-NEXT:  .cfi_offset w30, -8
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9  @ cfa - 16 - 16 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10  @ cfa - 16 - 24 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11  @ cfa - 16 - 32 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12  @ cfa - 16 - 40 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13  @ cfa - 16 - 48 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14  @ cfa - 16 - 56 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15  @ cfa - 16 - 64 * VG
 #
+# ASM:       .cfi_restore z8
+# ASM-NEXT:  .cfi_restore z9
+# ASM-NEXT:  .cfi_restore z10
+# ASM-NEXT:  .cfi_restore z11
+# ASM-NEXT:  .cfi_restore z12
+# ASM-NEXT:  .cfi_restore z13
+# ASM-NEXT:  .cfi_restore z14
+# ASM-NEXT:  .cfi_restore z15
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w30
+# ASM-NEXT:  .cfi_restore w29
+#
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:      DW_CFA_def_cfa: reg29 +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -32, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -40, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -48, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -56, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_restore_extended: reg104
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg107
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg108
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg109
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg110
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg111
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg30
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name: save_restore_sve_realign
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
   - { id: 1, stack-id: default, size: 16, alignment: 32 }
 body:             |
   bb.0.entry:
 
     $z8_z9_z10_z11   = IMPLICIT_DEF

 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: STR_PXI killed $p15, $sp, 6
 # CHECK-NEXT: STR_PXI killed $p4, $sp, 7
 # CHECK-NEXT: STR_ZXI killed $z23, $sp, 1
 # CHECK-NEXT: STR_ZXI killed $z8, $sp, 2
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -7
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION
+
+# CHECK:      $sp = frame-destroy ADDVL_XXI $sp, 7
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 6
+# CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 7
+# CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 1
+# CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 2
+# CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.10)
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION
 #
 # ASM-LABEL: frame_layout:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 80 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM-NEXT:  .cfi_restore z8
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +80, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name: frame_layout
 stack:
   - { id: 0, type: default,    size:  32, alignment: 16, stack-id: scalable-vector }
   - { id: 1, type: default,    size:   4, alignment:  2, stack-id: scalable-vector }
   - { id: 2, type: default,    size:  16, alignment: 16, stack-id: scalable-vector }
   - { id: 3, type: default,    size:   2, alignment:  2, stack-id: scalable-vector }
   - { id: 4, type: spill-slot, size:  16, alignment: 16, stack-id: scalable-vector }
   - { id: 5, type: spill-slot, size:   2, alignment:  2, stack-id: scalable-vector }

llvm/test/CodeGen/AArch64/framelayout-unaligned-fp.ll

 ; CHECK-LABEL: b:
 ; CHECK:       str     d8, [sp, #-32]!
 ; CHECK-NEXT:  .cfi_def_cfa_offset 32
 ; CHECK-NEXT:  stp     x29, x30, [sp, #8]
 ; CHECK-NEXT:  str     x19, [sp, #24]
 ; CHECK-NEXT:  add     x29, sp, #8
 
 ; CHECK:       sub     sp, x29, #8
+; CHECK-NEXT:  .cfi_def_cfa wsp, 32
 ; CHECK-NEXT:  ldr     x19, [sp, #24]
 ; CHECK-NEXT:  ldp     x29, x30, [sp, #8]
 ; CHECK-NEXT:  ldr     d8, [sp], #32
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w19
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: .cfi_restore b8
 ; CHECK-NEXT:  ret
 
 declare i64 @d()
 declare i64 @e()
 declare i64 @f(i64)

llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll

 ; RUN: llc < %s | FileCheck %s
 
 target triple = "aarch64--linux-android"
 
 define i8* @f1(i8* %x0, i8* %x1) {
   ; CHECK: f1:
   ; CHECK: str x30, [sp, #-16]!
   ; CHECK-NEXT: .cfi_def_cfa_offset 16
   ; CHECK-NEXT: .cfi_offset w30, -16
   ; CHECK-NEXT: mov x9, x0
   ; CHECK-NEXT: mov x0, x1
   ; CHECK-NEXT: bl __hwasan_check_x1_1
   ; CHECK-NEXT: ldr x30, [sp], #16
+  ; CHECK-NEXT: .cfi_def_cfa_offset 0
+  ; CHECK-NEXT: .cfi_restore w30
   ; CHECK-NEXT: ret
   call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 1)
   ret i8* %x1
 }
 
 define i8* @f2(i8* %x0, i8* %x1) {
   ; CHECK: f2:
   ; CHECK: stp x30, x20, [sp, #-16]!
   ; CHECK-NEXT: .cfi_def_cfa_offset 16
   ; CHECK-NEXT: .cfi_offset w20, -8
   ; CHECK-NEXT: .cfi_offset w30, -16
   ; CHECK-NEXT: mov x20, x1
   ; CHECK-NEXT: bl __hwasan_check_x0_2_short_v2
   ; CHECK-NEXT: ldp x30, x20, [sp], #16
+  ; CHECK-NEXT: .cfi_def_cfa_offset 0
+  ; CHECK-NEXT: .cfi_restore w20
+  ; CHECK-NEXT: .cfi_restore w30
   ; CHECK-NEXT: ret
   call void @llvm.hwasan.check.memaccess.shortgranules(i8* %x1, i8* %x0, i32 2)
   ret i8* %x0
 }
 
 define void @f3(i8* %x0, i8* %x1) {
   ; 0x3ff0000 (kernel, match-all = 0xff)
   call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 67043328)

llvm/test/CodeGen/AArch64/isinf.ll

 ; CHECK-NEXT:    adrp x8, .LCPI3_0
 ; CHECK-NEXT:    ldr q0, [sp]
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
 ; CHECK-NEXT:    bl __eqtf2
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %abs = tail call fp128 @llvm.fabs.f128(fp128 %x)
   %cmpinf = fcmp oeq fp128 %abs, 0xL00000000000000007FFF000000000000
   %ret = zext i1 %cmpinf to i32
   ret i32 %ret
 }

llvm/test/CodeGen/AArch64/large-stack.ll

 ; CHECK-NEXT:             add   x[[VAL3:[0-9]+]], sp, #8
 ; CHECK-NEXT:             mov x[[VAL4:[0-9]+]], #8
 ; CHECK-NEXT:             madd  x[[INDEX]], x[[INDEX]], x[[VAL4]], x[[VAL3]]
 ; CHECK-NEXT:             ldr x1, [x[[INDEX]]
 ; CHECK:                  bl  printf
 ; CHECK-COUNT-128:        add sp, sp, #[[STACK1]], lsl #12
 ; CHECK-NEXT:             add sp, sp, #[[STACK2]], lsl #12
 ; CHECK-NEXT:             add sp, sp, #[[STACK3]]
+; CHECK-NEXT:             .cfi_def_cfa wsp, [[SPILL_OFFSET1]]
 ; CHECK-NEXT:             ldr x[[SPILL_REG3]], [sp, #[[SPILL_OFFSET2]]]
 ; CHECK-NEXT:             ldp x[[SPILL_REG1]], x[[SPILL_REG2]], [sp], #[[SPILL_OFFSET1]]
+; CHECK-NEXT:             .cfi_def_cfa_offset 0
+; CHECK-NEXT:             .cfi_restore w[[SPILL_REG3]]
+; CHECK-NEXT:             .cfi_restore w[[SPILL_REG2]]
+; CHECK-NEXT:             .cfi_restore w[[SPILL_REG1]]

llvm/test/CodeGen/AArch64/ldst-paired-aliasing.ll

 ; CHECK-NEXT:    bl f
 ; CHECK-NEXT:    b .LBB0_3
 ; CHECK-NEXT:  .LBB0_2: // %if.then
 ; CHECK-NEXT:    bl f2
 ; CHECK-NEXT:  .LBB0_3: // %common.ret
 ; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
 
 for.body.lr.ph.i.i.i.i.i.i63:
   %b1 = alloca [10 x i32], align 16
   %x0 = bitcast [10 x i32]* %b1 to i8*
   %b2 = alloca [10 x i32], align 16
   %x1 = bitcast [10 x i32]* %b2 to i8*

llvm/test/CodeGen/AArch64/local_vars.ll

 
 ; Dont't really care, but it would be a Bad Thing if this came after the epilogue.
 ; CHECK-WITHFP-ARM64: bl foo
 ; CHECK: bl foo
   call void @foo()
   ret void
 
 ; CHECK: ldr x30, [sp], #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
 ; CHECK-NEXT: ret
 
-; CHECK-WITHFP-ARM64: ldp x29, x30, [sp], #16
+; CHECK-WITHFP-ARM64:      .cfi_def_cfa wsp, 16
+; CHECK-WITHFP-ARM64-NEXT: ldp x29, x30, [sp], #16
+; CHECK-WITHFP-ARM64-NEXT: .cfi_def_cfa_offset 0
+; CHECK-WITHFP-ARM64-NEXT: .cfi_restore w30
+; CHECK-WITHFP-ARM64-NEXT: .cfi_restore w29
 ; CHECK-WITHFP-ARM64-NEXT: ret
 }
 
 define void @stack_local() {
   %local_var = alloca i64
 ; CHECK-LABEL: stack_local:
 ; CHECK: sub sp, sp, #16
 

llvm/test/CodeGen/AArch64/ls64-inline-asm.ll

 ; CHECK-NEXT:    ldrsw x6, [x0, #100]
 ; CHECK-NEXT:    ldrsw x7, [x0, #144]
 ; CHECK-NEXT:    ldrsw x8, [x0, #196]
 ; CHECK-NEXT:    ldrsw x9, [x0, #256]
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    st64b x2, [x1]
 ; CHECK-NEXT:    //NO_APP
 ; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, i32* %in, align 4
   %conv = sext i32 %0 to i64
   %arrayidx1 = getelementptr inbounds i32, i32* %in, i64 1
   %1 = load i32, i32* %arrayidx1, align 4
   %conv2 = sext i32 %1 to i64
   %arrayidx4 = getelementptr inbounds i32, i32* %in, i64 4

llvm/test/CodeGen/AArch64/machine-licm-sink-instr.ll

 ; CHECK-NEXT:    b.ne .LBB0_2
 ; CHECK-NEXT:    b .LBB0_4
 ; CHECK-NEXT:  .LBB0_3:
 ; CHECK-NEXT:    mov w20, w19
 ; CHECK-NEXT:  .LBB0_4: // %for.cond.cleanup
 ; CHECK-NEXT:    mov w0, w20
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %cmp63 = icmp sgt i32 %n, 0
   br i1 %cmp63, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:
   %0 = load i32, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @A, i64 0, i64 0), align 4
   br label %for.body

 ; CHECK-NEXT:    b.ne .LBB1_2
 ; CHECK-NEXT:    b .LBB1_4
 ; CHECK-NEXT:  .LBB1_3:
 ; CHECK-NEXT:    mov w21, w19
 ; CHECK-NEXT:  .LBB1_4: // %for.cond.cleanup
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov w0, w21
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %cmp63 = icmp sgt i32 %n, 0
   br i1 %cmp63, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:
   %0 = load i32, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @A, i64 0, i64 0), align 4
   %call0 = tail call i32 @_Z3usei(i32 %n)

 ; CHECK-NEXT:    b.ne .LBB2_2
 ; CHECK-NEXT:    b .LBB2_4
 ; CHECK-NEXT:  .LBB2_3:
 ; CHECK-NEXT:    mov w20, w19
 ; CHECK-NEXT:  .LBB2_4: // %for.cond.cleanup
 ; CHECK-NEXT:    mov w0, w20
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %cmp63 = icmp sgt i32 %n, 0
   br i1 %cmp63, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:
   %0 = load i32, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @A, i64 0, i64 0), align 4
   store i32 42, i32* %store, align 4

llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll

define void @bar() #0 !dbg !27 {
   store i32 2, i32* %2, align 4
   store i32 3, i32* %3, align 4
   store i32 4, i32* %4, align 4
   store i32 5, i32* %5, align 4
   store i32 6, i32* %6, align 4, !dbg !35
   ret void
 }
 
-attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="none" "target-cpu"="cyclone" }
+attributes #0 = { noredzone nounwind ssp uwtable minsize "frame-pointer"="none" "target-cpu"="cyclone" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
 !llvm.ident = !{!7}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "machine-outliner-remarks.ll", directory: "/tmp")
 !2 = !{}

llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll

 ; CHECK-NEXT:       .cfi_negate_ra_state
 ; CHECK-NEXT:       ret
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 88
   ret i32 %add
 }
 
-attributes #0 = { "sign-return-address"="non-leaf" }
+attributes #0 = { "sign-return-address"="non-leaf" minsize }
 
 ; CHECK-NOT:        OUTLINED_FUNCTION_{{.*}}
 ; CHECK-NOT:         .cfi_b_key_frame
 ; CHECK-NOT:         paci{{[a,b]}}sp
 ; CHECK-NOT:         hint #2{{[5,7]}}
 ; CHECK-NOT:         .cfi_negate_ra_state
 ; CHECK-NOT:         auti{{[a,b]}}sp
 ; CHECK-NOT:         hint #{{[29,31]}}

llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll

 ; CHECK-LABEL: a:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_1
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
   %cx = add i32 %call, 8
   ret i32 %cx
 }
 
 define i32 @b() {
 ; CHECK-LABEL: b:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_1
 ; CHECK-NEXT:    add w0, w0, #88
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
   %cx = add i32 %call, 88
   ret i32 %cx
 }
 
 define hidden i32 @c(i32 (i32, i32, i32, i32)* %fptr) {
 ; CHECK-LABEL: c:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 8
   ret i32 %add
 }
 
 define hidden i32 @d(i32 (i32, i32, i32, i32)* %fptr) {
 ; CHECK-LABEL: d:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    add w0, w0, #88
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 88
   ret i32 %add
 }
 
 ; CHECK: OUTLINED_FUNCTION_0:

llvm/test/CodeGen/AArch64/merge-store-dependency.ll

 ; A53-NEXT:    lsl w9, w9, #1
 ; A53-NEXT:    cmp w9, w8
 ; A53-NEXT:    b.le .LBB0_2
 ; A53-NEXT:  // %bb.3: // %while.end.i
 ; A53-NEXT:    bl foo
 ; A53-NEXT:    adrp x8, gv1
 ; A53-NEXT:    str x0, [x8, :lo12:gv1]
 ; A53-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; A53-NEXT:    .cfi_def_cfa_offset 0
+; A53-NEXT:    .cfi_restore w19
+; A53-NEXT:    .cfi_restore w30
 ; A53-NEXT:    ret
 ; A53-NEXT:  .LBB0_4: // %while.body.i.split
 ; A53-NEXT:    // =>This Inner Loop Header: Depth=1
 ; A53-NEXT:    b .LBB0_4
 entry:
   %0 = bitcast %struct1* %fde to i8*
   tail call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 40, i1 false)
   %state = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 4

llvm/test/CodeGen/AArch64/neg-imm.ll

 ; CHECK-NEXT:  // %bb.3: // %if.then3
 ; CHECK-NEXT:    // in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    mov w0, w20
 ; CHECK-NEXT:    bl foo
 ; CHECK-NEXT:    b .LBB0_1
 ; CHECK-NEXT:  .LBB0_4: // %for.cond.cleanup
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %sub = add nsw i32 %px, -1
   %cmp = icmp slt i32 %px, 1
   %.sub = select i1 %cmp, i32 0, i32 %sub
   br label %for.body
 
 for.body:

llvm/test/CodeGen/AArch64/optimize-cond-branch.ll

 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    cbz wzr, .LBB0_4
 ; CHECK-NEXT:  // %bb.2: // %b3
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    and w0, w8, #0x100
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    cbz w0, .LBB0_5
 ; CHECK-NEXT:  .LBB0_3: // %common.ret.sink.split
 ; CHECK-NEXT:    b extfunc
 ; CHECK-NEXT:  .LBB0_4: // %b2
 ; CHECK-NEXT:    bl extfunc
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    cbnz w0, .LBB0_3
 ; CHECK-NEXT:  .LBB0_5: // %common.ret
 ; CHECK-NEXT:    ret
   %c0 = icmp sgt i64 0, 0
   br i1 %c0, label %b1, label %b6
 
 b1:
   br i1 undef, label %b3, label %b2

llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll

 ; CHECK-NEXT:    stp q22, q21, [x8, #272]
 ; CHECK-NEXT:    stp q20, q19, [x8, #304]
 ; CHECK-NEXT:    stp q18, q17, [x8, #336]
 ; CHECK-NEXT:    stp q16, q7, [x8, #368]
 ; CHECK-NEXT:    stp q6, q5, [x8, #400]
 ; CHECK-NEXT:    stp q4, q3, [x8, #432]
 ; CHECK-NEXT:    str q0, [x8, #496]
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
 ; CHECK-NEXT:    ret
 ; CH`ECK-NEXT:    .cfi_offset b9, -16
 entry:
   br label %for.cond1.preheader
 
 for.cond1.preheader:                              ; preds = %for.cond1.preheader, %entry
   %0 = phi <2 x i64> [ undef, %entry ], [ %118, %for.cond1.preheader ]
   %1 = phi <2 x i64> [ undef, %entry ], [ %116, %for.cond1.preheader ]

llvm/test/CodeGen/AArch64/settag.ll

 }
 
 define void @stg_alloca1() {
 ; CHECK-LABEL: stg_alloca1:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    stg sp, [sp], #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
   %a = alloca i8, i32 16, align 16
   call void @llvm.aarch64.settag(i8* %a, i64 16)
   ret void
 }
 
 define void @stg_alloca5() {
 ; CHECK-LABEL: stg_alloca5:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #80
 ; CHECK-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-NEXT:    st2g sp, [sp, #32]
 ; CHECK-NEXT:    stg sp, [sp, #64]
 ; CHECK-NEXT:    st2g sp, [sp], #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
   %a = alloca i8, i32 80, align 16
   call void @llvm.aarch64.settag(i8* %a, i64 80)
   ret void
 }
 
 define void @stg_alloca17() {
 ; CHECK-LABEL: stg_alloca17:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #288
 ; CHECK-NEXT:    .cfi_def_cfa_offset 288
 ; CHECK-NEXT:    str x29, [sp, #272] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    mov x8, #256
 ; CHECK-NEXT:  .LBB11_1: // %entry
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    st2g sp, [sp], #32
 ; CHECK-NEXT:    sub x8, x8, #32
 ; CHECK-NEXT:    cbnz x8, .LBB11_1
 ; CHECK-NEXT:  // %bb.2: // %entry
 ; CHECK-NEXT:    stg sp, [sp], #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 entry:
   %a = alloca i8, i32 272, align 16
   call void @llvm.aarch64.settag(i8* %a, i64 272)
   ret void
 }
 
 declare void @llvm.aarch64.settag(i8* %p, i64 %a)
 declare void @llvm.aarch64.settag.zero(i8* %p, i64 %a)

llvm/test/CodeGen/AArch64/shrink-wrapping-vla.ll

 
 ; Check that llvm.stackrestore() happens before CSRs are popped off the stack
 
 ; CHECK-LABEL: f
 
 ; CHECK:      stp x29, x30, [sp, #-16]!
 ; CHECK-NEXT: .cfi_def_cfa_offset 16
 ; CHECK-NEXT: mov x29, sp
+; CHECK-NEXT: .cfi_def_cfa w29, 16
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
+
 
 ; VLA allocation
 ; CHECK: mov [[X2:x[0-9]+]], sp
 ; CHECK: mov [[SAVE:x[0-9]+]], sp
 ; CHECK: add [[X1:x[0-9]+]], [[X1]], #15
 ; CHECK: and [[X1]], [[X1]], #0x7fffffff0
 ; Saving the SP via llvm.stacksave()
 ; CHECK: sub [[X2]], [[X2]], [[X1]]
 
 ; The next instruction comes from llvm.stackrestore()
 ; CHECK:      mov sp, [[SAVE]]
 ; Epilogue
 ; CHECK-NEXT: mov sp, x29
+; CHECK-NEXT: .cfi_def_cfa wsp, 16
 ; CHECK-NEXT: ldp x29, x30, [sp], #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT:  ret

llvm/test/CodeGen/AArch64/sibling-call.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #42
 ; CHECK-NEXT:    str x8, [sp]
 ; CHECK-NEXT:    bl callee_stack8
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
 ; Caller isn't going to clean up any extra stack we allocate, so it
 ; can't be a tail call.
   tail call void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 }
 

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl callee_stack16
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
 ; Shouldn't be a tail call: we can't use SP+8 because our caller might
 ; have something there. This may sound obvious but implementation does
 ; some funky aligning.
   tail call void @callee_stack16([8 x i64] undef, i64 undef, i64 undef)
   ret void
 }

llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll

 entry:
 ; CHECK-LABEL: deadload
 ; CHECK:       cmp     sp, #0
 ; CHECK-NEXT:  csetm   x16, ne
 ; CHECK-NEXT:  sub     sp, sp, #16
 ; CHECK-NEXT:  .cfi_def_cfa_offset 16
 ; CHECK-NEXT:  ldr     w8, [sp, #12]
 ; CHECK-NEXT:  add     sp, sp, #16
+; CHECK-NEXT:  .cfi_def_cfa_offset 0
 ; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
 ; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
 ; CHECK-NEXT:  mov     sp, [[TMPREG]]
 ; CHECK-NEXT:  ret
   %a = alloca i32, align 4
   %val = load volatile i32, i32* %a, align 4
   ret i32 undef
 }

llvm/test/CodeGen/AArch64/split-vector-insert.ll

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #2, mul vl]
 ; CHECK-NEXT:    csel x8, x8, x9, lo
 ; CHECK-NEXT:    addvl x9, sp, #3
 ; CHECK-NEXT:    lsl x8, x8, #3
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #3, mul vl]
 ; CHECK-NEXT:    str q4, [x9, x8]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #3, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
 
 
   %r = call <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> %a, <8 x i64> %b, i64 0)
   ret <vscale x 2 x i64> %r
 }
 
 define <vscale x 2 x double> @test_nxv2f64_v8f64(<vscale x 2 x double> %a, <8 x double> %b) #0 {

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #2, mul vl]
 ; CHECK-NEXT:    csel x8, x8, x9, lo
 ; CHECK-NEXT:    addvl x9, sp, #3
 ; CHECK-NEXT:    lsl x8, x8, #3
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #3, mul vl]
 ; CHECK-NEXT:    str q4, [x9, x8]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #3, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:   .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
 
 
   %r = call <vscale x 2 x double> @llvm.experimental.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> %a, <8 x double> %b, i64 0)
   ret <vscale x 2 x double> %r
 }

llvm/test/CodeGen/AArch64/stack-guard-sysreg.ll

 ; CHECK-257-OFFSET-NEXT: ldr x8, [x8]
 ; CHECK-MINUS-257-OFFSET:      sub x8, x8, #257
 ; CHECK-MINUS-257-OFFSET-NEXT: ldr x8, [x8]
 ; CHECK-NEXT:    ldur x9, [x29, #-8]
 ; CHECK-NEXT:    cmp x8, x9
 ; CHECK-NEXT:    b.ne .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w30
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %entry
 ; CHECK-NEXT:    bl __stack_chk_fail
 ; CHECK-NOT: __stack_chk_guard
 entry:
   %vla = alloca i32, i64 %t, align 4
   call void @baz(i32* nonnull %vla)
   ret void

llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll

 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    str x0, [sp, #8]
 ; CHECK-NEXT:    bl return_i1
 ; CHECK-NEXT:  .Ltmp7:
 ; CHECK-NEXT:    and w0, w0, #0x1
 ; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 ; Check that an ununsed relocate has no code-generation impact
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
   %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
   %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
   ret i1 %call2
 }

 ; CHECK-NEXT:    bl consume
 ; CHECK-NEXT:    b .LBB8_3
 ; CHECK-NEXT:  .LBB8_2:
 ; CHECK-NEXT:    mov w19, #1
 ; CHECK-NEXT:  .LBB8_3: // %common.ret
 ; CHECK-NEXT:    and w0, w19, #0x1
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
   br i1 %external_cond, label %left, label %right
 
 left:
   %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
   %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)

 ; CHECK-NEXT:    ldr x8, [sp, #48]
 ; CHECK-NEXT:    mov w0, #42
 ; CHECK-NEXT:    mov w1, #17
 ; CHECK-NEXT:    str q0, [sp]
 ; CHECK-NEXT:    str x8, [sp, #16]
 ; CHECK-NEXT:    bl consume_attributes
 ; CHECK-NEXT:  .Ltmp11:
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
 ; Check that arguments with attributes are lowered correctly.
 ; We call a function that has a nest argument and a byval argument.
   %statepoint_token = call token (i64, i32, void (i32, i8*, i32, %struct2*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32p0i8i32p0s_struct2sf(i64 0, i32 0, void (i32, i8*, i32, %struct2*)* @consume_attributes, i32 4, i32 0, i32 42, i8* nest null, i32 17, %struct2* byval(%struct2) %s, i32 0, i32 0)
   ret void
 }
 

llvm/test/CodeGen/AArch64/storepairsuppress_minsize.ll

 ; CHECK-NEXT:    add x8, x8, :lo12:in_block_store
 ; CHECK-NEXT:    str d0, [x8]
 ; CHECK-NEXT:    str d1, [x8, #8]
 ; CHECK-NEXT:    str d2, [x8, #16]
 ; CHECK-NEXT:    str d3, [x8, #24]
 ; CHECK-NEXT:    str d4, [x8, #32]
 ; CHECK-NEXT:    str d5, [x8, #40]
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = call %T_IN_BLOCK @return_in_block()
   store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store
   ret void
 }
 
 define void @test_minsize() minsize {
 ; CHECK-LABEL: test_minsize:

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl return_in_block
 ; CHECK-NEXT:    adrp x8, in_block_store
 ; CHECK-NEXT:    add x8, x8, :lo12:in_block_store
 ; CHECK-NEXT:    stp d0, d1, [x8]
 ; CHECK-NEXT:    stp d2, d3, [x8, #16]
 ; CHECK-NEXT:    stp d4, d5, [x8, #32]
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = call %T_IN_BLOCK @return_in_block()
   store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store
   ret void
 }

llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll

 ; CHECK-NEXT:    uzp1 p3.s, p5.s, p3.s
 ; CHECK-NEXT:    uzp1 p0.s, p0.s, p1.s
 ; CHECK-NEXT:    uzp1 p1.h, p2.h, p4.h
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    uzp1 p0.h, p0.h, p3.h
 ; CHECK-NEXT:    uzp1 p0.b, p0.b, p1.b
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = call <vscale x 14 x i1> @llvm.experimental.vector.extract.nxv14i1.nxv28i1(<vscale x 28 x i1> %in, i64 14)
   ret <vscale x 14 x i1> %res
 }
 
 declare <vscale x 14 x i1> @llvm.experimental.vector.extract.nxv14i1.nxv28i1(<vscale x 28 x i1>, i64)
 
 ;

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-vselect.ll

 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.h, z0.h, #0x1
 ; CHECK-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; CHECK-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w30
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x half>, <16 x half>* %a
   %op2 = load <16 x half>, <16 x half>* %b
   %sel = select <16 x i1> %mask, <16 x half> %op1, <16 x half> %op2
   store <16 x half> %sel, <16 x half>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_512-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:   .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:   .cfi_restore w30
+; VBITS_GE_512-NEXT:   .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x half>, <32 x half>* %a
   %op2 = load <32 x half>, <32 x half>* %b
   %sel = select <32 x i1> %mask, <32 x half> %op1, <32 x half> %op2
   store <32 x half> %sel, <32 x half>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; VBITS_GE_1024-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_1024-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_1024-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:   .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:   .cfi_restore w30
+; VBITS_GE_1024-NEXT:   .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x half>, <64 x half>* %a
   %op2 = load <64 x half>, <64 x half>* %b
   %sel = select <64 x i1> %mask, <64 x half> %op1, <64 x half> %op2
   store <64 x half> %sel, <64 x half>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; VBITS_GE_2048-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_2048-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_2048-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <128 x i1>, <128 x i1>* %c
   %op1 = load <128 x half>, <128 x half>* %a
   %op2 = load <128 x half>, <128 x half>* %b
   %sel = select <128 x i1> %mask, <128 x half> %op1, <128 x half> %op2
   store <128 x half> %sel, <128 x half>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.s, z0.s, #0x1
 ; CHECK-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; CHECK-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w30
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <8 x i1>, <8 x i1>* %c
   %op1 = load <8 x float>, <8 x float>* %a
   %op2 = load <8 x float>, <8 x float>* %b
   %sel = select <8 x i1> %mask, <8 x float> %op1, <8 x float> %op2
   store <8 x float> %sel, <8 x float>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_512-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:   .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:   .cfi_restore w30
+; VBITS_GE_512-NEXT:   .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x float>, <16 x float>* %a
   %op2 = load <16 x float>, <16 x float>* %b
   %sel = select <16 x i1> %mask, <16 x float> %op1, <16 x float> %op2
   store <16 x float> %sel, <16 x float>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; VBITS_GE_1024-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_1024-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:   .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:   .cfi_restore w30
+; VBITS_GE_1024-NEXT:   .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x float>, <32 x float>* %a
   %op2 = load <32 x float>, <32 x float>* %b
   %sel = select <32 x i1> %mask, <32 x float> %op1, <32 x float> %op2
   store <32 x float> %sel, <32 x float>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; VBITS_GE_2048-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_2048-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x float>, <64 x float>* %a
   %op2 = load <64 x float>, <64 x float>* %b
   %sel = select <64 x i1> %mask, <64 x float> %op1, <64 x float> %op2
   store <64 x float> %sel, <64 x float>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.d, z0.d, #0x1
 ; CHECK-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w30
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <4 x i1>, <4 x i1>* %c
   %op1 = load <4 x double>, <4 x double>* %a
   %op2 = load <4 x double>, <4 x double>* %b
   %sel = select <4 x i1> %mask, <4 x double> %op1, <4 x double> %op2
   store <4 x double> %sel, <4 x double>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_512-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:   .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:   .cfi_restore w30
+; VBITS_GE_512-NEXT:   .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <8 x i1>, <8 x i1>* %c
   %op1 = load <8 x double>, <8 x double>* %a
   %op2 = load <8 x double>, <8 x double>* %b
   %sel = select <8 x i1> %mask, <8 x double> %op1, <8 x double> %op2
   store <8 x double> %sel, <8 x double>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; VBITS_GE_1024-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_1024-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x double>, <16 x double>* %a
   %op2 = load <16 x double>, <16 x double>* %b
   %sel = select <16 x i1> %mask, <16 x double> %op1, <16 x double> %op2
   store <16 x double> %sel, <16 x double>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_2048-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x double>, <32 x double>* %a
   %op2 = load <32 x double>, <32 x double>* %b
   %sel = select <32 x i1> %mask, <32 x double> %op1, <32 x double> %op2
   store <32 x double> %sel, <32 x double>* %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-int-vselect.ll

 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x8]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; CHECK-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.b, z0.b, #0x1
 ; CHECK-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; CHECK-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:   .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %sel = select <32 x i1> %mask, <32 x i8> %op1, <32 x i8> %op2
   store <32 x i8> %sel, <32 x i8>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1b { z0.b }, p0/z, [x8]
 ; VBITS_GE_512-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.b, z0.b, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; VBITS_GE_512-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; VBITS_GE_512-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %sel = select <64 x i1> %mask, <64 x i8> %op1, <64 x i8> %op2
   store <64 x i8> %sel, <64 x i8>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1b { z0.b }, p0/z, [x8]
 ; VBITS_GE_1024-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.b, z0.b, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; VBITS_GE_1024-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; VBITS_GE_1024-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <128 x i1>, <128 x i1>* %c
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %sel = select <128 x i1> %mask, <128 x i8> %op1, <128 x i8> %op2
   store <128 x i8> %sel, <128 x i8>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1b { z0.b }, p0/z, [x8]
 ; VBITS_GE_2048-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.b, z0.b, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; VBITS_GE_2048-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; VBITS_GE_2048-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <256 x i1>, <256 x i1>* %c
   %op1 = load <256 x i8>, <256 x i8>* %a
   %op2 = load <256 x i8>, <256 x i8>* %b
   %sel = select <256 x i1> %mask, <256 x i8> %op1, <256 x i8> %op2
   store <256 x i8> %sel, <256 x i8>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.h, z0.h, #0x1
 ; CHECK-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; CHECK-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %sel = select <16 x i1> %mask, <16 x i16> %op1, <16 x i16> %op2
   store <16 x i16> %sel, <16 x i16>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_512-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %sel = select <32 x i1> %mask, <32 x i16> %op1, <32 x i16> %op2
   store <32 x i16> %sel, <32 x i16>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; VBITS_GE_1024-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_1024-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_1024-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %sel = select <64 x i1> %mask, <64 x i16> %op1, <64 x i16> %op2
   store <64 x i16> %sel, <64 x i16>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1h { z0.h }, p0/z, [x8]
 ; VBITS_GE_2048-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_2048-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_2048-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <128 x i1>, <128 x i1>* %c
   %op1 = load <128 x i16>, <128 x i16>* %a
   %op2 = load <128 x i16>, <128 x i16>* %b
   %sel = select <128 x i1> %mask, <128 x i16> %op1, <128 x i16> %op2
   store <128 x i16> %sel, <128 x i16>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.s, z0.s, #0x1
 ; CHECK-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; CHECK-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <8 x i1>, <8 x i1>* %c
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %sel = select <8 x i1> %mask, <8 x i32> %op1, <8 x i32> %op2
   store <8 x i32> %sel, <8 x i32>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_512-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %sel = select <16 x i1> %mask, <16 x i32> %op1, <16 x i32> %op2
   store <16 x i32> %sel, <16 x i32>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; VBITS_GE_1024-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_1024-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %sel = select <32 x i1> %mask, <32 x i32> %op1, <32 x i32> %op2
   store <32 x i32> %sel, <32 x i32>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [x8]
 ; VBITS_GE_2048-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_2048-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x i32>, <64 x i32>* %a
   %op2 = load <64 x i32>, <64 x i32>* %b
   %sel = select <64 x i1> %mask, <64 x i32> %op1, <64 x i32> %op2
   store <64 x i32> %sel, <64 x i32>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.d, z0.d, #0x1
 ; CHECK-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <4 x i1>, <4 x i1>* %c
   %op1 = load <4 x i64>, <4 x i64>* %a
   %op2 = load <4 x i64>, <4 x i64>* %b
   %sel = select <4 x i1> %mask, <4 x i64> %op1, <4 x i64> %op2
   store <4 x i64> %sel, <4 x i64>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_512-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <8 x i1>, <8 x i1>* %c
   %op1 = load <8 x i64>, <8 x i64>* %a
   %op2 = load <8 x i64>, <8 x i64>* %b
   %sel = select <8 x i1> %mask, <8 x i64> %op1, <8 x i64> %op2
   store <8 x i64> %sel, <8 x i64>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; VBITS_GE_1024-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_1024-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x i64>, <16 x i64>* %a
   %op2 = load <16 x i64>, <16 x i64>* %b
   %sel = select <16 x i1> %mask, <16 x i64> %op1, <16 x i64> %op2
   store <16 x i64> %sel, <16 x i64>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_2048-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i64>, <32 x i64>* %a
   %op2 = load <32 x i64>, <32 x i64>* %b
   %sel = select <32 x i1> %mask, <32 x i64> %op1, <32 x i64> %op2
   store <32 x i64> %sel, <32 x i64>* %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle.ll

 ; CHECK-NEXT:    mov z2.d, z1.d[1]
 ; CHECK-NEXT:    stp d1, d2, [sp, #16]
 ; CHECK-NEXT:    mov z1.d, z0.d[3]
 ; CHECK-NEXT:    mov z0.d, z0.d[2]
 ; CHECK-NEXT:    stp d0, d1, [sp]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %op1 = load <4 x double>, <4 x double>* %a
   %op2 = load <4 x double>, <4 x double>* %b
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   store <4 x double> %ret, <4 x double>* %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-insert-element.ll

 ; CHECK-NEXT:    strb w0, [x10, x8]
 ; CHECK-NEXT:    ld1b { z0.b }, p1/z, [sp]
 ; CHECK-NEXT:    ld1b { z1.b }, p1/z, [x10, #1, mul vl]
 ; CHECK-NEXT:    and z0.b, z0.b, #0x1
 ; CHECK-NEXT:    and z1.b, z1.b, #0x1
 ; CHECK-NEXT:    cmpne p0.b, p1/z, z0.b, #0
 ; CHECK-NEXT:    cmpne p1.b, p1/z, z1.b, #0
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = insertelement <vscale x 32 x i1> %val, i1 %elt, i32 %idx
   ret <vscale x 32 x i1> %res
 }

llvm/test/CodeGen/AArch64/sve-insert-vector.ll

 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x8, #2, mul vl]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x8, #3, mul vl]
 ; CHECK-NEXT:    ld1d { z3.d }, p0/z, [sp]
 ; CHECK-NEXT:    st1d { z2.d }, p0, [x0, #3, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0, #1, mul vl]
 ; CHECK-NEXT:    st1d { z3.d }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %v0 = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> undef, <2 x i64> %sv0, i64 0)
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> %v0, <2 x i64> %sv1, i64 4)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
 define void @insert_v2i64_nxv16i64_lo0(<2 x i64>* %psv, <vscale x 16 x i64>* %out) {

 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    str q0, [sp, #16]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [sp]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x1, #1, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x1]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %sv = load <2 x i64>, <2 x i64>* %psv
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> undef, <2 x i64> %sv, i64 2)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
 

llvm/test/CodeGen/AArch64/sve-ld1r.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    adrp x8, :got:g8
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:g8]
 ; CHECK-NEXT:    ldrb w8, [x8]
 ; CHECK-NEXT:    strb w8, [sp, #12]
 ; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [sp, #14]
 ; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
   %valp = alloca i8
   %valp2  = load volatile i8, i8* @g8
   store volatile i8 %valp2, i8* %valp
   %valp3 = getelementptr i8, i8* %valp, i32 2
   %val = load i8, i8* %valp3
   %1 = insertelement <vscale x 16 x i8> undef, i8 %val, i32 0
   %2 = shufflevector <vscale x 16 x i8> %1, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer

llvm/test/CodeGen/AArch64/sve-pred-arith.ll

 ; CHECK-NEXT:    eor p2.b, p8/z, p2.b, p7.b
 ; CHECK-NEXT:    eor p3.b, p8/z, p3.b, p6.b
 ; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = add <vscale x 64 x i1> %a, %b
   ret <vscale x 64 x i1> %res;
 }
 
 
 ; LEGAL SUBS
 

 ; CHECK-NEXT:    eor p2.b, p8/z, p2.b, p7.b
 ; CHECK-NEXT:    eor p3.b, p8/z, p3.b, p6.b
 ; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = sub <vscale x 64 x i1> %a, %b
   ret <vscale x 64 x i1> %res;
 }

llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll

 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    addvl x8, x8, #2
 ; CHECK-NEXT:    cmp x9, x8
 ; CHECK-NEXT:    st1b { z1.b }, p0, [x10, #1, mul vl]
 ; CHECK-NEXT:    csel x8, x9, x8, lo
 ; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
 ; CHECK-NEXT:    ldrb w0, [x10, x8]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 32 x i8> %a, i32 %idx
   ret i8 %ext
 }
 
 define i16 @split_extract_16i16_idx(<vscale x 16 x i16> %a, i32 %idx) {
 ; CHECK-LABEL: split_extract_16i16_idx:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    addvl x8, x8, #1
 ; CHECK-NEXT:    cmp x9, x8
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x10, #1, mul vl]
 ; CHECK-NEXT:    csel x8, x9, x8, lo
 ; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
 ; CHECK-NEXT:    ldrh w0, [x10, x8, lsl #1]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 16 x i16> %a, i32 %idx
   ret i16 %ext
 }
 
 define i32 @split_extract_8i32_idx(<vscale x 8 x i32> %a, i32 %idx) {
 ; CHECK-LABEL: split_extract_8i32_idx:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    mov x10, sp
 ; CHECK-NEXT:    cmp x9, x8
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    csel x8, x9, x8, lo
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x10, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    ldr w0, [x10, x8, lsl #2]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 8 x i32> %a, i32 %idx
   ret i32 %ext
 }
 
 define i64 @split_extract_8i64_idx(<vscale x 8 x i64> %a, i32 %idx) {
 ; CHECK-LABEL: split_extract_8i64_idx:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    csel x9, x10, x9, lo
 ; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #3, mul vl]
 ; CHECK-NEXT:    st1d { z2.d }, p0, [x8, #2, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
 ; CHECK-NEXT:    ldr x0, [x8, x9, lsl #3]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 8 x i64> %a, i32 %idx
   ret i64 %ext
 }
 
 ; EXTRACT VECTOR ELT, CONSTANT IDX
 
 define i16 @promote_extract_4i16(<vscale x 4 x i16> %a) {

 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    addvl x8, x8, #1
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    cmp x8, #128
 ; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
 ; CHECK-NEXT:    csel x8, x8, x10, lo
 ; CHECK-NEXT:    ldrh w0, [x9, x8, lsl #1]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 16 x i16> %a, i32 128
   ret i16 %ext
 }
 
 define i32 @split_extract_16i32(<vscale x 16 x i32> %a) {
 ; CHECK-LABEL: split_extract_16i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    cmp x8, x10
 ; CHECK-NEXT:    st1w { z3.s }, p0, [x9, #3, mul vl]
 ; CHECK-NEXT:    csel x8, x8, x10, lo
 ; CHECK-NEXT:    st1w { z2.s }, p0, [x9, #2, mul vl]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    ldr w0, [x9, x8, lsl #2]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 16 x i32> %a, i32 100000
   ret i32 %ext
 }
 
 define i64 @split_extract_4i64(<vscale x 4 x i64> %a) {
 ; CHECK-LABEL: split_extract_4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 ; CHECK-NEXT:    cntw x8
 ; CHECK-NEXT:    mov x9, sp
 ; CHECK-NEXT:    sub x8, x8, #1
 ; CHECK-NEXT:    mov w10, #10
 ; CHECK-NEXT:    cmp x8, #10
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    csel x8, x8, x10, lo
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
 ; CHECK-NEXT:    ldr x0, [x9, x8, lsl #3]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 4 x i64> %a, i32 10
   ret i64 %ext
 }

llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll

 ; CHECK-NEXT:    st1b { z1.b }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    cmp x1, x8
 ; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
 ; CHECK-NEXT:    csel x8, x1, x8, lo
 ; CHECK-NEXT:    strb w0, [x9, x8]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x9, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [sp]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 32 x i8> %a, i8 %elt, i64 %idx
   ret <vscale x 32 x i8> %ins
 }
 
 define <vscale x 8 x float> @split_insert_8f32_idx(<vscale x 8 x float> %a, float %elt, i64 %idx) {
 ; CHECK-LABEL: split_insert_8f32_idx:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    cmp x0, x8
 ; CHECK-NEXT:    csel x8, x0, x8, lo
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    str s2, [x9, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x9, #1, mul vl]
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 8 x float> %a, float %elt, i64 %idx
   ret <vscale x 8 x float> %ins
 }
 
 define <vscale x 8 x i64> @split_insert_8i64_idx(<vscale x 8 x i64> %a, i64 %elt, i64 %idx) {
 ; CHECK-LABEL: split_insert_8i64_idx:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    st1d { z1.d }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
 ; CHECK-NEXT:    str x0, [x9, x8, lsl #3]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x9, #1, mul vl]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x9, #2, mul vl]
 ; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x9, #3, mul vl]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 8 x i64> %a, i64 %elt, i64 %idx
   ret <vscale x 8 x i64> %ins
 }
 
 ; INSERT VECTOR ELT, CONSTANT IDX
 
 define <vscale x 4 x i16> @promote_insert_4i16(<vscale x 4 x i16> %a, i16 %elt) {

 ; CHECK-NEXT:    st1h { z1.h }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
 ; CHECK-NEXT:    strh w0, [x9, x8, lsl #1]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x9, #1, mul vl]
 ; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x9, #2, mul vl]
 ; CHECK-NEXT:    ld1h { z3.h }, p0/z, [x9, #3, mul vl]
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [sp]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 32 x i16> %a, i16 %elt, i64 128
   ret <vscale x 32 x i16> %ins
 }
 
 define <vscale x 8 x i32> @split_insert_8i32(<vscale x 8 x i32> %a, i32 %elt) {
 ; CHECK-LABEL: split_insert_8i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    csel x8, x8, x9, lo
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x10, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    str w0, [x10, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x10, #1, mul vl]
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 8 x i32> %a, i32 %elt, i64 1000000
   ret <vscale x 8 x i32> %ins
 }

llvm/test/CodeGen/AArch64/sve-split-int-pred-reduce.ll

 ; CHECK-NEXT:    and p1.b, p4/z, p1.b, p3.b
 ; CHECK-NEXT:    and p0.b, p4/z, p0.b, p2.b
 ; CHECK-NEXT:    and p0.b, p4/z, p0.b, p1.b
 ; CHECK-NEXT:    not p0.b, p4/z, p0.b
 ; CHECK-NEXT:    ptest p4, p0.b
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = call i1 @llvm.vector.reduce.and.nxv64i1(<vscale x 64 x i1> %a)
   ret i1 %res
 }
 
 ; ORV
 
 define i1 @orv_nxv32i1(<vscale x 32 x i1> %a) {

llvm/test/CodeGen/AArch64/sve-trunc.ll

 ; CHECK-NEXT:    uzp1 p3.s, p4.s, p3.s
 ; CHECK-NEXT:    uzp1 p0.s, p0.s, p5.s
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    uzp1 p1.h, p2.h, p1.h
 ; CHECK-NEXT:    uzp1 p0.h, p0.h, p3.h
 ; CHECK-NEXT:    uzp1 p0.b, p0.b, p1.b
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 entry:
   %out = trunc <vscale x 16 x i64> %in to <vscale x 16 x i1>
   ret <vscale x 16 x i1> %out
 }
 
 
 define <vscale x 4 x i1> @trunc_i32toi1(<vscale x 4 x i32> %in) {

llvm/test/CodeGen/AArch64/sve-varargs.ll

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x0, .str_1
 ; CHECK-NEXT:    add x0, x0, :lo12:.str_1
 ; CHECK-NEXT:    bl sve_printf
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %f = getelementptr [6 x i8], [6 x i8]* @.str_1, i64 0, i64 0
   call i32 (i8*, <vscale x 4 x i32>, ...) @sve_printf(i8* %f, <vscale x 4 x i32> %x)
   ret void
 }

llvm/test/CodeGen/AArch64/swifttail-call.ll

 
 define swifttailcc void @caller_to0_from8([8 x i64], i64) {
 ; COMMON-LABEL: caller_to0_from8:
 
   musttail call swifttailcc void @callee_stack0()
   ret void
 
 ; COMMON: add sp, sp, #16
+; COMMON-NEXT:  .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack0
 }
 
 define swifttailcc void @caller_to8_from0() {
 ; COMMON-LABEL: caller_to8_from0:
 
 ; Key point is that the "42" should go #16 below incoming stack
 ; pointer (we didn't have arg space to reuse).

 ; COMMON-LABEL: caller_to8_from24:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at #16 above SP on entry.
   musttail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack8
 }
 
 
 define swifttailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
 ; COMMON-LABEL: caller_to16_from16:
 ; COMMON-NOT: sub sp,
 

llvm/test/CodeGen/AArch64/tail-call.ll

 
 define fastcc void @caller_to0_from8([8 x i64], i64) {
 ; COMMON-LABEL: caller_to0_from8:
 
   tail call fastcc void @callee_stack0()
   ret void
 
 ; COMMON: add sp, sp, #16
+; COMMON: .cfi_def_cfa_offset  -16
 ; COMMON-NEXT: b callee_stack0
 }
 
 define fastcc void @caller_to8_from0() {
 ; COMMON-LABEL: caller_to8_from0:
 
 ; Key point is that the "42" should go #16 below incoming stack
 ; pointer (we didn't have arg space to reuse).

 ; COMMON-LABEL: caller_to8_from24:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at #16 above SP on entry.
   tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON: .cfi_def_cfa_offset  -16
 ; COMMON-NEXT: b callee_stack8
 }
 
 
 define fastcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
 ; COMMON-LABEL: caller_to16_from16:
 ; COMMON-NOT: sub sp,
 

llvm/test/CodeGen/AArch64/tailcc-tail-call.ll

 
 define tailcc void @caller_to0_from8([8 x i64], i64) {
 ; COMMON-LABEL: caller_to0_from8:
 
   tail call tailcc void @callee_stack0()
   ret void
 
 ; COMMON: add sp, sp, #16
+; COMMON: .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack0
 }
 
 define tailcc void @caller_to8_from0() "frame-pointer"="all"{
 ; COMMON-LABEL: caller_to8_from0:
 
 ; Key point is that the "42" should go #16 below incoming stack
 ; pointer (we didn't have arg space to reuse).
   tail call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [x29, #16]
 ; COMMON: ldp x29, x30, [sp], #16
   ; If there is a sub here then the 42 will be briefly exposed to corruption
   ; from an interrupt if the kernel does not honour a red-zone, and a larger
   ; call could well overflow the red zone even if it is present.
 ; COMMON-NOT: sub sp,
+; COMMON-NEXT: .cfi_def_cfa_offset 16
+; COMMON-NEXT: .cfi_restore w30
+; COMMON-NEXT: .cfi_restore w29
 ; COMMON-NEXT: b callee_stack8
 }
 
 define tailcc void @caller_to8_from8([8 x i64], i64 %a) {
 ; COMMON-LABEL: caller_to8_from8:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at SP on entry.

 ; COMMON-LABEL: caller_to8_from24:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at #16 above SP on entry.
   tail call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack8
 }
 
 
 define tailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
 ; COMMON-LABEL: caller_to16_from16:
 ; COMMON-NOT: sub sp,
 

llvm/test/CodeGen/AArch64/unwind-preserved.ll

 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    bl may_throw_sve
 ; CHECK-NEXT:  .Ltmp1:
 ; CHECK-NEXT:    str z0, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    b .LBB0_1
 ; CHECK-NEXT:  .LBB0_1: // %.Lcontinue
 ; CHECK-NEXT:    ldr z0, [sp, #1, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; CHECK-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
+; CHECK-NEXT:    .cfi_restore z8
+; CHECK-NEXT:    .cfi_restore z9
+; CHECK-NEXT:    .cfi_restore z10
+; CHECK-NEXT:    .cfi_restore z11
+; CHECK-NEXT:    .cfi_restore z12
+; CHECK-NEXT:    .cfi_restore z13
+; CHECK-NEXT:    .cfi_restore z14
+; CHECK-NEXT:    .cfi_restore z15
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %.Lunwind
 ; CHECK-NEXT:  .Ltmp2:
 ; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; CHECK-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
+; CHECK-NEXT:    .cfi_restore z8
+; CHECK-NEXT:    .cfi_restore z9
+; CHECK-NEXT:    .cfi_restore z10
+; CHECK-NEXT:    .cfi_restore z11
+; CHECK-NEXT:    .cfi_restore z12
+; CHECK-NEXT:    .cfi_restore z13
+; CHECK-NEXT:    .cfi_restore z14
+; CHECK-NEXT:    .cfi_restore z15
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: invoke_callee_may_throw_sve:
 ; GISEL:       .Lfunc_begin0:
 ; GISEL-NEXT:    .cfi_startproc
 ; GISEL-NEXT:  // %bb.0:
 ; GISEL-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; GISEL-NEXT:    .cfi_def_cfa_offset 16

 ; GISEL-NEXT:  .Ltmp0:
 ; GISEL-NEXT:    bl may_throw_sve
 ; GISEL-NEXT:  .Ltmp1:
 ; GISEL-NEXT:    str z0, [sp, #1, mul vl] // 16-byte Folded Spill
 ; GISEL-NEXT:    b .LBB0_1
 ; GISEL-NEXT:  .LBB0_1: // %.Lcontinue
 ; GISEL-NEXT:    ldr z0, [sp, #1, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #2
+; GISEL-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; GISEL-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; GISEL-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #18
+; GISEL-NEXT:    .cfi_def_cfa wsp, 16
+; GISEL-NEXT:    .cfi_restore z8
+; GISEL-NEXT:    .cfi_restore z9
+; GISEL-NEXT:    .cfi_restore z10
+; GISEL-NEXT:    .cfi_restore z11
+; GISEL-NEXT:    .cfi_restore z12
+; GISEL-NEXT:    .cfi_restore z13
+; GISEL-NEXT:    .cfi_restore z14
+; GISEL-NEXT:    .cfi_restore z15
 ; GISEL-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
 ; GISEL-NEXT:    ret
 ; GISEL-NEXT:  .LBB0_2: // %.Lunwind
 ; GISEL-NEXT:  .Ltmp2:
 ; GISEL-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #2
+; GISEL-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; GISEL-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; GISEL-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #18
+; GISEL-NEXT:    .cfi_def_cfa wsp, 16
+; GISEL-NEXT:    .cfi_restore z8
+; GISEL-NEXT:    .cfi_restore z9
+; GISEL-NEXT:    .cfi_restore z10
+; GISEL-NEXT:    .cfi_restore z11
+; GISEL-NEXT:    .cfi_restore z12
+; GISEL-NEXT:    .cfi_restore z13
+; GISEL-NEXT:    .cfi_restore z14
+; GISEL-NEXT:    .cfi_restore z15
 ; GISEL-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
 ; GISEL-NEXT:    ret
   %result = invoke <vscale x 4 x i32> @may_throw_sve(<vscale x 4 x i32> %v) to label %.Lcontinue unwind label %.Lunwind
 .Lcontinue:
   ret <vscale x 4 x i32> %result
 .Lunwind:
   %lp = landingpad { i8*, i32 } cleanup
   ret <vscale x 4 x i32> %v;
 }

 ; CHECK-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #304
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
+; CHECK-NEXT:    .cfi_restore b16
+; CHECK-NEXT:    .cfi_restore b17
+; CHECK-NEXT:    .cfi_restore b18
+; CHECK-NEXT:    .cfi_restore b19
+; CHECK-NEXT:    .cfi_restore b20
+; CHECK-NEXT:    .cfi_restore b21
+; CHECK-NEXT:    .cfi_restore b22
+; CHECK-NEXT:    .cfi_restore b23
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB1_2: // %.Lunwind
 ; CHECK-NEXT:  .Ltmp5:
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp q9, q8, [sp, #256] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #304
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
+; CHECK-NEXT:    .cfi_restore b16
+; CHECK-NEXT:    .cfi_restore b17
+; CHECK-NEXT:    .cfi_restore b18
+; CHECK-NEXT:    .cfi_restore b19
+; CHECK-NEXT:    .cfi_restore b20
+; CHECK-NEXT:    .cfi_restore b21
+; CHECK-NEXT:    .cfi_restore b22
+; CHECK-NEXT:    .cfi_restore b23
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: invoke_callee_may_throw_neon:
 ; GISEL:       .Lfunc_begin1:
 ; GISEL-NEXT:    .cfi_startproc
 ; GISEL-NEXT:  // %bb.0:
 ; GISEL-NEXT:    sub sp, sp, #304
 ; GISEL-NEXT:    .cfi_def_cfa_offset 304

 ; GISEL-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; GISEL-NEXT:    add sp, sp, #304
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
+; GISEL-NEXT:    .cfi_restore b8
+; GISEL-NEXT:    .cfi_restore b9
+; GISEL-NEXT:    .cfi_restore b10
+; GISEL-NEXT:    .cfi_restore b11
+; GISEL-NEXT:    .cfi_restore b12
+; GISEL-NEXT:    .cfi_restore b13
+; GISEL-NEXT:    .cfi_restore b14
+; GISEL-NEXT:    .cfi_restore b15
+; GISEL-NEXT:    .cfi_restore b16
+; GISEL-NEXT:    .cfi_restore b17
+; GISEL-NEXT:    .cfi_restore b18
+; GISEL-NEXT:    .cfi_restore b19
+; GISEL-NEXT:    .cfi_restore b20
+; GISEL-NEXT:    .cfi_restore b21
+; GISEL-NEXT:    .cfi_restore b22
+; GISEL-NEXT:    .cfi_restore b23
 ; GISEL-NEXT:    ret
 ; GISEL-NEXT:  .LBB1_2: // %.Lunwind
 ; GISEL-NEXT:  .Ltmp5:
 ; GISEL-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp q9, q8, [sp, #256] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; GISEL-NEXT:    add sp, sp, #304
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
+; GISEL-NEXT:    .cfi_restore b8
+; GISEL-NEXT:    .cfi_restore b9
+; GISEL-NEXT:    .cfi_restore b10
+; GISEL-NEXT:    .cfi_restore b11
+; GISEL-NEXT:    .cfi_restore b12
+; GISEL-NEXT:    .cfi_restore b13
+; GISEL-NEXT:    .cfi_restore b14
+; GISEL-NEXT:    .cfi_restore b15
+; GISEL-NEXT:    .cfi_restore b16
+; GISEL-NEXT:    .cfi_restore b17
+; GISEL-NEXT:    .cfi_restore b18
+; GISEL-NEXT:    .cfi_restore b19
+; GISEL-NEXT:    .cfi_restore b20
+; GISEL-NEXT:    .cfi_restore b21
+; GISEL-NEXT:    .cfi_restore b22
+; GISEL-NEXT:    .cfi_restore b23
 ; GISEL-NEXT:    ret
   %result = invoke aarch64_vector_pcs <4 x i32> @may_throw_neon(<4 x i32> %v) to label %.Lcontinue unwind label %.Lunwind
 .Lcontinue:
   ret <4 x i32> %result
 .Lunwind:
   %lp = landingpad { i8*, i32 } cleanup
   ret <4 x i32> %v;
 }
 
 declare aarch64_vector_pcs <4 x i32> @may_throw_neon(<4 x i32> %v);

llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll

 ; CHECK-NEXT:    // in Loop: Header=BB3_1 Depth=1
 ; CHECK-NEXT:    add w9, w20, #1
 ; CHECK-NEXT:    stp w9, w20, [x8]
 ; CHECK-NEXT:    mov w20, w9
 ; CHECK-NEXT:    b .LBB3_1
 ; CHECK-NEXT:  .LBB3_4: // %while_end
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB3_5: // %cleanup
 ; CHECK-NEXT:  .Ltmp2:
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl foo2
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    bl _Unwind_Resume
 entry:

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.generated.expected

 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    ldur w8, [x29, #-8]
 ; CHECK-NEXT:    cbnz w8, .LBB0_2
 ; CHECK-NEXT:  .LBB0_4:
 ; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:  .LBB0_5:
 ; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
 ; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-LABEL: main:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    stp x29, x30, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x29, sp, #32

 ; CHECK-NEXT:    mov w9, #4
 ; CHECK-NEXT:    stp w8, wzr, [x29, #-8]
 ; CHECK-NEXT:    stur w10, [x29, #-12]
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
 ; CHECK-NEXT:    stp w10, w8, [x29, #-12]
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
 ; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-LABEL: OUTLINED_FUNCTION_0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w9, #2
 ; CHECK-NEXT:    mov w10, #3
 ; CHECK-NEXT:    mov w11, #4
 ; CHECK-NEXT:    stp w9, w8, [x29, #-12]
 ; CHECK-NEXT:    stp w11, w10, [sp, #12]
 ; CHECK-NEXT:    ret

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.nogenerated.expected

 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    ldur w8, [x29, #-8]
 ; CHECK-NEXT:    cbnz w8, .LBB0_2
 ; CHECK-NEXT:  .LBB0_4:
 ; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:  .LBB0_5:
 ; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
 ; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
   %4 = alloca i32, align 4
   %5 = alloca i32, align 4
   store i32 0, i32* %1, align 4
   store i32 0, i32* %2, align 4

 ; CHECK-NEXT:    mov w9, #4
 ; CHECK-NEXT:    stp w8, wzr, [x29, #-8]
 ; CHECK-NEXT:    stur w10, [x29, #-12]
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
 ; CHECK-NEXT:    stp w10, w8, [x29, #-12]
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
 ; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
   %4 = alloca i32, align 4
   %5 = alloca i32, align 4
 
   store i32 0, i32* %1, align 4