[AArch64] Async unwind - function epilogues

[chill]

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

1922

Note to self: remove.

[danielkiss]

LGTM

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

2016–2018

looks it can be removed.

[MaskRay]

LGTM.

[chill]

Committing this change caused failures in the asan/hwasan regression tests. The reason was that emitting unwind info
in epilogues caused the unwind information in case like below:

foo:
  ...
  CSR saves
  ...
  CSR restores
  ret
L:
  ...
  b <somewhere>

to no longer be accidentally correct for the region starting at L. That could be a miscompilation
if a throwing call somehow ends up in that region.

Thus, I've changed the order of the patches to bring CFIFixup before the this patch.

[chill]

Updated on top of latest main, will commit it the following days, based on previous
acceptance, as it contains no non-trivial changes.

[MaskRay]

Note: arm64-shrink-wrapping.ll and merge-store-dependency.ll need rebase.

llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp

130–139

if (!NeedsAsyncDwarfUnwindInfo) {
  const Function &F = MF.getFunction();
  // TODO hasMinSize() is to ...
  NeedsAsyncDwarfUnwindInfo = 
}
return *NeedsAsyncDwarfUnwindInfo;

operator* is more common than getValue. The comment is too long to use the in-line style.

llvm/test/CodeGen/AArch64/framelayout-sve.mir

276

Specify the operands of CFI_INSTRUCTION

356

Specify the operands of CFI_INSTRUCTION

517

Specify the operands of CFI_INSTRUCTION

[chill]

llvm/test/CodeGen/AArch64/framelayout-sve.mir

276

Looks redundant to me. The operands are evident from the ASM and UNWINDINFO checks.

[MaskRay]

llvm/test/CodeGen/AArch64/framelayout-sve.mir

276

SVE has quite complex code paths in the implementation, so it's good to test at least the CFI instruction name to have some coverage. Other operands may be a bit redundant.

llvm/lib/Target/AArch64/AArch64FrameLowering.h

int64_t assignSVEStackObjectOffsets(MachineFrameInfo &MF,
                                       int &MinCSFrameIndex,
                                       int &MaxCSFrameIndex) const;
   bool shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock &MBB,
                                                 unsigned StackBumpBytes) const;
   void emitCalleeSavedGPRLocations(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MBBI) const;
   void emitCalleeSavedSVELocations(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MBBI) const;
+  void emitCalleeSavedGPRRestores(MachineBasicBlock &MBB,
+                                  MachineBasicBlock::iterator MBBI) const;
+  void emitCalleeSavedSVERestores(MachineBasicBlock &MBB,
+                                  MachineBasicBlock::iterator MBBI) const;
 };
 
 } // End llvm namespace
 
 #endif

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

for (const auto &Info : CSI) {
     unsigned Reg = Info.getReg();
     if (!TRI.regNeedsCFI(Reg, Reg))
       continue;
     insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
                        TRI.getDwarfRegNum(Reg, true));
   }
 }
 
+static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator MBBI,
+                                    bool SVE) {
+  MachineFunction &MF = *MBB.getParent();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+
+  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
+  if (CSI.empty())
+    return;
+
+  const TargetSubtargetInfo &STI = MF.getSubtarget();
+  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
+  const TargetInstrInfo &TII = *STI.getInstrInfo();
+  DebugLoc DL = MBB.findDebugLoc(MBBI);
+
+  for (const auto &Info : CSI) {
+    if (SVE !=
+        (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
+      continue;
+
+    unsigned Reg = Info.getReg();
+    if (SVE &&
+        !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
+      continue;
+
+    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
+        nullptr, TRI.getDwarfRegNum(Info.getReg(), true)));
+    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameDestroy);
+  }
+}
+
+void AArch64FrameLowering::emitCalleeSavedGPRRestores(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
+  emitCalleeSavedRestores(MBB, MBBI, false);
+}
+
+void AArch64FrameLowering::emitCalleeSavedSVERestores(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
+  emitCalleeSavedRestores(MBB, MBBI, true);
+}
+
 // Find a scratch register that we can use at the start of the prologue to
 // re-align the stack pointer.  We avoid using callee-save registers since they
 // may appear to be free when this is called from canUseAsPrologue (during
 // shrink wrapping), but then no longer be free when this is called from
 // emitPrologue.
 //
 // FIXME: This is a bit conservative, since in the above case we could use one
 // of the callee-save registers as a scratch temp to re-align the stack pointer,

 }
 
 // Convert callee-save register save/restore instruction to do stack pointer
 // decrement/increment to allocate/deallocate the callee-save stack area by
 // converting store/load to use pre/post increment version.
 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
-    bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, bool InProlog = true) {
+    bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI,
+    MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup,
+    int CFAOffset = 0) {
   unsigned NewOpc;
   switch (MBBI->getOpcode()) {
   default:
     llvm_unreachable("Unexpected callee-save save/restore opcode!");
   case AArch64::STPXi:
     NewOpc = AArch64::STPXpre;
     break;
   case AArch64::STPDi:
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
   assert(Success && "unknown load/store opcode");
 
   // If the first store isn't right where we want SP then we can't fold the
   // update in so create a normal arithmetic instruction instead.
   MachineFunction &MF = *MBB.getParent();
   if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 ||
       CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
     emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
-                    StackOffset::getFixed(CSStackSizeInc), TII,
-                    InProlog ? MachineInstr::FrameSetup
-                             : MachineInstr::FrameDestroy,
-                    false, false, nullptr, EmitCFI && InProlog);
+                    StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag,
+                    false, false, nullptr, EmitCFI,
+                    StackOffset::getFixed(CFAOffset));
 
     return std::prev(MBBI);
   }
 
   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
   MIB.addReg(AArch64::SP, RegState::Define);
 
   // Copy all operands other than the immediate offset.
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
   MIB.addImm(CSStackSizeInc / (int)Scale);
 
   MIB.setMIFlags(MBBI->getFlags());
   MIB.setMemRefs(MBBI->memoperands());
 
   // Generate a new SEH code that corresponds to the new instruction.
   if (NeedsWinCFI) {
     *HasWinCFI = true;
-    InsertSEH(*MIB, *TII,
-              InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy);
+    InsertSEH(*MIB, *TII, FrameFlag);
   }
 
-  if (EmitCFI && InProlog) {
+  if (EmitCFI) {
     unsigned CFIIndex = MF.addFrameInst(
-        MCCFIInstruction::cfiDefCfaOffset(nullptr, -CSStackSizeInc));
+        MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset - CSStackSizeInc));
     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
         .addCFIIndex(CFIIndex)
-        .setMIFlags(MachineInstr::FrameSetup);
+        .setMIFlags(FrameFlag);
   }
 
   return std::prev(MBB.erase(MBBI));
 }
 
 // Fixup callee-save register save/restore instructions to take into account
 // combined SP bump by adding the local stack size to the stack offsets.
 static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII,
                                         const DebugLoc &DL) {
   // Shadow call stack epilog: ldr x30, [x18, #-8]!
   BuildMI(MBB, MBBI, DL, TII.get(AArch64::LDRXpre))
       .addReg(AArch64::X18, RegState::Define)
       .addReg(AArch64::LR, RegState::Define)
       .addReg(AArch64::X18)
       .addImm(-8)
       .setMIFlag(MachineInstr::FrameDestroy);
+
+  if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo()) {
+    unsigned CFIIndex =
+        MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, 18));
+    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameDestroy);
+  }
 }
 
 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
                                         MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   const Function &F = MF.getFunction();
   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   DebugLoc DL;
 
   const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
   if (needsShadowCallStackPrologueEpilogue(MF))
     emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI,
                                 MFnI.needsDwarfUnwindInfo());
 
   if (MFnI.shouldSignReturnAddress()) {
-
     unsigned PACI;
     if (MFnI.shouldSignWithBKey()) {
       BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
           .setMIFlag(MachineInstr::FrameSetup);
       PACI = Subtarget.hasPAuth() ? AArch64::PACIB : AArch64::PACIBSP;
     } else {
       PACI = Subtarget.hasPAuth() ? AArch64::PACIA : AArch64::PACIASP;
     }

 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
                                         MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
   DebugLoc DL;
   bool NeedsWinCFI = needsWinCFI(MF);
+  bool EmitCFI = MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo();
   bool HasWinCFI = false;
   bool IsFunclet = false;
   auto WinCFI = make_scope_exit([&]() { assert(HasWinCFI == MF.hasWinCFI()); });
 
   if (MBB.end() != MBBI) {
     DL = MBBI->getDebugLoc();
     IsFunclet = isFuncletReturnInstr(*MBBI);
   }
 
-  auto ShadowStackEpilogue = make_scope_exit([&]() {
+  auto FinishingTouches = make_scope_exit([&]() {
+    InsertReturnAddressAuth(MF, MBB);
     if (needsShadowCallStackPrologueEpilogue(MF))
       emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL);
+    if (EmitCFI)
+      emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
   });
 
   int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
                                : MFI.getStackSize();
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
 
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.
   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     return;
 
   // How much of the stack used by incoming arguments this function is expected
   // to restore in this particular epilogue.
   int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
-
-  // The stack frame should be like below,
-  //
-  //      ----------------------                     ---
-  //      |                    |                      |
-  //      | BytesInStackArgArea|              CalleeArgStackSize
-  //      | (NumReusableBytes) |                (of tail call)
-  //      |                    |                     ---
-  //      |                    |                      |
-  //      ---------------------|        ---           |
-  //      |                    |         |            |
-  //      |   CalleeSavedReg   |         |            |
-  //      | (CalleeSavedStackSize)|      |            |
-  //      |                    |         |            |
-  //      ---------------------|         |         NumBytes
-  //      |                    |     StackSize  (StackAdjustUp)
-  //      |   LocalStackSize   |         |            |
-  //      | (covering callee   |         |            |
-  //      |       args)        |         |            |
-  //      |                    |         |            |
-  //      ----------------------        ---          ---
-  //
-  // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
-  //             = StackSize + ArgumentPopSize
-  //
-  // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
-  // it as the 2nd argument of AArch64ISD::TC_RETURN.
-
-  auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
-
   bool IsWin64 =
       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
   unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
 
   int64_t AfterCSRPopSize = ArgumentStackToRestore;
   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
   // We cannot rely on the local stack size set in emitPrologue if the function
   // has funclets, as funclets have different local stack size requirements, and
if (homogeneousPrologEpilog(MF, &MBB)) {
     // SP has been already adjusted while restoring callee save regs.
     // We've bailed-out the case with adjusting SP for arguments.
     assert(AfterCSRPopSize == 0);
     return;
   }
   bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
   // Assume we can't combine the last pop with the sp restore.
 
+  bool CombineAfterCSRBump = false;
   if (!CombineSPBump && PrologueSaveSize != 0) {
     MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
-    while (AArch64InstrInfo::isSEHInstruction(*Pop))
+    while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
+           AArch64InstrInfo::isSEHInstruction(*Pop))
       Pop = std::prev(Pop);
     // Converting the last ldp to a post-index ldp is valid only if the last
     // ldp's offset is 0.
     const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
     // If the offset is 0 and the AfterCSR pop is not actually trying to
     // allocate more stack for arguments (in space that an untimely interrupt
     // may clobber), convert it to a post-index ldp.
-    if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0)
-      convertCalleeSaveRestoreToSPPrePostIncDec(MBB, Pop, DL, TII,
-                                                PrologueSaveSize, NeedsWinCFI,
-                                                &HasWinCFI, false, false);
-    else {
+    if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
+      convertCalleeSaveRestoreToSPPrePostIncDec(
+          MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
+          MachineInstr::FrameDestroy, PrologueSaveSize);
+    } else {
       // If not, make sure to emit an add after the last ldp.
       // We're doing this by transfering the size to be restored from the
       // adjustment *before* the CSR pops to the adjustment *after* the CSR
       // pops.
       AfterCSRPopSize += PrologueSaveSize;
+      CombineAfterCSRBump = true;
     }
   }
 
   // Move past the restores of the callee-saved registers.
   // If we plan on combining the sp bump of the local stack size and the callee
   // save stack size, we might need to adjust the CSR save and restore offsets.
   MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
   MachineBasicBlock::iterator Begin = MBB.begin();
if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
     }
   }
 
   const StackOffset &SVEStackSize = getSVEStackSize(MF);
 
   // If there is a single SP update, insert it before the ret and we're done.
   if (CombineSPBump) {
     assert(!SVEStackSize && "Cannot combine SP bump with SVE");
+
+    // When we are about to restore the CSRs, the CFA register is SP again.
+    if (EmitCFI && hasFP(MF)) {
+      const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
+      unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
+      unsigned CFIIndex =
+          MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, NumBytes));
+      BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+          .addCFIIndex(CFIIndex)
+          .setMIFlags(MachineInstr::FrameDestroy);
+    }
+
     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
                     TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
-                    &HasWinCFI);
+                    &HasWinCFI, EmitCFI, StackOffset::getFixed(NumBytes));
     if (HasWinCFI)
       BuildMI(MBB, MBB.getFirstTerminator(), DL,
               TII->get(AArch64::SEH_EpilogEnd))
           .setMIFlag(MachineInstr::FrameDestroy);
     return;
   }
 
   NumBytes -= PrologueSaveSize;
   assert(NumBytes >= 0 && "Negative stack allocation size!?");
 
   // Process the SVE callee-saves to determine what space needs to be
   // deallocated.
   StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
StackOffset CalleeSavedSizeAsOffset =
         StackOffset::getScalable(CalleeSavedSize);
     DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
     DeallocateAfter = CalleeSavedSizeAsOffset;
   }
 
   // Deallocate the SVE area.
   if (SVEStackSize) {
     if (AFI->isStackRealigned()) {
-      if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize())
+      if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
         // Set SP to start of SVE callee-save area from which they can
         // be reloaded. The code below will deallocate the stack space
         // space by moving FP -> SP.
         emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
                         StackOffset::getScalable(-CalleeSavedSize), TII,
                         MachineInstr::FrameDestroy);
+      }
     } else {
       if (AFI->getSVECalleeSavedStackSize()) {
         // Deallocate the non-SVE locals first before we can deallocate (and
         // restore callee saves) from the SVE area.
-        emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
-                        StackOffset::getFixed(NumBytes), TII,
-                        MachineInstr::FrameDestroy);
+        emitFrameOffset(
+            MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
+            StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy,
+            false, false, nullptr, EmitCFI && !hasFP(MF),
+            SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
         NumBytes = 0;
       }
 
       emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
-                      DeallocateBefore, TII, MachineInstr::FrameDestroy);
+                      DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
+                      false, nullptr, EmitCFI && !hasFP(MF),
+                      SVEStackSize +
+                          StackOffset::getFixed(NumBytes + PrologueSaveSize));
 
       emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
-                      DeallocateAfter, TII, MachineInstr::FrameDestroy);
+                      DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
+                      false, nullptr, EmitCFI && !hasFP(MF),
+                      DeallocateAfter +
+                          StackOffset::getFixed(NumBytes + PrologueSaveSize));
     }
+    if (EmitCFI)
+      emitCalleeSavedSVERestores(MBB, RestoreEnd);
   }
 
   if (!hasFP(MF)) {
     bool RedZone = canUseRedZone(MF);
     // If this was a redzone leaf function, we don't need to restore the
     // stack pointer (but we may need to pop stack args for fastcc).
     if (RedZone && AfterCSRPopSize == 0)
       return;
 
+    // Pop the local variables off the stack. If there are no callee-saved
+    // registers, it means we are actually positioned at the terminator and can
+    // combine stack increment for the locals and the stack increment for
+    // callee-popped arguments into (possibly) a single instruction and be done.
     bool NoCalleeSaveRestore = PrologueSaveSize == 0;
     int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
     if (NoCalleeSaveRestore)
       StackRestoreBytes += AfterCSRPopSize;
 
-    emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
-                    StackOffset::getFixed(StackRestoreBytes), TII,
-                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
+    emitFrameOffset(
+        MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
+        MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI,
+        StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize));
+
     // If we were able to combine the local stack pop with the argument pop,
     // then we're done.
     if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
       if (HasWinCFI) {
         BuildMI(MBB, MBB.getFirstTerminator(), DL,
                 TII->get(AArch64::SEH_EpilogEnd))
             .setMIFlag(MachineInstr::FrameDestroy);
       }
       return;
     }
 
     NumBytes = 0;
   }
 
   // Restore the original stack pointer.
   // FIXME: Rather than doing the math here, we should instead just use
   // non-post-indexed loads for the restores if we aren't actually going to
   // be able to save any instructions.
   if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
     emitFrameOffset(
         MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
         StackOffset::getFixed(-AFI->getCalleeSaveBaseToFrameRecordOffset()),
         TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
   } else if (NumBytes)
     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(NumBytes), TII,
                     MachineInstr::FrameDestroy, false, NeedsWinCFI);
 
+  // When we are about to restore the CSRs, the CFA register is SP again.
+  if (EmitCFI && hasFP(MF)) {
+    const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
+    unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
+    unsigned CFIIndex = MF.addFrameInst(
+        MCCFIInstruction::cfiDefCfa(nullptr, Reg, PrologueSaveSize));
+    BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlags(MachineInstr::FrameDestroy);
+  }
+
   // This must be placed after the callee-save restore code because that code
   // assumes the SP is at the same location as it was after the callee-save save
   // code in the prologue.
   if (AfterCSRPopSize) {
     assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
                                   "interrupt may have clobbered");
 
-    emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
-                    StackOffset::getFixed(AfterCSRPopSize), TII,
-                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
+    emitFrameOffset(
+        MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
+        StackOffset::getFixed(AfterCSRPopSize), TII, MachineInstr::FrameDestroy,
+        false, NeedsWinCFI, &HasWinCFI, EmitCFI,
+        StackOffset::getFixed(CombineAfterCSRBump ? PrologueSaveSize : 0));
   }
   if (HasWinCFI)
     BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
         .setMIFlag(MachineInstr::FrameDestroy);
 }
 
 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
 /// debug info.  It's the same as what we use for resolving the code-gen
MIB.addReg(Reg1, getDefRegState(true))
         .addImm(RPI.Offset) // [sp, #offset*scale]
                             // where factor*scale is implicit
         .setMIFlag(MachineInstr::FrameDestroy);
     MIB.addMemOperand(MF.getMachineMemOperand(
         MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
         MachineMemOperand::MOLoad, Size, Alignment));
     if (NeedsWinCFI)
       InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
+
     return MIB->getIterator();
   };
 
   // SVE objects are always restored in reverse order.
   for (const RegPairInfo &RPI : reverse(RegPairs))
     if (RPI.isScalable())
       EmitMI(RPI);
 

llvm/lib/Target/AArch64/AArch64InstrInfo.h

 
 /// Return true if there is an instruction /after/ \p DefMI and before \p UseMI
 /// which either reads or clobbers NZCV.
 bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
                                      const MachineInstr &UseMI,
                                      const TargetRegisterInfo *TRI);
 
 MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg,
-                              unsigned Reg, const StackOffset &Offset);
+                              unsigned Reg, const StackOffset &Offset,
+                              bool LastAdjustmentWasScalable = true);
 MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,
                                  const StackOffset &OffsetFromDefCFA);
 
 /// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg
 /// plus Offset.  This is intended to be used from within the prolog/epilog
 /// insertion (PEI) pass, where a virtual scratch register may be allocated
 /// if necessary, to be replaced by the scavenger at the end of PEI.
 void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
   DefCfaExpr.append(buffer, buffer + encodeULEB128(Expr.size(), buffer));
   DefCfaExpr.append(Expr.str());
   return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(),
                                         Comment.str());
 }
 
 MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
                                     unsigned FrameReg, unsigned Reg,
-                                    const StackOffset &Offset) {
+                                    const StackOffset &Offset,
+                                    bool LastAdjustmentWasScalable) {
   if (Offset.getScalable())
     return createDefCFAExpression(TRI, Reg, Offset);
 
-  if (FrameReg == Reg)
+  if (FrameReg == Reg && !LastAdjustmentWasScalable)
     return MCCFIInstruction::cfiDefCfaOffset(nullptr, int(Offset.getFixed()));
 
   unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
   return MCCFIInstruction::cfiDefCfa(nullptr, DwarfReg, (int)Offset.getFixed());
 }
 
 MCCFIInstruction llvm::createCFAOffset(const TargetRegisterInfo &TRI,
                                        unsigned Reg,
if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
       CFAOffset += Change;
     else
       CFAOffset -= Change;
     if (EmitCFAOffset && DestReg == TmpReg) {
       MachineFunction &MF = *MBB.getParent();
       const TargetSubtargetInfo &STI = MF.getSubtarget();
       const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
 
-      unsigned CFIIndex =
-          MF.addFrameInst(createDefCFA(TRI, FrameReg, DestReg, CFAOffset));
+      unsigned CFIIndex = MF.addFrameInst(
+          createDefCFA(TRI, FrameReg, DestReg, CFAOffset, VScale != 1));
       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
           .addCFIIndex(CFIIndex)
           .setMIFlags(Flag);
     }
 
     if (NeedsWinCFI) {
       assert(Sign == 1 && "SEH directives should always have a positive sign");
       int Imm = (int)(ThisVal << LocalShiftSize);

llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h

class AArch64FunctionInfo final : public MachineFunctionInfo {
 
   /// The stack slot where the Swift asynchronous context is stored.
   int SwiftAsyncContextFrameIdx = std::numeric_limits<int>::max();
 
   /// True if the function need unwind information.
   mutable Optional<bool> NeedsDwarfUnwindInfo;
 
   /// True if the function need asynchronous unwind information.
-  mutable Optional<bool> NeedsDwarfAsyncUnwindInfo;
+  mutable Optional<bool> NeedsAsyncDwarfUnwindInfo;
 
 public:
   explicit AArch64FunctionInfo(MachineFunction &MF);
 
   void initializeBaseYamlFields(const yaml::AArch64FunctionInfo &YamlMFI);
 
   unsigned getBytesInStackArgArea() const { return BytesInStackArgArea; }
   void setBytesInStackArgArea(unsigned bytes) { BytesInStackArgArea = bytes; }

llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp

 /// \file
 /// This file implements AArch64-specific per-machine-function
 /// information.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "AArch64MachineFunctionInfo.h"
 #include "AArch64InstrInfo.h"
-#include "llvm/MC/MCAsmInfo.h"
+#include "AArch64Subtarget.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/MC/MCAsmInfo.h"
 
 using namespace llvm;
 
 yaml::AArch64FunctionInfo::AArch64FunctionInfo(
     const llvm::AArch64FunctionInfo &MFI)
     : HasRedZone(MFI.hasRedZone()) {}
 
 void yaml::AArch64FunctionInfo::mappingImpl(yaml::IO &YamlIO) {

 
 bool AArch64FunctionInfo::shouldSignReturnAddress() const {
   return shouldSignReturnAddress(llvm::any_of(
       MF.getFrameInfo().getCalleeSavedInfo(),
       [](const auto &Info) { return Info.getReg() == AArch64::LR; }));
 }
 
 bool AArch64FunctionInfo::needsDwarfUnwindInfo() const {
-  if (!NeedsDwarfUnwindInfo.hasValue())
+  if (!NeedsDwarfUnwindInfo)
     NeedsDwarfUnwindInfo = MF.needsFrameMoves() &&
                            !MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
 
-  return NeedsDwarfUnwindInfo.getValue();
+  return *NeedsDwarfUnwindInfo;
 }
 
 bool AArch64FunctionInfo::needsAsyncDwarfUnwindInfo() const {
-  if (!NeedsDwarfAsyncUnwindInfo.hasValue())
-    NeedsDwarfAsyncUnwindInfo =
-        needsDwarfUnwindInfo() &&
-        MF.getFunction().getUWTableKind() == UWTableKind::Async;
-  return NeedsDwarfAsyncUnwindInfo.getValue();
+  if (!NeedsAsyncDwarfUnwindInfo) {
+    const Function &F = MF.getFunction();
+    //  The check got "minsize" is because epilogue unwind info is not emitted
+    //  (yet) for homogeneous epilogues, outlined functions, and functions
+    //  outlined from.
+    NeedsAsyncDwarfUnwindInfo = needsDwarfUnwindInfo() &&
+                                F.getUWTableKind() == UWTableKind::Async &&
+                                !F.hasMinSize();
+  }
+  return *NeedsAsyncDwarfUnwindInfo;
 }

llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
 
 declare void @byval_i32(i32* byval(i32) %ptr)
 
-define void @call_byval_i32(i32* %incoming) {
+define void @call_byval_i32(i32* %incoming) uwtable {
 ; CHECK-LABEL: call_byval_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    str w8, [sp]
 ; CHECK-NEXT:    bl byval_i32
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   call void @byval_i32(i32* byval(i32) %incoming)
   ret void
 }
 
 declare void @byval_a64i32([64 x i32]* byval([64 x i32]) %ptr)
 
-define void @call_byval_a64i32([64 x i32]* %incoming) {
+define void @call_byval_a64i32([64 x i32]* %incoming) uwtable {
 ; CHECK-LABEL: call_byval_a64i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #288
 ; CHECK-NEXT:    .cfi_def_cfa_offset 288
 ; CHECK-NEXT:    stp x29, x30, [sp, #256] // 16-byte Folded Spill
 ; CHECK-NEXT:    str x28, [sp, #272] // 8-byte Folded Spill
 ; CHECK-NEXT:    add x29, sp, #256
 ; CHECK-NEXT:    .cfi_def_cfa w29, 32

 ; CHECK-NEXT:    str q0, [sp, #192]
 ; CHECK-NEXT:    ldr q0, [x0, #208]
 ; CHECK-NEXT:    str q0, [sp, #208]
 ; CHECK-NEXT:    ldr q0, [x0, #224]
 ; CHECK-NEXT:    str q0, [sp, #224]
 ; CHECK-NEXT:    ldr q0, [x0, #240]
 ; CHECK-NEXT:    str q0, [sp, #240]
 ; CHECK-NEXT:    bl byval_a64i32
+; CHECK-NEXT:    .cfi_def_cfa wsp, 288
 ; CHECK-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x28, [sp, #272] // 8-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #288
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   call void @byval_a64i32([64 x i32]* byval([64 x i32]) %incoming)
   ret void
 }

llvm/test/CodeGen/AArch64/arm64-fp128.ll

 
 
 define dso_local i32 @test_br_cc() uwtable {
 ; CHECK-LABEL: test_br_cc:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    adrp x8, lhs
 ; CHECK-NEXT:    ldr q0, [x8, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, rhs
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:rhs]
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    b.ge .LBB11_2
 ; CHECK-NEXT:  // %bb.1: // %iftrue
 ; CHECK-NEXT:    mov w0, #42
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB11_2: // %iffalse
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:    mov w0, #29
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
   ; olt == !uge, which LLVM optimizes this to.
   %cond = fcmp olt fp128 %lhs, %rhs
   br i1 %cond, label %iftrue, label %iffalse

llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll

 ; ENABLE-NEXT:  LBB4_2: ; %for.body
 ; ENABLE-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    bl _something
 ; ENABLE-NEXT:    subs w20, w20, #1
 ; ENABLE-NEXT:    add w19, w0, w19
 ; ENABLE-NEXT:    b.ne LBB4_2
 ; ENABLE-NEXT:  ; %bb.3: ; %for.end
 ; ENABLE-NEXT:    lsl w0, w19, #3
+; ENABLE-NEXT:    .cfi_def_cfa wsp, 32
 ; ENABLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; ENABLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; ENABLE-NEXT:    .cfi_def_cfa_offset 0
+; ENABLE-NEXT:    .cfi_restore w30
+; ENABLE-NEXT:    .cfi_restore w29
+; ENABLE-NEXT:    .cfi_restore w19
+; ENABLE-NEXT:    .cfi_restore w20
 ; ENABLE-NEXT:    ret
 ; ENABLE-NEXT:  LBB4_4: ; %if.else
-; ENABLE-NEXT:    .cfi_def_cfa wsp, 0
-; ENABLE-NEXT:    .cfi_same_value w30
-; ENABLE-NEXT:    .cfi_same_value w29
-; ENABLE-NEXT:    .cfi_same_value w19
-; ENABLE-NEXT:    .cfi_same_value w20
 ; ENABLE-NEXT:    lsl w0, w1, #1
 ; ENABLE-NEXT:    ret
 ;
 ; DISABLE-LABEL: loopInfoRestoreOutsideLoop:
 ; DISABLE:       ; %bb.0: ; %entry
 ; DISABLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; DISABLE-NEXT:    .cfi_def_cfa_offset 32
 ; DISABLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill

 ; DISABLE-NEXT:    add w19, w0, w19
 ; DISABLE-NEXT:    b.ne LBB4_2
 ; DISABLE-NEXT:  ; %bb.3: ; %for.end
 ; DISABLE-NEXT:    lsl w0, w19, #3
 ; DISABLE-NEXT:    b LBB4_5
 ; DISABLE-NEXT:  LBB4_4: ; %if.else
 ; DISABLE-NEXT:    lsl w0, w1, #1
 ; DISABLE-NEXT:  LBB4_5: ; %if.end
+; DISABLE-NEXT:    .cfi_def_cfa wsp, 32
 ; DISABLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; DISABLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; DISABLE-NEXT:    .cfi_def_cfa_offset 0
+; DISABLE-NEXT:    .cfi_restore w30
+; DISABLE-NEXT:    .cfi_restore w29
+; DISABLE-NEXT:    .cfi_restore w19
+; DISABLE-NEXT:    .cfi_restore w20
 ; DISABLE-NEXT:    ret
 entry:
   %tobool = icmp eq i32 %cond, 0
   br i1 %tobool, label %if.else, label %if.then
 
 if.then:                                          ; preds = %entry
   tail call void bitcast (void (...)* @somethingElse to void ()*)()
   br label %for.body

 ; ENABLE-NEXT:    add x9, x8, #8
 ; ENABLE-NEXT:    str x9, [sp, #8]
 ; ENABLE-NEXT:    ldr w8, [x8]
 ; ENABLE-NEXT:    add w0, w0, w8
 ; ENABLE-NEXT:    subs w1, w1, #1
 ; ENABLE-NEXT:    b.ne LBB6_2
 ; ENABLE-NEXT:  LBB6_3: ; %for.end
 ; ENABLE-NEXT:    add sp, sp, #16
+; ENABLE-NEXT:    .cfi_def_cfa_offset 0
 ; ENABLE-NEXT:    ret
 ; ENABLE-NEXT:  LBB6_4: ; %if.else
-; ENABLE-NEXT:    .cfi_def_cfa wsp, 0
 ; ENABLE-NEXT:    lsl w0, w1, #1
 ; ENABLE-NEXT:    ret
 ;
 ; DISABLE-LABEL: variadicFunc:
 ; DISABLE:       ; %bb.0: ; %entry
 ; DISABLE-NEXT:    sub sp, sp, #16
 ; DISABLE-NEXT:    .cfi_def_cfa_offset 16
-; DISABLE-NEXT:    cbz w0, LBB6_4
+; DISABLE-NEXT:    cbz w0, LBB6_3
 ; DISABLE-NEXT:  ; %bb.1: ; %if.then
 ; DISABLE-NEXT:    add x8, sp, #16
 ; DISABLE-NEXT:    cmp w1, #1
 ; DISABLE-NEXT:    str x8, [sp, #8]
 ; DISABLE-NEXT:    mov w0, wzr
-; DISABLE-NEXT:    b.lt LBB6_3
+; DISABLE-NEXT:    b.lt LBB6_4
 ; DISABLE-NEXT:  LBB6_2: ; %for.body
 ; DISABLE-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    ldr x8, [sp, #8]
 ; DISABLE-NEXT:    add x9, x8, #8
 ; DISABLE-NEXT:    str x9, [sp, #8]
 ; DISABLE-NEXT:    ldr w8, [x8]
 ; DISABLE-NEXT:    add w0, w0, w8
 ; DISABLE-NEXT:    subs w1, w1, #1
 ; DISABLE-NEXT:    b.ne LBB6_2
-; DISABLE-NEXT:  LBB6_3: ; %if.end
-; DISABLE-NEXT:    add sp, sp, #16
-; DISABLE-NEXT:    ret
-; DISABLE-NEXT:  LBB6_4: ; %if.else
+; DISABLE-NEXT:    b LBB6_4
+; DISABLE-NEXT:  LBB6_3: ; %if.else
 ; DISABLE-NEXT:    lsl w0, w1, #1
+; DISABLE-NEXT:  LBB6_4: ; %if.end
 ; DISABLE-NEXT:    add sp, sp, #16
+; DISABLE-NEXT:    .cfi_def_cfa_offset 0
 ; DISABLE-NEXT:    ret
 entry:
   %ap = alloca i8*, align 8
   %tobool = icmp eq i32 %cond, 0
   br i1 %tobool, label %if.else, label %if.then
 
 if.then:                                          ; preds = %entry
   %ap1 = bitcast i8** %ap to i8*

llvm/test/CodeGen/AArch64/cfi-fixup.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux %s -o - | FileCheck %s
+
+define i32 @f0(i32 %x) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
+; CHECK-NEXT:    cbz w0, .LBB0_4
+; CHECK-NEXT:  // %bb.1: // %entry
+; CHECK-NEXT:    cmp w0, #2
+; CHECK-NEXT:    b.eq .LBB0_5
+; CHECK-NEXT:  // %bb.2: // %entry
+; CHECK-NEXT:    cmp w0, #1
+; CHECK-NEXT:    b.ne .LBB0_6
+; CHECK-NEXT:  // %bb.3: // %if.then2
+; CHECK-NEXT:    bl g1
+; CHECK-NEXT:    add w0, w0, #1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    .cfi_remember_state
+; CHECK-NEXT:    mov w0, #1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_5: // %if.then5
+; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    .cfi_remember_state
+; CHECK-NEXT:    bl g0
+; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    sub w0, w8, w0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_6: // %if.end7
+; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    ret
+entry:
+  switch i32 %x, label %if.end7 [
+    i32 0, label %return
+    i32 1, label %if.then2
+    i32 2, label %if.then5
+  ]
+
+if.then2:
+  %call = tail call i32 @g1(i32 1)
+  %add = add nsw i32 %call, 1
+  br label %return
+
+if.then5:
+  %call6 = tail call i32 @g0(i32 2)
+  %sub = sub nsw i32 1, %call6
+  br label %return
+
+if.end7:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ %add, %if.then2 ], [ %sub, %if.then5 ], [ 0, %if.end7 ], [ 1, %entry ]
+  ret i32 %retval.0
+}
+
+declare i32 @g1(i32)
+
+declare i32 @g0(i32)
+
+define i32 @f1(i32 %x) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cbz w0, .LBB1_2
+; CHECK-NEXT:  // %bb.1: // %if.end
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl g0
+; CHECK-NEXT:    add w0, w0, #1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:  .LBB1_2: // %return
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:
+  %call = tail call i32 @g0(i32 %x)
+  %add = add nsw i32 %call, 1
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ %add, %if.end ], [ 0, %entry ]
+  ret i32 %retval.0
+}
+
+define i32 @f2(i32 %x) #0 {
+; CHECK-LABEL: f2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
+; CHECK-NEXT:    cbz w0, .LBB2_2
+; CHECK-NEXT:  // %bb.1: // %if.end
+; CHECK-NEXT:    bl g1
+; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    sub w0, w8, w0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_2: // %if.then
+; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    bl g0
+; CHECK-NEXT:    add w0, w0, #1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    ret
+entry:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %call = tail call i32 @g0(i32 0)
+  %add = add nsw i32 %call, 1
+  br label %return
+
+if.end:
+  %call1 = tail call i32 @g1(i32 %x)
+  %sub = sub nsw i32 1, %call1
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ %add, %if.then ], [ %sub, %if.end ]
+  ret i32 %retval.0
+}
+
+attributes #0 = { uwtable }

llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll

 ; CHECK-NEXT:    bl Update
 ; CHECK-NEXT:    sub x21, x21, #2
 ; CHECK-NEXT:    cmp x20, x21
 ; CHECK-NEXT:    b.lt .LBB6_1
 ; CHECK-NEXT:  .LBB6_3: // %while.end
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w22
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %size = getelementptr inbounds %struct.Struct, %struct.Struct* %hdCall, i64 0, i32 0
   %0 = load i64, i64* %size, align 8
   br label %land.rhs
 
 land.rhs:
   %rp.06 = phi i64 [ %0, %entry ], [ %sub, %while.body ]

 ; CHECK-NEXT:  // %bb.6:
 ; CHECK-NEXT:    mov w0, #123
 ; CHECK-NEXT:    b .LBB7_8
 ; CHECK-NEXT:  .LBB7_7: // %if.end
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:  .LBB7_8: // %return
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, i32* @a, align 4
   %cmp4 = icmp slt i32 %0, -1
   br i1 %cmp4, label %while.body.preheader, label %while.end
 
 while.body.preheader:                             ; preds = %entry
   br label %while.body

 
 define i32 @do_nothing_if_compares_can_not_be_adjusted_to_each_other() #0 {
 ; CHECK-LABEL: do_nothing_if_compares_can_not_be_adjusted_to_each_other:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    adrp x8, :got:a
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:a]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    b.gt .LBB8_3
 ; CHECK-NEXT:  // %bb.1: // %while.body.preheader
 ; CHECK-NEXT:    sub w19, w8, #1
 ; CHECK-NEXT:  .LBB8_2: // %while.body

 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
 ; CHECK-NEXT:    b.ne .LBB8_6
 ; CHECK-NEXT:  // %bb.5:
 ; CHECK-NEXT:    mov w0, #123
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB8_6: // %if.end
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, i32* @a, align 4
   %cmp4 = icmp slt i32 %0, 1
   br i1 %cmp4, label %while.body.preheader, label %while.end
 
 while.body.preheader:                             ; preds = %entry
   br label %while.body

 
 ; Test in the following case, we don't hit 'cmp' and trigger a false positive
 ; cmp  w19, #0
 ; cinc w0, w19, gt
 ; ...
 ; fcmp d8, #0.0
 ; b.gt .LBB0_5
 
-define i32 @fcmpri(i32 %argc, i8** nocapture readonly %argv) {
+define i32 @fcmpri(i32 %argc, i8** nocapture readonly %argv) #0 {
 ; CHECK-LABEL: fcmpri:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    .cfi_offset b8, -32

 ; CHECK-NEXT:    fmov d1, #-2.00000000
 ; CHECK-NEXT:    fadd d0, d8, d0
 ; CHECK-NEXT:    fcsel d0, d8, d0, gt
 ; CHECK-NEXT:    bl woo
 ; CHECK-NEXT:    mov w0, #4
 ; CHECK-NEXT:  .LBB9_4: // %return
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    ret
 
 ; CHECK-LABEL-DAG: .LBB9_3
 
 entry:
   %cmp = icmp sgt i32 %argc, 1
   br i1 %cmp, label %land.lhs.true, label %if.end
 
cond.end14:                                       ; preds = %if.end, %cond.false12
   %call16 = call i32 @woo(double %cond15, double -2.000000e+00)
   br label %return
 
 return:                                           ; preds = %land.lhs.true, %cond.end14
   %retval.0 = phi i32 [ 4, %cond.end14 ], [ 3, %land.lhs.true ]
   ret i32 %retval.0
 }
 
-define void @cmp_shifted(i32 %in, i32 %lhs, i32 %rhs) {
+define void @cmp_shifted(i32 %in, i32 %lhs, i32 %rhs) #0 {
 ; CHECK-LABEL: cmp_shifted:
 ; CHECK:       // %bb.0: // %common.ret
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #42
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csinc w8, w8, wzr, gt
 ; CHECK-NEXT:    cmp w0, #2, lsl #12 // =8192
 ; CHECK-NEXT:    mov w9, #128
 ; CHECK-NEXT:    csel w0, w9, w8, ge
 ; CHECK-NEXT:    bl zoo
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; [...]
 
   %tst_low = icmp sgt i32 %in, 8191
   br i1 %tst_low, label %true, label %false
 
 true:
   call i32 @zoo(i32 128)

 
 declare i32 @zoo(i32)
 
 declare double @yoo(i32)
 
 declare i32 @xoo(i32, i32)
 
 declare i32 @woo(double, double)
+
+attributes #0 = { uwtable }

llvm/test/CodeGen/AArch64/cond-br-tuning.ll

 L2:
   ret void
 }
 
 declare void @foo()
 declare void @bar(i32)
 
 ; Don't transform since the call will clobber the NZCV bits.
-define void @test_call_clobber(i32 %unused, i32 %a) {
+define void @test_call_clobber(i32 %unused, i32 %a) uwtable {
 ; CHECK-LABEL: test_call_clobber:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    and w19, w1, #0x6
 ; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    cbnz w19, .LBB9_2
 ; CHECK-NEXT:  // %bb.1: // %if.end
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB9_2: // %if.then
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:    bl foo
 entry:
   %c = and i32 %a, 6
   call void @bar(i32 %c)
   %tobool = icmp eq i32 %c, 0
   br i1 %tobool, label %if.end, label %if.then
 
 if.then:
   tail call void @foo()
   unreachable
 
 if.end:
   ret void
 }

llvm/test/CodeGen/AArch64/csr-split.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=aarch64-unknown-linux < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=arm64-apple-ios < %s | FileCheck %s --check-prefix=CHECK-APPLE
 
 ; Check CSR split can work properly for tests below.
 
 @a = common dso_local local_unnamed_addr global i32 0, align 4
 
-define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
+define dso_local signext i32 @test1(i32* %b) local_unnamed_addr uwtable  {
 ; CHECK-LABEL: test1:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    adrp x8, a
 ; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
 ; CHECK-NEXT:    cmp x8, x0
 ; CHECK-NEXT:    b.eq .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %if.end
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %if.then
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl callVoid
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    b callNonVoid
 ;
 ; CHECK-APPLE-LABEL: test1:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
 ; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
 ; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
 ; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
+; CHECK-APPLE-NEXT:    .cfi_remember_state
 ; CHECK-APPLE-NEXT:  Lloh0:
 ; CHECK-APPLE-NEXT:    adrp x8, _a@PAGE
 ; CHECK-APPLE-NEXT:  Lloh1:
 ; CHECK-APPLE-NEXT:    ldrsw x8, [x8, _a@PAGEOFF]
 ; CHECK-APPLE-NEXT:    cmp x8, x0
 ; CHECK-APPLE-NEXT:    b.eq LBB0_2
 ; CHECK-APPLE-NEXT:  ; %bb.1: ; %if.end
 ; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-APPLE-NEXT:    .cfi_restore w30
+; CHECK-APPLE-NEXT:    .cfi_restore w29
+; CHECK-APPLE-NEXT:    .cfi_restore w19
+; CHECK-APPLE-NEXT:    .cfi_restore w20
 ; CHECK-APPLE-NEXT:    ret
 ; CHECK-APPLE-NEXT:  LBB0_2: ; %if.then
+; CHECK-APPLE-NEXT:    .cfi_restore_state
 ; CHECK-APPLE-NEXT:    mov x19, x0
 ; CHECK-APPLE-NEXT:    bl _callVoid
 ; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-APPLE-NEXT:    mov x0, x19
 ; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-APPLE-NEXT:    .cfi_restore w30
+; CHECK-APPLE-NEXT:    .cfi_restore w29
+; CHECK-APPLE-NEXT:    .cfi_restore w19
+; CHECK-APPLE-NEXT:    .cfi_restore w20
 ; CHECK-APPLE-NEXT:    b _callNonVoid
 ; CHECK-APPLE-NEXT:    .loh AdrpLdr Lloh0, Lloh1
 entry:
   %0 = load i32, i32* @a, align 4, !tbaa !2
   %conv = sext i32 %0 to i64
   %1 = inttoptr i64 %conv to i32*
   %cmp = icmp eq i32* %1, %b
   br i1 %cmp, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
   %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
   %call2 = tail call signext i32 @callNonVoid(i32* %b)
   br label %if.end
 
 if.end:                                           ; preds = %if.then, %entry
   %retval.0 = phi i32 [ %call2, %if.then ], [ undef, %entry ]
   ret i32 %retval.0
 }
 
 declare signext i32 @callVoid(...) local_unnamed_addr
 
 declare signext i32 @callNonVoid(i32*) local_unnamed_addr
 
-define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr  {
+define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr uwtable  {
 ; CHECK-LABEL: test2:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    cbz x0, .LBB1_3
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    adrp x8, a
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
 ; CHECK-NEXT:    cmp x8, x0
 ; CHECK-NEXT:    b.ne .LBB1_3
 ; CHECK-NEXT:  // %bb.2: // %if.then2
 ; CHECK-NEXT:    bl callVoid
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    b callNonVoid
 ; CHECK-NEXT:  .LBB1_3: // %return
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-APPLE-LABEL: test2:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
 ; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
 ; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
 ; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
+; CHECK-APPLE-NEXT:    .cfi_remember_state
 ; CHECK-APPLE-NEXT:    cbz x0, LBB1_3
 ; CHECK-APPLE-NEXT:  ; %bb.1: ; %entry
 ; CHECK-APPLE-NEXT:  Lloh2:
 ; CHECK-APPLE-NEXT:    adrp x8, _a@PAGE
 ; CHECK-APPLE-NEXT:    mov x19, x0
 ; CHECK-APPLE-NEXT:  Lloh3:
 ; CHECK-APPLE-NEXT:    ldrsw x8, [x8, _a@PAGEOFF]
 ; CHECK-APPLE-NEXT:    cmp x8, x0
 ; CHECK-APPLE-NEXT:    b.ne LBB1_3
 ; CHECK-APPLE-NEXT:  ; %bb.2: ; %if.then2
 ; CHECK-APPLE-NEXT:    bl _callVoid
 ; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-APPLE-NEXT:    mov x0, x19
 ; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-APPLE-NEXT:    .cfi_restore w30
+; CHECK-APPLE-NEXT:    .cfi_restore w29
+; CHECK-APPLE-NEXT:    .cfi_restore w19
+; CHECK-APPLE-NEXT:    .cfi_restore w20
 ; CHECK-APPLE-NEXT:    b _callNonVoid
 ; CHECK-APPLE-NEXT:  LBB1_3: ; %return
+; CHECK-APPLE-NEXT:    .cfi_restore_state
 ; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-APPLE-NEXT:    mov w0, wzr
 ; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-APPLE-NEXT:    .cfi_restore w30
+; CHECK-APPLE-NEXT:    .cfi_restore w29
+; CHECK-APPLE-NEXT:    .cfi_restore w19
+; CHECK-APPLE-NEXT:    .cfi_restore w20
 ; CHECK-APPLE-NEXT:    ret
 ; CHECK-APPLE-NEXT:    .loh AdrpLdr Lloh2, Lloh3
 entry:
   %tobool = icmp eq i32* %p1, null
   br i1 %tobool, label %return, label %if.end
 
 if.end:                                           ; preds = %entry
   %0 = load i32, i32* @a, align 4, !tbaa !2
   %conv = sext i32 %0 to i64
   %1 = inttoptr i64 %conv to i32*
   %cmp = icmp eq i32* %1, %p1
   br i1 %cmp, label %if.then2, label %return
 
 if.then2:                                         ; preds = %if.end
   %call = tail call signext i32 bitcast (i32 (...)* @callVoid to i32 ()*)()
   %call3 = tail call signext i32 @callNonVoid(i32* nonnull %p1)
   br label %return
 
 return:                                           ; preds = %if.end, %entry, %if.then2
   %retval.0 = phi i32 [ %call3, %if.then2 ], [ 0, %entry ], [ 0, %if.end ]
   ret i32 %retval.0
 }
 
 
-define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr  {
+define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_addr uwtable  {
 ; CHECK-LABEL: test3:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w30, -32
 ; CHECK-NEXT:    ldr x19, [x0]
 ; CHECK-NEXT:    cbz x19, .LBB2_2
 ; CHECK-NEXT:  // %bb.1: // %land.rhs
 ; CHECK-NEXT:    mov x20, x0
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    str x0, [x20]
 ; CHECK-NEXT:  .LBB2_2: // %land.end
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-APPLE-LABEL: test3:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
 ; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
 ; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
 ; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
 ; CHECK-APPLE-NEXT:    ldr x19, [x0]
 ; CHECK-APPLE-NEXT:    cbz x19, LBB2_2
 ; CHECK-APPLE-NEXT:  ; %bb.1: ; %land.rhs
 ; CHECK-APPLE-NEXT:    mov x20, x0
 ; CHECK-APPLE-NEXT:    mov x0, x19
 ; CHECK-APPLE-NEXT:    bl _bar
 ; CHECK-APPLE-NEXT:    str x0, [x20]
 ; CHECK-APPLE-NEXT:  LBB2_2: ; %land.end
 ; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-APPLE-NEXT:    mov x0, x19
 ; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-APPLE-NEXT:    .cfi_restore w30
+; CHECK-APPLE-NEXT:    .cfi_restore w29
+; CHECK-APPLE-NEXT:    .cfi_restore w19
+; CHECK-APPLE-NEXT:    .cfi_restore w20
 ; CHECK-APPLE-NEXT:    ret
 entry:
   %0 = load i8*, i8** %p1, align 8, !tbaa !6
   %tobool = icmp eq i8* %0, null
   br i1 %tobool, label %land.end, label %land.rhs
 
 land.rhs:                                         ; preds = %entry
   %call = tail call i8* @bar(i8* nonnull %0, i8 zeroext %p2)

llvm/test/CodeGen/AArch64/emutls.ll

 ; RUN: llc -emulated-tls -mtriple=aarch64-linux-android \
 ; RUN:     -relocation-model=pic -frame-pointer=all < %s | FileCheck -check-prefix=ARM64 %s
 ; RUN: llc -mtriple=aarch64-linux-android \
 ; RUN:     -relocation-model=pic -frame-pointer=all < %s | FileCheck -check-prefix=ARM64 %s
 
 ; Copied from X86/emutls.ll
 
 ; Use my_emutls_get_address like __emutls_get_address.
 @my_emutls_v_xyz = external global i8*, align 4
 declare i8* @my_emutls_get_address(i8*)
 
-define i32 @my_get_xyz() {
+define i32 @my_get_xyz() uwtable {
 ; ARM64-LABEL: my_get_xyz:
 ; ARM64:        adrp x0, :got:my_emutls_v_xyz
 ; ARM64-NEXT:   ldr x0, [x0, :got_lo12:my_emutls_v_xyz]
 ; ARM64-NEXT:   bl my_emutls_get_address
 ; ARM64-NEXT:   ldr  w0, [x0]
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
   %0 = bitcast i8* %call to i32*
   %1 = load i32, i32* %0, align 4
   ret i32 %1
 }
 
 @i1 = thread_local global i32 15
 @i2 = external thread_local global i32
 @i3 = internal thread_local global i32 15
 @i4 = hidden thread_local global i32 15
 @i5 = external hidden thread_local global i32
 @s1 = thread_local global i16 15
 @b1 = thread_local global i8 0
 
-define i32 @f1() {
+define i32 @f1() uwtable {
 ; ARM64-LABEL: f1:
 ; ARM64:        adrp x0, :got:__emutls_v.i1
 ; ARM64-NEXT:   ldr x0, [x0, :got_lo12:__emutls_v.i1]
 ; ARM64-NEXT:   bl __emutls_get_address
 ; ARM64-NEXT:   ldr  w0, [x0]
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   %tmp1 = load i32, i32* @i1
   ret i32 %tmp1
 }
 
-define i32* @f2() {
+define i32* @f2() uwtable {
 ; ARM64-LABEL: f2:
 ; ARM64:        adrp x0, :got:__emutls_v.i1
 ; ARM64-NEXT:   ldr x0, [x0, :got_lo12:__emutls_v.i1]
 ; ARM64-NEXT:   bl __emutls_get_address
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   ret i32* @i1
 }
 
 define i32 @f5() nounwind {
 ; ARM64-LABEL: f5:
 ; ARM64:        adrp x0, __emutls_v.i3
 ; ARM64:        add x0, x0, :lo12:__emutls_v.i3
 ; ARM64:        bl __emutls_get_address
 ; ARM64-NEXT:   ldr w0, [x0]
 
 entry:
   %tmp1 = load i32, i32* @i3
   ret i32 %tmp1
 }
 
-define i32* @f6() {
+define i32* @f6() uwtable {
 ; ARM64-LABEL: f6:
 ; ARM64:        adrp x0, __emutls_v.i3
 ; ARM64:        add x0, x0, :lo12:__emutls_v.i3
 ; ARM64-NEXT:   bl __emutls_get_address
+; ARM64-NEXT:   .cfi_def_cfa wsp, 16
 ; ARM64-NEXT:   ldp x29, x30, [sp]
 
 entry:
   ret i32* @i3
 }
 
 ; Simple test of comdat __thread variables.
 ; template <class T> struct A { static __thread T x; };

llvm/test/CodeGen/AArch64/fastcc.ll

 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=non-leaf -tailcallopt | FileCheck %s -check-prefix CHECK-TAIL
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=non-leaf | FileCheck %s
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=non-leaf -tailcallopt -aarch64-redzone | FileCheck %s -check-prefix CHECK-TAIL-RZ
 
 ; Without tailcallopt fastcc still means the caller cleans up the
 ; stack, so try to make sure this is respected.
 
-define fastcc void @func_stack0() {
+define fastcc void @func_stack0() uwtable {
 ; CHECK-LABEL: func_stack0:
 ; CHECK: sub sp, sp, #48
 ; CHECK: add x29, sp, #32
 ; CHECK: str w{{[0-9]+}}, [sp]
 
 ; CHECK-TAIL-LABEL: func_stack0:
 ; CHECK-TAIL: sub sp, sp, #48
 ; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 48

 
 
 ; CHECK-TAIL: bl func_stack0
 ; CHECK-TAIL-NOT: sub sp, sp
 
   ret void
 ; CHECK: ldp     x29, x30, [sp, #32]
 ; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
 ; CHECK-NEXT: ret
 
 
 ; CHECK-TAIL: ldp     x29, x30, [sp, #32]
 ; CHECK-TAIL-NEXT: add sp, sp, #48
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-NEXT: .cfi_restore w30
+; CHECK-TAIL-NEXT: .cfi_restore w29
 ; CHECK-TAIL-NEXT: ret
 }
 
-define fastcc void @func_stack8([8 x i64], i32 %stacked) {
+define fastcc void @func_stack8([8 x i64], i32 %stacked) uwtable {
 ; CHECK-LABEL: func_stack8:
 ; CHECK: sub sp, sp, #48
 ; CHECK: stp x29, x30, [sp, #32]
 ; CHECK: add x29, sp, #32
 ; CHECK: str w{{[0-9]+}}, [sp]
 
 
 ; CHECK-TAIL-LABEL: func_stack8:
; CHECK-TAIL: sub sp, sp, #32
   call fastcc void @func_stack0()
 ; CHECK: bl func_stack0
 ; CHECK-NOT: sub sp, sp
 
 ; CHECK-TAIL: bl func_stack0
 ; CHECK-TAIL-NOT: sub sp, sp
 
   ret void
+; CHECK-NEXT: .cfi_def_cfa wsp, 48
 ; CHECK-NEXT: ldp     x29, x30, [sp, #32]
-; CHECK: add sp, sp, #48
+; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
 ; CHECK-NEXT: ret
 
 
 ; CHECK-TAIL: ldp     x29, x30, [sp, #32]
 ; CHECK-TAIL-NEXT: add     sp, sp, #64
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -16
+; CHECK-TAIL-NEXT: .cfi_restore w30
+; CHECK-TAIL-NEXT: .cfi_restore w29
 ; CHECK-TAIL-NEXT: ret
 }
 
-define fastcc void @func_stack32([8 x i64], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32([8 x i64], i128 %stacked0, i128 %stacked1) uwtable {
 ; CHECK-LABEL: func_stack32:
 ; CHECK: add x29, sp, #32
 
 ; CHECK-TAIL-LABEL: func_stack32:
 ; CHECK-TAIL: add x29, sp, #32
 
 
   call fastcc void @func_stack8([8 x i64] undef, i32 42)

 ; CHECK: bl func_stack0
 ; CHECK-NOT: sub sp, sp
 
 
 ; CHECK-TAIL: bl func_stack0
 ; CHECK-TAIL-NOT: sub sp, sp
 
   ret void
-; CHECK: ldp     x29, x30, [sp, #32]
+; CHECK:      .cfi_def_cfa wsp, 48
+; CHECK-NEXT: ldp x29, x30, [sp, #32]
 ; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL: ldp     x29, x30, [sp, #32]
 ; CHECK-TAIL-NEXT: add     sp, sp, #80
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-NEXT: .cfi_restore w30
+; CHECK-TAIL-NEXT: .cfi_restore w29
 ; CHECK-TAIL-NEXT: ret
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
-define fastcc void @func_stack32_leaf([8 x i64], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32_leaf([8 x i64], i128 %stacked0, i128 %stacked1) uwtable {
 ; CHECK-LABEL: func_stack32_leaf:
 ; CHECK: str     x20, [sp, #-16]!
 ; CHECK: nop
 ; CHECK-NEXT: //NO_APP
 ; CHECK-NEXT: ldr     x20, [sp], #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w20
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL-LABEL: func_stack32_leaf:
 ; CHECK-TAIL: str     x20, [sp, #-16]!
 ; CHECK-TAIL: nop
 ; CHECK-TAIL-NEXT: //NO_APP
 ; CHECK-TAIL-NEXT: ldr     x20, [sp], #16
-; CHECK-TAIL-NEXT: add     sp, sp, #32
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-NEXT: add  sp, sp, #32
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-NEXT: .cfi_restore w20
 ; CHECK-TAIL-NEXT: ret
 
 ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf:
 ; CHECK-TAIL-RZ: str     x20, [sp, #-16]!
 ; CHECK-TAIL-RZ-NOT: sub     sp, sp
 ; CHECK-TAIL-RZ: nop
 ; CHECK-TAIL-RZ-NEXT: //NO_APP
 ; CHECK-TAIL-RZ-NEXT: ldr     x20, [sp], #16
-; CHECK-TAIL-RZ-NEXT: add     sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-RZ-NEXT: add sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-RZ-NEXT: .cfi_restore w20
 ; CHECK-TAIL-RZ-NEXT: ret
 
   ; Make sure there is a callee-save register to save/restore.
   call void asm sideeffect "nop", "~{x20}"() nounwind
   ret void
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
-define fastcc void @func_stack32_leaf_local([8 x i64], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32_leaf_local([8 x i64], i128 %stacked0, i128 %stacked1) uwtable {
 ; CHECK-LABEL: func_stack32_leaf_local:
 ; CHECK: sub     sp, sp, #32
 ; CHECK-NEXT:  .cfi_def_cfa_offset 32
 ; CHECK-NEXT: str     x20, [sp, #16]
 ; CHECK: nop
 ; CHECK-NEXT: //NO_APP
 ; CHECK-NEXT: ldr     x20, [sp, #16]
 ; CHECK-NEXT: add     sp, sp, #32
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w20
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL-LABEL: func_stack32_leaf_local:
 ; CHECK-TAIL: sub     sp, sp, #32
 ; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 32
 ; CHECK-TAIL-NEXT: str     x20, [sp, #16]
 ; CHECK-TAIL: nop
 ; CHECK-TAIL-NEXT: //NO_APP
 ; CHECK-TAIL-NEXT: ldr     x20, [sp, #16]
 ; CHECK-TAIL-NEXT: add     sp, sp, #64
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-NEXT: .cfi_restore w20
 ; CHECK-TAIL-NEXT: ret
 
 ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf_local:
 ; CHECK-TAIL-RZ: str     x20, [sp, #-16]!
 ; CHECK-TAIL-RZ-NOT: sub     sp, sp
 ; CHECK-TAIL-RZ: nop
 ; CHECK-TAIL-RZ-NEXT: //NO_APP
 ; CHECK-TAIL-RZ-NEXT: ldr     x20, [sp], #16
-; CHECK-TAIL-RZ-NEXT: add     sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset 0
+; CHECK-TAIL-RZ-NEXT: add sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32
+; CHECK-TAIL-RZ-NEXT: .cfi_restore w20
 ; CHECK-TAIL-RZ-NEXT: ret
 
   %val0 = alloca [2 x i64], align 8
 
   ; Make sure there is a callee-save register to save/restore.
   call void asm sideeffect "nop", "~{x20}"() nounwind
   ret void
 }
 
 ; Check that arg stack pop is done after callee-save restore when no frame pointer is used.
-define fastcc void @func_stack32_leaf_local_nocs([8 x i64], i128 %stacked0, i128 %stacked1) {
+define fastcc void @func_stack32_leaf_local_nocs([8 x i64], i128 %stacked0, i128 %stacked1) uwtable {
 ; CHECK-LABEL: func_stack32_leaf_local_nocs:
 ; CHECK: sub     sp, sp, #16
-; CHECK: add     sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
 ; CHECK-NEXT: ret
 
 ; CHECK-TAIL-LABEL: func_stack32_leaf_local_nocs:
 ; CHECK-TAIL: sub     sp, sp, #16
-; CHECK-TAIL: add     sp, sp, #48
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset 16
+; CHECK-TAIL-NEXT: add  sp, sp, #48
+; CHECK-TAIL-NEXT: .cfi_def_cfa_offset -32
 ; CHECK-TAIL-NEXT: ret
 
 ; CHECK-TAIL-RZ-LABEL: func_stack32_leaf_local_nocs:
 ; CHECK-TAIL-RZ: add     sp, sp, #32
+; CHECK-TAIL-RZ-NEXT: .cfi_def_cfa_offset -32
 ; CHECK-TAIL-RZ-NEXT: ret
 
   %val0 = alloca [2 x i64], align 8
 
   ret void
 }

llvm/test/CodeGen/AArch64/framelayout-sve-calleesaves-fix.mir

 # NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 # RUN: llc -mattr=+sve -mtriple=aarch64-none-linux-gnu -start-before=prologepilog %s -o - | FileCheck %s
 
 --- |
-  define aarch64_sve_vector_pcs void @fix_restorepoint_p4() { entry: unreachable }
+  define aarch64_sve_vector_pcs void @fix_restorepoint_p4() uwtable { entry: unreachable }
   ; CHECK-LABEL: fix_restorepoint_p4:
   ; CHECK:       // %bb.0: // %entry
   ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
   ; CHECK-NEXT:    .cfi_def_cfa_offset 16
   ; CHECK-NEXT:    .cfi_offset w29, -16
   ; CHECK-NEXT:    addvl sp, sp, #-2
   ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
   ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
   ; CHECK-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
   ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
   ; CHECK-NEXT:    addvl sp, sp, #-1
   ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
   ; CHECK-NEXT:    // implicit-def: $z8
   ; CHECK-NEXT:    // implicit-def: $p4
   ; CHECK-NEXT:    addvl sp, sp, #1
+  ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
   ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
   ; CHECK-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
   ; CHECK-NEXT:    addvl sp, sp, #2
+  ; CHECK-NEXT:    .cfi_def_cfa wsp, 16
+  ; CHECK-NEXT:    .cfi_restore z8
   ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+  ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+  ; CHECK-NEXT:    .cfi_restore w29
   ; CHECK-NEXT:    ret
 ...
 name: fix_restorepoint_p4
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
 body:             |
   bb.0.entry:
     $z8 = IMPLICIT_DEF
     $p4 = IMPLICIT_DEF
     B %bb.1
 
   bb.1.entry:
     RET_ReallyLR
 ---

llvm/test/CodeGen/AArch64/framelayout-sve.mir

 #     |/////////////| alignment gap.
 #     |     :       |
 #     | Stack objs  |
 #     |     :       |
 #     +-------------+ <- SP after call and frame-setup
 #
 --- |
 
-  define void @test_allocate_sve() { entry: unreachable }
-  define void @test_allocate_sve_gpr_callee_saves() { entry: unreachable }
-  define void @test_allocate_sve_gpr_realigned() { entry: unreachable }
-  define void @test_address_sve() { entry: unreachable }
-  define void @test_address_sve_fp() { entry: unreachable }
-  define void @test_stack_arg_sve() { entry: unreachable }
-  define void @test_address_sve_out_of_range() { entry: unreachable }
-  define void @test_address_gpr_vla() { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_pregs_sve() { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_zregs_sve() { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_sve() { entry: unreachable }
-  define aarch64_sve_vector_pcs void @save_restore_sve_realign() { entry: unreachable }
-  define aarch64_sve_vector_pcs void @frame_layout() { entry: unreachable }
-  define void @fp_relative_index_with_float_save() { entry: unreachable }
+  define void @test_allocate_sve() uwtable { entry: unreachable }
+  define void @test_allocate_sve_gpr_callee_saves() uwtable { entry: unreachable }
+  define void @test_allocate_sve_gpr_realigned() uwtable { entry: unreachable }
+  define void @test_address_sve() uwtable { entry: unreachable }
+  define void @test_address_sve_fp() uwtable { entry: unreachable }
+  define void @test_stack_arg_sve() uwtable { entry: unreachable }
+  define void @test_address_sve_out_of_range() uwtable { entry: unreachable }
+  define void @test_address_gpr_vla() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_pregs_sve() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_zregs_sve() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_sve() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @save_restore_sve_realign() uwtable { entry: unreachable }
+  define aarch64_sve_vector_pcs void @frame_layout() uwtable { entry: unreachable }
+  define void @fp_relative_index_with_float_save() uwtable { entry: unreachable }
 
 ...
 # +----------+
 # |scratchreg|  // x29 is used as scratch reg.
 # +----------+
 # | %stack.0 |  // scalable SVE object of n * 18 bytes, aligned to 16 bytes,
 # |          |  // to be materialized with 2*ADDVL (<=> 2 * n * 16bytes)
 # +----------+
 # | %stack.1 |  // not scalable
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_allocate_sve
 # CHECK:       stackSize: 32
 
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 32
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 
 # ASM-LABEL: test_allocate_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 16 * VG
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 #
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +16
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 
 name:            test_allocate_sve
 stack:
   - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2 }
   - { id: 1, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     RET_ReallyLR
 ---
 ...
 # +----------+
 # | x20, x21 |  // callee saves
 # |scratchreg|  // x29 is used as scratch reg.
 # +----------+
 # | %stack.0 |  // scalable objects
 # +----------+
 # | %stack.1 |  // not scalable
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_allocate_sve_gpr_callee_saves
 # CHECK:       stackSize: 48
 
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 32
 # CHECK-NEXT: frame-setup STPXi killed $x21, killed $x20, $sp, 2
-# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w20, -8
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w21, -16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -32
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22
 #
 # CHECK-NEXT: $x20 = IMPLICIT_DEF
 # CHECK-NEXT: $x21 = IMPLICIT_DEF
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 2
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 48
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 32
 # CHECK-NEXT: $x21, $x20 = frame-destroy LDPXi $sp, 2
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 32
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w20
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w21
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_allocate_sve_gpr_callee_saves:
 # ASM:       .cfi_def_cfa_offset 32
 # ASM:       .cfi_offset w20, -8
 # ASM-NEXT:  .cfi_offset w21, -16
 # ASM-NEXT:  .cfi_offset w29, -32
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 16 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 16 * VG
 #
+# ASM:       .cfi_def_cfa wsp, 48
+# ASM:       .cfi_def_cfa_offset 32
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w20
+# ASM-NEXT:  .cfi_restore w21
+# ASM-NEXT:  .cfi_restore w29
+
+#
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +32
 # UNWINDINFO:       DW_CFA_offset: reg20 -8
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg21 -16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -32
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +48
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +32
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg20
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg21
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 name:            test_allocate_sve_gpr_callee_saves
 stack:
   - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2 }
   - { id: 1, stack-id: default, size: 16, alignment: 8 }
 body:             |
   bb.0.entry:
     $x20 = IMPLICIT_DEF
     $x21 = IMPLICIT_DEF
     RET_ReallyLR
 ---
 ...
 # +----------+
 # |  lr, fp  |  // frame record
 # +----------+ <- FP
 # | %stack.0 |  // scalable objects
 # +----------+
 # |//////////|  // alignment gap
 # | %stack.1 |  // not scalable
 # +----------+ <- SP
 # CHECK-LABEL: name: test_allocate_sve_gpr_realigned
 # CHECK:       stackSize: 32
 
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
-# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa $w29, 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -8
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -2
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
 # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_allocate_sve_gpr_realigned:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM:       .cfi_def_cfa w29, 16
 # ASM-NEXT:  .cfi_offset w30, -8
 # ASM-NEXT:  .cfi_offset w29, -16
 #
+# ASM:        .cfi_def_cfa wsp, 16
+# ASM:        .cfi_def_cfa_offset 0
+# ASM-NEXT:   .cfi_restore w30
+# ASM-NEXT:   .cfi_restore w29
+
+#
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:       DW_CFA_def_cfa: reg29 +16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg30 -8
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
+#
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg30
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 
 name:            test_allocate_sve_gpr_realigned
 stack:
   - { id: 0, stack-id: scalable-vector, size: 18, alignment: 2  }
   - { id: 1, stack-id: default, size: 16, alignment: 32 }
 body:             |
   bb.0.entry:
     RET_ReallyLR

 # | %stack.3 |  // not scalable
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_address_sve
 # CHECK:       stackSize: 32
 
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
-# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22
 
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = ADDXri $sp, 16
 # CHECK-NEXT: STR_ZXI $z0, killed $[[TMP]], 2
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = ADDXri $sp, 16
 # CHECK-NEXT: STR_ZXI $z1, killed $[[TMP]], 1
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = ADDXri $sp, 16
 # CHECK-NEXT: STR_PXI $p0, killed $[[TMP]], 7
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 32
+
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL:  test_address_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 24 * VG
 #
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+#
 # UNWINDINFO:       DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT:  DW_CFA_offset: reg29 -16
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:       DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:       DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +16
+# UNWINDINFO:       DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:  DW_CFA_restore: reg29
+
 
 name:            test_address_sve
 frameInfo:
   maxAlignment:  16
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 1, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 2, stack-id: scalable-vector, size:  2, alignment: 2 }

 # | %stack.3  |  // not scalable
 # +-----------+ <- SP
 
 # CHECK-LABEL: name: test_address_sve_fp
 # CHECK:       stackSize: 32
 
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
-# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa $w29, 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -8
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
 
 # CHECK-NEXT: STR_ZXI $z0, $fp, -1
 # CHECK-NEXT: STR_ZXI $z1, $fp, -2
 # CHECK-NEXT: STR_PXI $p0, $fp, -17
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3
 # CHECK:      $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_address_sve_fp:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM:       .cfi_def_cfa w29, 16
 # ASM-NEXT:  .cfi_offset w30, -8
 # ASM-NEXT:  .cfi_offset w29, -16
 #
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w30
+# ASM-NEXT:  .cfi_restore w29
+#
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:      DW_CFA_def_cfa: reg29 +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg30
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name:            test_address_sve_fp
 frameInfo:
   maxAlignment:  16
   isFrameAddressTaken: true
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 1, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 2, stack-id: scalable-vector, size:  2, alignment: 2 }

 # +---------- +
 # | %stack.1  |  // not scalable, 16 bytes
 # +-----------+ <- SP
 # CHECK-LABEL: name: test_stack_arg_sve
 # CHECK:       stackSize: 32
 
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
-# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
+
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
-
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK:      $[[TMP:x[0-9]+]] = ADDVL_XXI $sp, 1
 # CHECK-NEXT: $x0 = LDRXui killed $[[TMP]], 4
 
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 32
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_stack_arg_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 8 * VG
 #
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
 
 name:             test_stack_arg_sve
 fixedStack:
   - { id: 0, stack-id: default, size: 16, alignment: 16, offset: 0 }
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
   - { id: 1, stack-id: default, size: 16, alignment: 16 }
 body:             |

 # |:        :|
 # |::::::::::|
 # +----------+ <- SP
 # CHECK-LABEL: name: test_address_sve_out_of_range
 # CHECK:       stackSize: 16
 
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
-# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x04, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x06, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0a, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0c, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -32
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22
 
 # CHECK-NEXT: $[[TMP2:x[0-9]+]] = ADDVL_XXI $sp, 1
 # CHECK-NEXT: STR_ZXI $z0, killed $[[TMP2]], 255
 
 # CHECK-NEXT: $[[TMP2:x[0-9]+]] = ADDPL_XXI $sp, 1
 # CHECK-NEXT: STR_PXI $p0, killed $[[TMP2]], 255
 
 # CHECK:      $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x98, 0x0c, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa0, 0x0a, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa8, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb0, 0x06, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb8, 0x04, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc0, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 9
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
 # CHECK-NEXT: $sp, $[[SCRATCH]] = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_address_sve_out_of_range:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 256 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x04, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 512 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x06, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 768 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1024 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0a, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1280 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0c, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1536 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1792 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x80, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 2048 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 2056 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1808 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x98, 0x0c, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1560 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa0, 0x0a, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1312 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa8, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1064 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb0, 0x06, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 816 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xb8, 0x04, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 568 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc0, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 320 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc8, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
+# ASM:        .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +256, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +512, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +768, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1024, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1280, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1536, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1792, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +2048, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +2056, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1808, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1560, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1312, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +1064, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +816, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +568, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +320, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +72, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name:            test_address_sve_out_of_range
 frameInfo:
   maxAlignment:  16
 stack:
   - { id: 0, stack-id: scalable-vector, size:   16, alignment: 16 }
   - { id: 1, stack-id: scalable-vector, size: 3584, alignment: 16 }
   - { id: 2, stack-id: scalable-vector, size:  512, alignment: 16 }
 

 # +----------+ <- @FP - 16 scalable bytes
 # | %stack.1 |  // 16 bytes
 # +----------+ <- @BP
 # : %stack.2 :  // variable length
 # +----------+ <- SP
 
 # CHECK-LABEL: name: test_address_gpr_vla
 # CHECK: bb.0.entry:
-# CHECK: STRXui $xzr, $x19, 0
-# CHECK: RET_ReallyLR
+# CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -4
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 32
+# CHECK-NEXT: frame-setup STRXui killed $x19, $sp, 2
+# CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa $w29, 32
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w19, -16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -24
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -32
+# CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
+# CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-NEXT: $x19 = ADDXri $sp, 0, 0
+# CHECK-NEXT: STRXui $xzr, $x19, 0
+# CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1
+# CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 32
+# CHECK-NEXT: $x19 = frame-destroy LDRXui $sp, 2
+# CHECK-NEXT: early-clobber $sp, $fp, $lr = frame-destroy LDPXpost $sp, 4
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w19
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
+# CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: test_address_gpr_vla:
 # ASM:       .cfi_def_cfa_offset 32
 # ASM:       .cfi_def_cfa w29, 32
 # ASM-NEXT:  .cfi_offset w19, -16
 # ASM-NEXT:  .cfi_offset w30, -24
 # ASM-NEXT:  .cfi_offset w29, -32
 #
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w19
+# ASM-NEXT:  .cfi_restore w30
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +32
 # UNWINDINFO:      DW_CFA_def_cfa: reg29 +32
 # UNWINDINFO-NEXT: DW_CFA_offset: reg19 -16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -24
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32
+#
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg19
+# UNWINDINFO-NEXT: DW_CFA_restore: reg30
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name:            test_address_gpr_vla
 frameInfo:
   maxAlignment:  16
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 8 }
   - { id: 1, stack-id: default, size: 16, alignment: 8 }
   - { id: 2, stack-id: default, type: variable-sized }
 body:             |
   bb.0.entry:
     liveins: $xzr
 
     STRXui $xzr, %stack.1, 0
 
     RET_ReallyLR
 ---
 ...
 # CHECK-LABEL: name: save_restore_pregs_sve
-# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
-# CHECK: $sp = frame-setup ADDVL_XXI $sp, -1
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
-# CHECK: frame-setup STR_PXI killed $p6, $sp, 5
-# CHECK: frame-setup STR_PXI killed $p5, $sp, 6
-# CHECK: frame-setup STR_PXI killed $p4, $sp, 7
-# CHECK: $sp = frame-setup SUBXri $sp, 32, 0
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK:      $sp = frame-setup STRXpre killed $fp, $sp, -16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
+# CHECK:      $sp = frame-setup ADDVL_XXI $sp, -1
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK:      frame-setup STR_PXI killed $p6, $sp, 5
+# CHECK:      frame-setup STR_PXI killed $p5, $sp, 6
+# CHECK:      frame-setup STR_PXI killed $p4, $sp, 7
+# CHECK:      $sp = frame-setup SUBXri $sp, 32, 0
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
 
-# CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
-# CHECK: $p6 = frame-destroy LDR_PXI $sp, 5
-# CHECK: $p5 = frame-destroy LDR_PXI $sp, 6
-# CHECK: $p4 = frame-destroy LDR_PXI $sp, 7
-# CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1
-# CHECK: RET_ReallyLR
+# CHECK:      $sp = frame-destroy ADDXri $sp, 32, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK:      $p6 = frame-destroy LDR_PXI $sp, 5
+# CHECK:      $p5 = frame-destroy LDR_PXI $sp, 6
+# CHECK:      $p4 = frame-destroy LDR_PXI $sp, 7
+# CHECK:      $sp = frame-destroy ADDVL_XXI $sp, 1
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
+# CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.4)
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
+# CHECK-NEXT: RET_ReallyLR
+
 #
 # ASM-LABEL: save_restore_pregs_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 8 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:         DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:         DW_CFA_offset: reg29 -16
 # UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:         DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:         DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:         DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT:    DW_CFA_restore: reg29
+
 name: save_restore_pregs_sve
 stack:
   - { id: 0, stack-id: default, size: 32, alignment: 16 }
 body:             |
   bb.0.entry:
 
     $p4 = IMPLICIT_DEF
     $p5 = IMPLICIT_DEF
     $p6 = IMPLICIT_DEF
 
     RET_ReallyLR
 ---
 ...
 # CHECK-LABEL: name: save_restore_zregs_sve
 # CHECK:      $sp = frame-setup STRXpre killed $fp, $sp, -16
-# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: frame-setup STR_ZXI killed $z10, $sp, 0
 # CHECK-NEXT: frame-setup STR_ZXI killed $z9, $sp, 1
 # CHECK-NEXT: frame-setup STR_ZXI killed $z8, $sp, 2
-# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup SUBXri $sp, 32, 0
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22
 
 # CHECK:      $sp  = frame-destroy ADDXri $sp, 32, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $z10 = frame-destroy LDR_ZXI $sp, 0
 # CHECK-NEXT: $z9  = frame-destroy LDR_ZXI $sp, 1
 # CHECK-NEXT: $z8  = frame-destroy LDR_ZXI $sp, 2
 # CHECK-NEXT: $sp  = frame-destroy ADDVL_XXI $sp, 3
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z8
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z9
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z10
 # CHECK-NEXT: $sp, $fp = frame-destroy LDRXpost $sp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: save_restore_zregs_sve:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9  @ cfa - 16 - 16 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10  @ cfa - 16 - 24 * VG
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 48 + 24 * VG
+#
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM-NEXT:  .cfi_restore z8
+# ASM-NEXT:  .cfi_restore z9
+# ASM-NEXT:  .cfi_restore z10
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
 
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +48, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
 
 name: save_restore_zregs_sve
 stack:
   - { id: 0, stack-id: default, size: 32, alignment: 16 }
 body:             |
   bb.0.entry:
 
     $z8 = IMPLICIT_DEF
     $z9 = IMPLICIT_DEF
     $z10 = IMPLICIT_DEF
 
     RET_ReallyLR
 ---
 ...
 # Test allocation/deallocation of the stack frame together with the
 # saving/restoring of callee save registers. Fixed-stack objects
 # are allocated before the callee-saves.
 # This also adds some non-SVE callee-saves, to ensure that those are
 # paired correctly.
 #
 # CHECK-LABEL: name: save_restore_sve
-# CHECK: $sp = frame-setup STPXpre killed ${{[a-z0-9]+}}, killed $x21, $sp, -4
-# CHECK: frame-setup CFI_INSTRUCTION
-# CHECK: frame-setup STPXi killed $x20, killed $x19, $sp, 2
-# CHECK-COUNT-4: frame-setup CFI_INSTRUCTION
-# CHECK: $sp = frame-setup ADDVL_XXI $sp, -18
-# CHECK: frame-setup CFI_INSTRUCTION
-# CHECK: frame-setup STR_PXI killed $p15, $sp, 4
-# CHECK: frame-setup STR_PXI killed $p14, $sp, 5
-# CHECK: frame-setup STR_PXI killed $p5, $sp, 14
-# CHECK: frame-setup STR_PXI killed $p4, $sp, 15
-# CHECK: frame-setup STR_ZXI killed $z23, $sp, 2
-# CHECK: frame-setup STR_ZXI killed $z22, $sp, 3
-# CHECK: frame-setup STR_ZXI killed $z9, $sp, 16
-# CHECK: frame-setup STR_ZXI killed $z8, $sp, 17
-# CHECK-COUNT-8: frame-setup CFI_INSTRUCTION
-# CHECK: $sp = frame-setup ADDVL_XXI $sp, -1
-# CHECK: frame-setup CFI_INSTRUCTION
-# CHECK: $sp = frame-setup SUBXri $sp, 32, 0
-# CHECK: frame-setup CFI_INSTRUCTION
+# CHECK:      $sp = frame-setup STPXpre killed ${{[a-z0-9]+}}, killed $x21, $sp, -4
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 32
+# CHECK-NEXT: frame-setup STPXi killed $x20, killed $x19, $sp, 2
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w19, -8
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w20, -16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w21, -24
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -32
+
+# CHECK:      $sp = frame-setup ADDVL_XXI $sp, -18
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK:      frame-setup STR_PXI killed $p15, $sp, 4
+# CHECK:      frame-setup STR_PXI killed $p14, $sp, 5
+# CHECK:      frame-setup STR_PXI killed $p5, $sp, 14
+# CHECK:      frame-setup STR_PXI killed $p4, $sp, 15
+# CHECK:      frame-setup STR_ZXI killed $z23, $sp, 2
+# CHECK:      frame-setup STR_ZXI killed $z22, $sp, 3
+# CHECK:      frame-setup STR_ZXI killed $z9, $sp, 16
+# CHECK:      frame-setup STR_ZXI killed $z8, $sp, 17
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x48, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x49, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4a, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4b, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4c, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4d, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4e, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4f, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK:      $sp = frame-setup ADDVL_XXI $sp, -1
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK:      $sp = frame-setup SUBXri $sp, 32, 0
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0e, 0x8f, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22
 
-# CHECK: $sp = frame-destroy ADDXri $sp, 32, 0
-# CHECK: $sp = frame-destroy ADDVL_XXI $sp, 1
-# CHECK: $p15 = frame-destroy LDR_PXI $sp, 4
-# CHECK: $p14 = frame-destroy LDR_PXI $sp, 5
-# CHECK: $p5 = frame-destroy LDR_PXI $sp, 14
-# CHECK: $p4 = frame-destroy LDR_PXI $sp, 15
-# CHECK: $z23 = frame-destroy LDR_ZXI $sp, 2
-# CHECK: $z22 = frame-destroy LDR_ZXI $sp, 3
-# CHECK: $z9 = frame-destroy LDR_ZXI $sp, 16
-# CHECK: $z8 = frame-destroy LDR_ZXI $sp, 17
-# CHECK: $sp = frame-destroy ADDVL_XXI $sp, 18
+# CHECK:      $sp = frame-destroy ADDXri $sp, 32, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK:      $sp = frame-destroy ADDVL_XXI $sp, 1
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK:      $p15 = frame-destroy LDR_PXI $sp, 4
+# CHECK:      $p14 = frame-destroy LDR_PXI $sp, 5
+# CHECK:      $p5 = frame-destroy LDR_PXI $sp, 14
+# CHECK:      $p4 = frame-destroy LDR_PXI $sp, 15
+# CHECK:      $z23 = frame-destroy LDR_ZXI $sp, 2
+# CHECK:      $z22 = frame-destroy LDR_ZXI $sp, 3
+# CHECK:      $z9 = frame-destroy LDR_ZXI $sp, 16
+# CHECK:      $z8 = frame-destroy LDR_ZXI $sp, 17
+# CHECK:      $sp = frame-destroy ADDVL_XXI $sp, 18
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 32
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z8
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z9
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z10
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z11
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z12
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z13
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z14
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z15
+
 # CHECK: $x20, $x19 = frame-destroy LDPXi $sp, 2
 # CHECK: $sp, ${{[a-z0-9]+}}, $x21 = frame-destroy LDPXpost $sp, 4
-# CHECK: RET_ReallyLR
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w19
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w20
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w21
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
+
+# CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: save_restore_sve:
 # ASM:       .cfi_def_cfa_offset 32
 # ASM:       .cfi_offset w19, -8
 # ASM-NEXT:  .cfi_offset w20, -16
 # ASM-NEXT:  .cfi_offset w21, -24
 # ASM-NEXT:  .cfi_offset w29, -32
-# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG 
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 32 - 8 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 32 - 16 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 32 - 24 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 32 - 32 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 32 - 40 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 32 - 48 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 32 - 56 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x60, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 32 - 64 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 152 * VG
 # ASM:       .cfi_escape 0x0f, 0x0e, 0x8f, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 64 + 152 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x98, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 152 * VG
+# ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x20, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 32 + 144 * VG
+# ASM:       .cfi_def_cfa wsp, 32
+# ASM-NEXT:  .cfi_restore z8
+# ASM-NEXT:  .cfi_restore z9
+# ASM-NEXT:  .cfi_restore z10
+# ASM-NEXT:  .cfi_restore z11
+# ASM-NEXT:  .cfi_restore z12
+# ASM-NEXT:  .cfi_restore z13
+# ASM-NEXT:  .cfi_restore z14
+# ASM-NEXT:  .cfi_restore z15
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w19
+# ASM-NEXT:  .cfi_restore w20
+# ASM-NEXT:  .cfi_restore w21
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +32
 # UNWINDINFO:      DW_CFA_offset: reg19 -8
 # UNWINDINFO-NEXT: DW_CFA_offset: reg20 -16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg21 -24
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -32
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +144, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -32, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -40, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -48, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -56, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -32, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +64, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +152, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +32, DW_OP_plus, DW_OP_consts +144, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +32
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg107
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg108
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg109
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg110
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg111
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg19
+# UNWINDINFO-NEXT: DW_CFA_restore: reg20
+# UNWINDINFO-NEXT: DW_CFA_restore: reg21
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
 
 name: save_restore_sve
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
   - { id: 1, stack-id: default, size: 32, alignment: 16 }
 body:             |
   bb.0.entry:
 

 ---
 ...
 # Test allocation/deallocation of the stack frame together with the
 # saving/restoring of callee save registers. Fixed-stack objects
 # are allocated before the callee-saves.
 #
 # CHECK-LABEL: name: save_restore_sve_realign
 # CHECK:      $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: $fp = frame-setup ADDXri $sp, 0, 0
-# CHECK-COUNT-3: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa $w29, 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -8
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -18
 # CHECK-NEXT: STR_PXI killed $p15, $sp, 4
 # CHECK-NEXT: STR_PXI killed $p14, $sp, 5
 # CHECK:      STR_PXI killed $p5, $sp, 14
 # CHECK-NEXT: STR_PXI killed $p4, $sp, 15
 # CHECK-NEXT: STR_ZXI killed $z23, $sp, 2
 # CHECK-NEXT: STR_ZXI killed $z22, $sp, 3
 # CHECK:      STR_ZXI killed $z9, $sp, 16
 # CHECK-NEXT: STR_ZXI killed $z8, $sp, 17
-# CHECK-COUNT-8: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
 # CHECK-NEXT: $[[TMP:x[0-9]+]] = frame-setup SUBXri $sp, 16, 0
 # CHECK-NEXT: $sp = ANDXri killed $[[TMP]]
 
 # CHECK:      $sp = frame-destroy ADDVL_XXI $fp, -18
 # CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 4
 # CHECK-NEXT: $p14 = frame-destroy LDR_PXI $sp, 5
 # CHECK:      $p5 = frame-destroy LDR_PXI $sp, 14
 # CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 15
 # CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 2
 # CHECK-NEXT: $z22 = frame-destroy LDR_ZXI $sp, 3
 # CHECK:      $z9 = frame-destroy LDR_ZXI $sp, 16
 # CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 17
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z8
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z9
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z10
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z11
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z12
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z13
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z14
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z15
 # CHECK-NEXT: $sp = frame-destroy ADDXri $fp, 0, 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
 # CHECK-NEXT: $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
 # CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: save_restore_sve_realign:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM:       .cfi_def_cfa w29, 16
 # ASM-NEXT:  .cfi_offset w30, -8
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9  @ cfa - 16 - 16 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10  @ cfa - 16 - 24 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11  @ cfa - 16 - 32 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12  @ cfa - 16 - 40 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13  @ cfa - 16 - 48 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14  @ cfa - 16 - 56 * VG
 # ASM-NEXT:  .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15  @ cfa - 16 - 64 * VG
 #
+# ASM:       .cfi_restore z8
+# ASM-NEXT:  .cfi_restore z9
+# ASM-NEXT:  .cfi_restore z10
+# ASM-NEXT:  .cfi_restore z11
+# ASM-NEXT:  .cfi_restore z12
+# ASM-NEXT:  .cfi_restore z13
+# ASM-NEXT:  .cfi_restore z14
+# ASM-NEXT:  .cfi_restore z15
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w30
+# ASM-NEXT:  .cfi_restore w29
+#
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO:      DW_CFA_def_cfa: reg29 +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg30 -8
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -16, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -32, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -40, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -48, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -56, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -64, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_restore_extended: reg104
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg105
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg106
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg107
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg108
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg109
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg110
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg111
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg30
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name: save_restore_sve_realign
 stack:
   - { id: 0, stack-id: scalable-vector, size: 16, alignment: 16 }
   - { id: 1, stack-id: default, size: 16, alignment: 32 }
 body:             |
   bb.0.entry:
 
     $z8_z9_z10_z11   = IMPLICIT_DEF

 # CHECK-NEXT:       stack-id: scalable-vector, callee-saved-register: '$p4',
 # CHECK:        - { id: 9, name: '', type: spill-slot, offset: -36, size: 2, alignment: 2,
 # CHECK-NEXT:       stack-id: scalable-vector, callee-saved-register: '$p15',
 # CHECK:        - { id: 10, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
 # CHECK-NEXT:       stack-id: default, callee-saved-register: '$fp',
 #
 # CHECK:      bb.0.entry:
 # CHECK-NEXT: $sp = frame-setup STRXpre killed $[[SCRATCH:[a-z0-9]+]], $sp, -16
-# CHECK-COUNT-2: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -3
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: STR_PXI killed $p15, $sp, 6
 # CHECK-NEXT: STR_PXI killed $p4, $sp, 7
 # CHECK-NEXT: STR_ZXI killed $z23, $sp, 1
 # CHECK-NEXT: STR_ZXI killed $z8, $sp, 2
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22
 # CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -7
-# CHECK-NEXT: frame-setup CFI_INSTRUCTION
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22
+
+# CHECK:      $sp = frame-destroy ADDVL_XXI $sp, 7
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22
+# CHECK-NEXT: $p15 = frame-destroy LDR_PXI $sp, 6
+# CHECK-NEXT: $p4 = frame-destroy LDR_PXI $sp, 7
+# CHECK-NEXT: $z23 = frame-destroy LDR_ZXI $sp, 1
+# CHECK-NEXT: $z8 = frame-destroy LDR_ZXI $sp, 2
+# CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 3
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $z8
+# CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.10)
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
+# CHECK-NEXT: RET_ReallyLR
 #
 # ASM-LABEL: frame_layout:
 # ASM:       .cfi_def_cfa_offset 16
 # ASM-NEXT:  .cfi_offset w29, -16
 # ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 # ASM:       .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8  @ cfa - 16 - 8 * VG
 # ASM:       .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xd0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 80 * VG
 #
+# ASM:       .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+# ASM:       .cfi_def_cfa wsp, 16
+# ASM-NEXT:  .cfi_restore z8
+# ASM:       .cfi_def_cfa_offset 0
+# ASM-NEXT:  .cfi_restore w29
+
 # UNWINDINFO:      DW_CFA_def_cfa_offset: +16
 # UNWINDINFO-NEXT: DW_CFA_offset: reg29 -16
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_consts -16, DW_OP_plus, DW_OP_consts -8, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
 # UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +80, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+#
+# UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +0, DW_OP_consts +16, DW_OP_plus, DW_OP_consts +24, DW_OP_bregx 0x2e +0, DW_OP_mul, DW_OP_plus
+# UNWINDINFO:      DW_CFA_def_cfa: reg31 +16
+# UNWINDINFO-NEXT: DW_CFA_restore_extended: reg104
+# UNWINDINFO:      DW_CFA_def_cfa_offset: +0
+# UNWINDINFO-NEXT: DW_CFA_restore: reg29
+
 name: frame_layout
 stack:
   - { id: 0, type: default,    size:  32, alignment: 16, stack-id: scalable-vector }
   - { id: 1, type: default,    size:   4, alignment:  2, stack-id: scalable-vector }
   - { id: 2, type: default,    size:  16, alignment: 16, stack-id: scalable-vector }
   - { id: 3, type: default,    size:   2, alignment:  2, stack-id: scalable-vector }
   - { id: 4, type: spill-slot, size:  16, alignment: 16, stack-id: scalable-vector }
   - { id: 5, type: spill-slot, size:   2, alignment:  2, stack-id: scalable-vector }

llvm/test/CodeGen/AArch64/framelayout-unaligned-fp.ll

 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 
 ; The purpose of this test is to construct a scenario where an odd number
 ; of callee-saved GPRs as well as an odd number of callee-saved FPRs are
 ; used. This caused the frame pointer to be aligned to a multiple of 8
 ; on non-Darwin platforms, rather than a multiple of 16 as usual.
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
 @a = global i64 0, align 4
 
 
-define i64 @b() {
+define i64 @b() uwtable {
 entry:
   %call = tail call i64 @d()
   %0 = alloca i8, i64 ptrtoint (i64 ()* @d to i64), align 16
   %1 = ptrtoint i8* %0 to i64
   store i64 %1, i64* @a, align 4
   %call1 = call i64 @e()
   %conv = sitofp i64 %call1 to float
   %2 = load i64, i64* @a, align 4
   %call2 = call i64 @f(i64 %2)
   %conv3 = fptosi float %conv to i64
   ret i64 %conv3
 }
 
 ; CHECK-LABEL: b:
 ; CHECK:       str     d8, [sp, #-32]!
 ; CHECK-NEXT:  .cfi_def_cfa_offset 32
 ; CHECK-NEXT:  stp     x29, x30, [sp, #8]
 ; CHECK-NEXT:  str     x19, [sp, #24]
 ; CHECK-NEXT:  add     x29, sp, #8
 
 ; CHECK:       sub     sp, x29, #8
+; CHECK-NEXT:  .cfi_def_cfa wsp, 32
 ; CHECK-NEXT:  ldp     x29, x30, [sp, #8]
 ; CHECK-NEXT:  ldr     x19, [sp, #24]
 ; CHECK-NEXT:  ldr     d8, [sp], #32
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w19
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: .cfi_restore b8
 ; CHECK-NEXT:  ret
 
 declare i64 @d()
 declare i64 @e()
 declare i64 @f(i64)

llvm/test/CodeGen/AArch64/large-stack.ll

entry:
   %arrayidx1 = getelementptr inbounds [268435456 x i64], [268435456 x i64]* %large, i64 0, i64 %1
   %2 = load i64, i64* %arrayidx1, align 8
   %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i64 0, i64 0), i64 %2)
   ret void
 }
 
 declare dso_local i32 @printf(i8*, ...)
 
-attributes #0 = { noinline optnone "frame-pointer"="all" }
+attributes #0 = { noinline optnone "frame-pointer"="all" uwtable }
 
 ; CHECK:                  stp x[[SPILL_REG1:[0-9]+]], x[[SPILL_REG2:[0-9]+]], [sp, #-[[SPILL_OFFSET1:[0-9]+]]]
 ; CHECK-NEXT:             .cfi_def_cfa_offset [[SPILL_OFFSET1]]
 ; CHECK-NEXT:             str x[[SPILL_REG3:[0-9]+]], [sp, #[[SPILL_OFFSET2:[0-9]+]]]
 ; CHECK-NEXT:             mov x[[FRAME:[0-9]+]], sp
 ; CHECK-NEXT:             .cfi_def_cfa w[[FRAME]], [[SPILL_OFFSET1]]
 ; CHECK-COUNT-128:        sub sp, sp, #[[STACK1:[0-9]+]], lsl #12
 ; CHECK-NEXT:             sub sp, sp, #[[STACK2:[0-9]+]], lsl #12

 ; CHECK-NEXT:             add   x[[VAL3:[0-9]+]], sp, #8
 ; CHECK-NEXT:             mov x[[VAL4:[0-9]+]], #8
 ; CHECK-NEXT:             madd  x[[INDEX]], x[[INDEX]], x[[VAL4]], x[[VAL3]]
 ; CHECK-NEXT:             ldr x1, [x[[INDEX]]
 ; CHECK:                  bl  printf
 ; CHECK-COUNT-128:        add sp, sp, #[[STACK1]], lsl #12
 ; CHECK-NEXT:             add sp, sp, #[[STACK2]], lsl #12
 ; CHECK-NEXT:             add sp, sp, #[[STACK3]]
+; CHECK-NEXT:             .cfi_def_cfa wsp, [[SPILL_OFFSET1]]
 ; CHECK-NEXT:             ldr x[[SPILL_REG3]], [sp, #[[SPILL_OFFSET2]]]
 ; CHECK-NEXT:             ldp x[[SPILL_REG1]], x[[SPILL_REG2]], [sp], #[[SPILL_OFFSET1]]
+; CHECK-NEXT:             .cfi_def_cfa_offset 0
+; CHECK-NEXT:             .cfi_restore w[[SPILL_REG3]]
+; CHECK-NEXT:             .cfi_restore w[[SPILL_REG2]]
+; CHECK-NEXT:             .cfi_restore w[[SPILL_REG1]]

llvm/test/CodeGen/AArch64/local_vars.ll

 define void @trivial_func() nounwind {
 ; CHECK-LABEL: trivial_func: // @trivial_func
 ; CHECK-NEXT: // %bb.0
 ; CHECK-NEXT: ret
 
   ret void
 }
 
-define void @trivial_fp_func() {
+define void @trivial_fp_func() uwtable {
 ; CHECK-LABEL: trivial_fp_func:
 ; CHECK: str x30, [sp, #-16]!
 ; CHECK-NOT: mov x29, sp
 
 ; CHECK-WITHFP-ARM64-LABEL: trivial_fp_func:
 ; CHECK-WITHFP-ARM64: stp x29, x30, [sp, #-16]!
 ; CHECK-WITHFP-ARM64-NEXT: .cfi_def_cfa_offset 16
 ; CHECK-WITHFP-ARM64-NEXT: mov x29, sp
 ; CHECK-WITHFP-ARM64-NEXT: .cfi_def_cfa w29, 16
 ; CHECK-WITHFP-ARM64-NEXT: .cfi_offset w30, -8
 ; CHECK-WITHFP-ARM64-NEXT: .cfi_offset w29, -16
 
 ; Dont't really care, but it would be a Bad Thing if this came after the epilogue.
 ; CHECK-WITHFP-ARM64: bl foo
 ; CHECK: bl foo
   call void @foo()
   ret void
 
 ; CHECK: ldr x30, [sp], #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
 ; CHECK-NEXT: ret
 
-; CHECK-WITHFP-ARM64: ldp x29, x30, [sp], #16
+; CHECK-WITHFP-ARM64:      .cfi_def_cfa wsp, 16
+; CHECK-WITHFP-ARM64-NEXT: ldp x29, x30, [sp], #16
+; CHECK-WITHFP-ARM64-NEXT: .cfi_def_cfa_offset 0
+; CHECK-WITHFP-ARM64-NEXT: .cfi_restore w30
+; CHECK-WITHFP-ARM64-NEXT: .cfi_restore w29
 ; CHECK-WITHFP-ARM64-NEXT: ret
 }
 
 define void @stack_local() {
   %local_var = alloca i64
 ; CHECK-LABEL: stack_local:
 ; CHECK: sub sp, sp, #16
 

llvm/test/CodeGen/AArch64/machine-outliner-remarks.ll

define void @bar() #0 !dbg !27 {
   store i32 2, i32* %2, align 4
   store i32 3, i32* %3, align 4
   store i32 4, i32* %4, align 4
   store i32 5, i32* %5, align 4
   store i32 6, i32* %6, align 4, !dbg !35
   ret void
 }
 
-attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="none" "target-cpu"="cyclone" }
+attributes #0 = { noredzone nounwind ssp uwtable minsize "frame-pointer"="none" "target-cpu"="cyclone" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5, !6}
 !llvm.ident = !{!7}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "machine-outliner-remarks.ll", directory: "/tmp")
 !2 = !{}

llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll

 ; V83A:             autiasp
 ; CHECK-NEXT:       ret
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 88
   ret i32 %add
 }
 
-attributes #0 = { "sign-return-address"="non-leaf" }
+attributes #0 = { "sign-return-address"="non-leaf" minsize }
 
 ; CHECK-NOT:        OUTLINED_FUNCTION_{{.*}}
 ; CHECK-NOT:         .cfi_b_key_frame
 ; CHECK-NOT:         paci{{[a,b]}}sp
 ; CHECK-NOT:         hint #2{{[5,7]}}
 ; CHECK-NOT:         .cfi_negate_ra_state
 ; CHECK-NOT:         auti{{[a,b]}}sp
 ; CHECK-NOT:         hint #{{[29,31]}}

llvm/test/CodeGen/AArch64/merge-store-dependency.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mcpu cortex-a53 -mtriple=aarch64-eabi | FileCheck %s --check-prefix=A53
 
 ; PR26827 - Merge stores causes wrong dependency.
 %struct1 = type { %struct1*, %struct1*, i32, i32, i16, i16, void (i32, i32, i8*)*, i8* }
 @gv0 = internal unnamed_addr global i32 0, align 4
 @gv1 = internal unnamed_addr global %struct1** null, align 8
 
-define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg)  {
+define void @test(%struct1* %fde, i32 %fd, void (i32, i32, i8*)* %func, i8* %arg) uwtable {
 ;CHECK-LABEL: test
 ; A53-LABEL: test:
 ; A53:       // %bb.0: // %entry
 ; A53-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
 ; A53-NEXT:    .cfi_def_cfa_offset 16
 ; A53-NEXT:    .cfi_offset w19, -8
 ; A53-NEXT:    .cfi_offset w30, -16
+; A53-NEXT:    .cfi_remember_state
 ; A53-NEXT:    movi v0.2d, #0000000000000000
 ; A53-NEXT:    mov x8, x0
 ; A53-NEXT:    mov x19, x8
 ; A53-NEXT:    mov w0, w1
 ; A53-NEXT:    mov w9, #256
 ; A53-NEXT:    stp x2, x3, [x8, #32]
 ; A53-NEXT:    mov x2, x8
 ; A53-NEXT:    str q0, [x19, #16]!

 ; A53-NEXT:    lsl w9, w9, #1
 ; A53-NEXT:    cmp w9, w8
 ; A53-NEXT:    b.le .LBB0_2
 ; A53-NEXT:  // %bb.3: // %while.end.i
 ; A53-NEXT:    bl foo
 ; A53-NEXT:    adrp x8, gv1
 ; A53-NEXT:    str x0, [x8, :lo12:gv1]
 ; A53-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; A53-NEXT:    .cfi_def_cfa_offset 0
+; A53-NEXT:    .cfi_restore w19
+; A53-NEXT:    .cfi_restore w30
 ; A53-NEXT:    ret
 ; A53-NEXT:    .p2align 4, 0x0, 8
 ; A53-NEXT:  .LBB0_4: // %while.body.i.split
 ; A53-NEXT:    // =>This Inner Loop Header: Depth=1
+; A53-NEXT:    .cfi_restore_state
 ; A53-NEXT:    b .LBB0_4
 entry:
   %0 = bitcast %struct1* %fde to i8*
   tail call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 40, i1 false)
   %state = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 4
   store i16 256, i16* %state, align 8
   %fd1 = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 2
   store i32 %fd, i32* %fd1, align 8

llvm/test/CodeGen/AArch64/nomerge.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s
 
 define void @foo(i32 %i) uwtable {
 ; CHECK-LABEL: foo:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    cmp w0, #7
 ; CHECK-NEXT:    b.eq .LBB0_3
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    cmp w0, #5
 ; CHECK-NEXT:    b.ne .LBB0_4
 ; CHECK-NEXT:  // %bb.2: // %if.then
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    b bar
 ; CHECK-NEXT:  .LBB0_3: // %if.then2
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:  .LBB0_4: // %if.end3
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    b bar
 entry:
   switch i32 %i, label %if.end3 [
     i32 5, label %if.then
     i32 7, label %if.then2
   ]
 
 if.then:

llvm/test/CodeGen/AArch64/optimize-cond-branch.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -o - %s | FileCheck %s
 target triple = "arm64--"
 
 ; AArch64InstrInfo::optimizeCondBranch() optimizes the
 ; "x = and y, 256; cmp x, 0; br" from an "and; cbnz" to a tbnz instruction.
 ; It forgot to clear the a flag resulting in a MachineVerifier complaint.
 ;
 ; Writing a stable/simple test is tricky since most tbz instructions are already
 ; formed in SelectionDAG, optimizeCondBranch() only triggers if the and
 ; instruction is in a different block than the conditional jump.
 
-define void @func() {
+define void @func() uwtable {
 ; CHECK-LABEL: func:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    cbnz w8, .LBB0_3
 ; CHECK-NEXT:  // %bb.1: // %b1
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    cbz wzr, .LBB0_4
 ; CHECK-NEXT:  // %bb.2: // %b3
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    and w0, w8, #0x100
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    cbz w0, .LBB0_5
 ; CHECK-NEXT:  .LBB0_3: // %common.ret.sink.split
-; CHECK-NEXT:    .cfi_def_cfa wsp, 0
-; CHECK-NEXT:    .cfi_same_value w30
 ; CHECK-NEXT:    b extfunc
 ; CHECK-NEXT:  .LBB0_4: // %b2
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:    bl extfunc
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    cbnz w0, .LBB0_3
 ; CHECK-NEXT:  .LBB0_5: // %common.ret
 ; CHECK-NEXT:    ret
   %c0 = icmp sgt i64 0, 0
   br i1 %c0, label %b1, label %b6
 
 b1:
   br i1 undef, label %b3, label %b2

llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-arm-none-eabi < %s | FileCheck %s
 
 @A = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8
 @B = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8
 @C = external dso_local local_unnamed_addr global [8 x [8 x i64]], align 8
 
-define dso_local void @run_test() local_unnamed_addr #0 {
+define dso_local void @run_test() local_unnamed_addr uwtable {
 ; CHECK-LABEL: run_test:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill

 ; CHECK-NEXT:    stp q24, q23, [x8, #240]
 ; CHECK-NEXT:    stp q22, q21, [x8, #272]
 ; CHECK-NEXT:    stp q20, q19, [x8, #304]
 ; CHECK-NEXT:    stp q18, q17, [x8, #336]
 ; CHECK-NEXT:    stp q16, q7, [x8, #368]
 ; CHECK-NEXT:    stp q6, q5, [x8, #400]
 ; CHECK-NEXT:    str q2, [x8, #496]
 ; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
 ; CHECK-NEXT:    ret
 ; CH`ECK-NEXT:    .cfi_offset b9, -16
 entry:
   br label %for.cond1.preheader
 
 for.cond1.preheader:                              ; preds = %for.cond1.preheader, %entry
   %0 = phi <2 x i64> [ undef, %entry ], [ %118, %for.cond1.preheader ]
   %1 = phi <2 x i64> [ undef, %entry ], [ %116, %for.cond1.preheader ]

llvm/test/CodeGen/AArch64/settag.ll

 ; CHECK-NEXT:    cbnz x8, .LBB8_1
 ; CHECK-NEXT:  // %bb.2: // %entry
 ; CHECK-NEXT:    ret
 entry:
   call void @llvm.aarch64.settag.zero(i8* %p, i64 272)
   ret void
 }
 
-define void @stg_alloca1() {
+define void @stg_alloca1() uwtable {
 ; CHECK-LABEL: stg_alloca1:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    stg sp, [sp], #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
   %a = alloca i8, i32 16, align 16
   call void @llvm.aarch64.settag(i8* %a, i64 16)
   ret void
 }
 
-define void @stg_alloca5() {
+define void @stg_alloca5() uwtable {
 ; CHECK-LABEL: stg_alloca5:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #80
 ; CHECK-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-NEXT:    st2g sp, [sp, #32]
 ; CHECK-NEXT:    stg sp, [sp, #64]
 ; CHECK-NEXT:    st2g sp, [sp], #80
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    ret
 entry:
   %a = alloca i8, i32 80, align 16
   call void @llvm.aarch64.settag(i8* %a, i64 80)
   ret void
 }
 
-define void @stg_alloca17() {
+define void @stg_alloca17() uwtable {
 ; CHECK-LABEL: stg_alloca17:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #288
 ; CHECK-NEXT:    .cfi_def_cfa_offset 288
 ; CHECK-NEXT:    str x29, [sp, #272] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    mov x8, #256
 ; CHECK-NEXT:  .LBB11_1: // %entry
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    st2g sp, [sp], #32
 ; CHECK-NEXT:    sub x8, x8, #32
 ; CHECK-NEXT:    cbnz x8, .LBB11_1
 ; CHECK-NEXT:  // %bb.2: // %entry
 ; CHECK-NEXT:    stg sp, [sp], #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 entry:
   %a = alloca i8, i32 272, align 16
   call void @llvm.aarch64.settag(i8* %a, i64 272)
   ret void
 }
 
 declare void @llvm.aarch64.settag(i8* %p, i64 %a)
 declare void @llvm.aarch64.settag.zero(i8* %p, i64 %a)

llvm/test/CodeGen/AArch64/shrink-wrapping-vla.ll

 ;    a[i] = x[n - i - 1];
 ;
 ;  for (int i = 0; i < n; i++)
 ;    x[i] = a[i] + 1;
 ; }
 ;
 ; RUN: llc -mtriple aarch64-linux %s -o - | FileCheck %s
 
-define dso_local void @f(i32 %n, i32* nocapture %x) {
+define dso_local void @f(i32 %n, i32* nocapture %x) uwtable {
 entry:
   %cmp = icmp slt i32 %n, 0
   br i1 %cmp, label %return, label %if.end
 
 if.end:                                           ; preds = %entry
   %0 = zext i32 %n to i64
   %1 = tail call i8* @llvm.stacksave()
   %vla = alloca i32, i64 %0, align 16

 
 ; Check that llvm.stackrestore() happens before CSRs are popped off the stack
 
 ; CHECK-LABEL: f
 
 ; CHECK:      stp x29, x30, [sp, #-16]!
 ; CHECK-NEXT: .cfi_def_cfa_offset 16
 ; CHECK-NEXT: mov x29, sp
+; CHECK-NEXT: .cfi_def_cfa w29, 16
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
+
 
 ; VLA allocation
 ; CHECK: mov [[X2:x[0-9]+]], sp
 ; CHECK: mov [[SAVE:x[0-9]+]], sp
 ; CHECK: add [[X1:x[0-9]+]], [[X1]], #15
 ; CHECK: and [[X1]], [[X1]], #0x7fffffff0
 ; Saving the SP via llvm.stacksave()
 ; CHECK: sub [[X2]], [[X2]], [[X1]]
 
 ; The next instruction comes from llvm.stackrestore()
 ; CHECK:      mov sp, [[SAVE]]
 ; Epilogue
 ; CHECK-NEXT: mov sp, x29
+; CHECK-NEXT: .cfi_def_cfa wsp, 16
 ; CHECK-NEXT: ldp x29, x30, [sp], #16
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT:  ret

llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll

 ; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
 ; CHECK-NEXT:  mov     sp, [[TMPREG]]
 ; CHECK-NEXT:  ret
   %0 = load double, double* %b, align 16
   %vld1_lane = insertelement <2 x double> <double undef, double 0.000000e+00>, double %0, i32 0
   ret <2 x double> %vld1_lane
 }
 
-define i32 @deadload() speculative_load_hardening {
+define i32 @deadload() speculative_load_hardening uwtable {
 entry:
 ; CHECK-LABEL: deadload
 ; CHECK:       cmp     sp, #0
 ; CHECK-NEXT:  csetm   x16, ne
 ; CHECK-NEXT:  sub     sp, sp, #16
 ; CHECK-NEXT:  .cfi_def_cfa_offset 16
 ; CHECK-NEXT:  ldr     w8, [sp, #12]
 ; CHECK-NEXT:  add     sp, sp, #16
+; CHECK-NEXT:  .cfi_def_cfa_offset 0
 ; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
 ; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
 ; CHECK-NEXT:  mov     sp, [[TMPREG]]
 ; CHECK-NEXT:  ret
   %a = alloca i32, align 4
   %val = load volatile i32, i32* %a, align 4
   ret i32 undef
 }

llvm/test/CodeGen/AArch64/split-vector-insert.ll

 ; RUN: llc < %s -debug-only=legalize-types 2>&1 | FileCheck %s --check-prefix=CHECK-LEGALIZATION
 ; RUN: llc < %s | FileCheck %s
 ; REQUIRES: asserts
 
 target triple = "aarch64-unknown-linux-gnu"
-attributes #0 = {"target-features"="+sve"}
+attributes #0 = {"target-features"="+sve" uwtable}
 
 declare <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64>, <8 x i64>, i64)
 declare <vscale x 2 x double> @llvm.experimental.vector.insert.nxv2f64.v8f64(<vscale x 2 x double>, <8 x double>, i64)
 
 define <vscale x 2 x i64> @test_nxv2i64_v8i64(<vscale x 2 x i64> %a, <8 x i64> %b) #0 {
 ; CHECK-LEGALIZATION: Legally typed node: [[T1:t[0-9]+]]: nxv2i64 = insert_subvector {{t[0-9]+}}, {{t[0-9]+}}, Constant:i64<0>
 ; CHECK-LEGALIZATION: Legally typed node: [[T2:t[0-9]+]]: nxv2i64 = insert_subvector [[T1]], {{t[0-9]+}}, Constant:i64<2>
 ; CHECK-LEGALIZATION: Legally typed node: [[T3:t[0-9]+]]: nxv2i64 = insert_subvector [[T2]], {{t[0-9]+}}, Constant:i64<4>

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    csel x8, x8, x9, lo
 ; CHECK-NEXT:    addvl x9, sp, #2
 ; CHECK-NEXT:    lsl x8, x8, #3
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #2, mul vl]
 ; CHECK-NEXT:    str q4, [x9, x8]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #2, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
 
 
 
 
   %r = call <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> %a, <8 x i64> %b, i64 0)
   ret <vscale x 2 x i64> %r
 }

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    csel x8, x8, x9, lo
 ; CHECK-NEXT:    addvl x9, sp, #2
 ; CHECK-NEXT:    lsl x8, x8, #3
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #2, mul vl]
 ; CHECK-NEXT:    str q4, [x9, x8]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #2, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:   .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
 
 
 
 
   %r = call <vscale x 2 x double> @llvm.experimental.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> %a, <8 x double> %b, i64 0)
   ret <vscale x 2 x double> %r
 }

llvm/test/CodeGen/AArch64/stack-guard-sysreg.ll

 ; CHECK-LABEL:   foo:
 ; CHECK:         // %bb.0: // %entry
 ; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    mov x29, sp
 ; CHECK-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    mrs x8, SP_EL0
 ; CHECK-NEXT:    lsl x9, x0, #2
 ; CHECK-NO-OFFSET:       ldr x8, [x8]
 ; CHECK-POSITIVE-OFFSET: ldr x8, [x8, #8]
 ; CHECK-NEGATIVE-OFFSET: ldur x8, [x8, #-8]
 ; CHECK-NPOT-OFFSET:     ldur x8, [x8, #1]
 ; CHECK-NPOT-NEG-OFFSET: ldur x8, [x8, #-1]

 ; CHECK-257-OFFSET-NEXT: ldr x8, [x8]
 ; CHECK-MINUS-257-OFFSET:      sub x8, x8, #257
 ; CHECK-MINUS-257-OFFSET-NEXT: ldr x8, [x8]
 ; CHECK-NEXT:    ldur x9, [x29, #-8]
 ; CHECK-NEXT:    cmp x8, x9
 ; CHECK-NEXT:    b.ne .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:   .cfi_def_cfa_offset 0
+; CHECK-NEXT:   .cfi_restore w30
+; CHECK-NEXT:   .cfi_restore w29
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %entry
+; CHECK-NEXT:   .cfi_restore_state
 ; CHECK-NEXT:    bl __stack_chk_fail
 ; CHECK-NOT: __stack_chk_guard
 entry:
   %vla = alloca i32, i64 %t, align 4
   call void @baz(i32* nonnull %vla)
   ret void
 }
 
 declare void @baz(i32*)
 
 ; CHECK-BAD-OFFSET: LLVM ERROR: Unable to encode Stack Protector Guard Offset
 
-attributes #0 = { sspstrong }
+attributes #0 = { sspstrong uwtable }
 !llvm.module.flags = !{!1, !2, !3}
 
 !1 = !{i32 2, !"stack-protector-guard", !"sysreg"}
 !2 = !{i32 2, !"stack-protector-guard-reg", !"sp_el0"}
 
 ;--- a.ll
 !3 = !{i32 2, !"stack-protector-guard-offset", i32 0}
 ;--- b.ll

llvm/test/CodeGen/AArch64/storepairsuppress_minsize.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-none-eabi -mcpu=cortex-a55 -o - %s | FileCheck %s
 
 ; Check that stp are not suppressed at minsize.
 
 %T_IN_BLOCK = type [ 2 x { double, { double, double } } ]
 declare %T_IN_BLOCK @return_in_block()
 @in_block_store = dso_local global %T_IN_BLOCK zeroinitializer, align 8
 
-define void @test_default() {
+define void @test_default() uwtable {
 ; CHECK-LABEL: test_default:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl return_in_block
 ; CHECK-NEXT:    adrp x8, in_block_store
 ; CHECK-NEXT:    add x8, x8, :lo12:in_block_store
 ; CHECK-NEXT:    str d0, [x8]
 ; CHECK-NEXT:    str d1, [x8, #8]
 ; CHECK-NEXT:    str d2, [x8, #16]
 ; CHECK-NEXT:    str d3, [x8, #24]
 ; CHECK-NEXT:    str d4, [x8, #32]
 ; CHECK-NEXT:    str d5, [x8, #40]
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = call %T_IN_BLOCK @return_in_block()
   store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store
   ret void
 }
 
-define void @test_minsize() minsize {
+define void @test_minsize() minsize uwtable {
 ; CHECK-LABEL: test_minsize:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl return_in_block
 ; CHECK-NEXT:    adrp x8, in_block_store
 ; CHECK-NEXT:    add x8, x8, :lo12:in_block_store
 ; CHECK-NEXT:    stp d0, d1, [x8]
 ; CHECK-NEXT:    stp d2, d3, [x8, #16]
 ; CHECK-NEXT:    stp d4, d5, [x8, #32]
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %1 = call %T_IN_BLOCK @return_in_block()
   store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store
   ret void
 }
 
-define void @test_optsize() optsize {
+define void @test_optsize() optsize uwtable {
 ; CHECK-LABEL: test_optsize:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl return_in_block
 ; CHECK-NEXT:    adrp x8, in_block_store
 ; CHECK-NEXT:    add x8, x8, :lo12:in_block_store
 ; CHECK-NEXT:    stp d0, d1, [x8]
 ; CHECK-NEXT:    stp d2, d3, [x8, #16]
 ; CHECK-NEXT:    stp d4, d5, [x8, #32]
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %1 = call %T_IN_BLOCK @return_in_block()
   store %T_IN_BLOCK %1, %T_IN_BLOCK* @in_block_store
   ret void
 }

llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll

 define <vscale x 14 x i1> @extract_nxv14i1_nxv28i1_0(<vscale x 28 x i1> %in) {
 ; CHECK-LABEL: extract_nxv14i1_nxv28i1_0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %res = call <vscale x 14 x i1> @llvm.experimental.vector.extract.nxv14i1.nxv28i1(<vscale x 28 x i1> %in, i64 0)
   ret <vscale x 14 x i1> %res
 }
 
-define <vscale x 14 x i1> @extract_nxv14i1_nxv28i1_14(<vscale x 28 x i1> %in) {
+define <vscale x 14 x i1> @extract_nxv14i1_nxv28i1_14(<vscale x 28 x i1> %in) uwtable {
 ; CHECK-LABEL: extract_nxv14i1_nxv28i1_14:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    punpkhi p2.h, p1.b

 ; CHECK-NEXT:    uzp1 p3.s, p5.s, p3.s
 ; CHECK-NEXT:    uzp1 p0.s, p0.s, p1.s
 ; CHECK-NEXT:    uzp1 p1.h, p2.h, p4.h
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    uzp1 p0.h, p0.h, p3.h
 ; CHECK-NEXT:    uzp1 p0.b, p0.b, p1.b
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = call <vscale x 14 x i1> @llvm.experimental.vector.extract.nxv14i1.nxv28i1(<vscale x 28 x i1> %in, i64 14)
   ret <vscale x 14 x i1> %res
 }
 
 declare <vscale x 14 x i1> @llvm.experimental.vector.extract.nxv14i1.nxv28i1(<vscale x 28 x i1>, i64)
 
 ;

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -aarch64-sve-vector-bits-min=128  < %s | FileCheck %s -D#VBYTES=16 -check-prefix=NO_SVE
 ; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32
 ; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32
 ; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
 ; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256 -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
 
 target triple = "aarch64-unknown-linux-gnu"
 
 ; Don't use SVE when its registers are no bigger than NEON.
 ; NO_SVE-NOT: ptrue
 
 ; Don't use SVE for 64-bit vectors.
 define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v4f16:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w0, #0x1
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    dup v2.4h, w8
+; NO_SVE-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v4f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    csetm w8, ne
 ; CHECK-NEXT:    dup v2.4h, w8
 ; CHECK-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-NEXT:    ret
   %sel = select i1 %mask, <4 x half> %op1, <4 x half> %op2
   ret <4 x half> %sel
 }
 
 ; Don't use SVE for 128-bit vectors.
 define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v8f16:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w0, #0x1
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    dup v2.8h, w8
+; NO_SVE-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v8f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    csetm w8, ne
 ; CHECK-NEXT:    dup v2.8h, w8
 ; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    ret
   %sel = select i1 %mask, <8 x half> %op1, <8 x half> %op2
   ret <8 x half> %sel
 }
 
 define void @select_v16f16(<16 x half>* %a, <16 x half>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v16f16:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0, #16]
+; NO_SVE-NEXT:    ldr q2, [x1]
+; NO_SVE-NEXT:    ldr q3, [x1, #16]
+; NO_SVE-NEXT:    dup v4.8h, w8
+; NO_SVE-NEXT:    bif v0.16b, v2.16b, v4.16b
+; NO_SVE-NEXT:    bif v1.16b, v3.16b, v4.16b
+; NO_SVE-NEXT:    stp q0, q1, [x0]
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w2, #0x1
 ; CHECK-NEXT:    ptrue p0.h, vl16
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    ptrue p1.h
 ; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    and z2.h, z2.h, #0x1
 ; CHECK-NEXT:    cmpne p1.h, p1/z, z2.h, #0
 ; CHECK-NEXT:    sel z0.h, p1, z0.h, z1.h
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load volatile <16 x half>, <16 x half>* %a
   %op2 = load volatile <16 x half>, <16 x half>* %b
   %sel = select i1 %mask, <16 x half> %op1, <16 x half> %op2
   store <16 x half> %sel, <16 x half>* %a
   ret void
 }
 
 define void @select_v32f16(<32 x half>* %a, <32 x half>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v32f16:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #48]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0]
+; NO_SVE-NEXT:    ldr q2, [x0, #16]
+; NO_SVE-NEXT:    ldr q3, [x0, #32]
+; NO_SVE-NEXT:    ldr q4, [x1, #48]
+; NO_SVE-NEXT:    dup v6.8h, w8
+; NO_SVE-NEXT:    ldr q5, [x1]
+; NO_SVE-NEXT:    ldr q7, [x1, #16]
+; NO_SVE-NEXT:    ldr q16, [x1, #32]
+; NO_SVE-NEXT:    bif v1.16b, v5.16b, v6.16b
+; NO_SVE-NEXT:    bif v2.16b, v7.16b, v6.16b
+; NO_SVE-NEXT:    bif v0.16b, v4.16b, v6.16b
+; NO_SVE-NEXT:    bif v3.16b, v16.16b, v6.16b
+; NO_SVE-NEXT:    stp q1, q2, [x0]
+; NO_SVE-NEXT:    stp q3, q0, [x0, #32]
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_512-LABEL: select_v32f16:
 ; VBITS_GE_512:       // %bb.0:
 ; VBITS_GE_512-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
 ; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    ptrue p1.h
 ; VBITS_GE_512-NEXT:    mov z2.h, w8
 ; VBITS_GE_512-NEXT:    and z2.h, z2.h, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.h, p1/z, z2.h, #0
 ; VBITS_GE_512-NEXT:    sel z0.h, p1, z0.h, z1.h
 ; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_512-NEXT:    ret
   %op1 = load volatile <32 x half>, <32 x half>* %a
   %op2 = load volatile <32 x half>, <32 x half>* %b
   %sel = select i1 %mask, <32 x half> %op1, <32 x half> %op2
   store <32 x half> %sel, <32 x half>* %a
   ret void
 }
 
 define void @select_v64f16(<64 x half>* %a, <64 x half>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v64f16:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #16]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0]
+; NO_SVE-NEXT:    ldr q2, [x0, #48]
+; NO_SVE-NEXT:    ldr q3, [x0, #32]
+; NO_SVE-NEXT:    ldr q4, [x0, #80]
+; NO_SVE-NEXT:    dup v21.8h, w8
+; NO_SVE-NEXT:    ldr q5, [x0, #64]
+; NO_SVE-NEXT:    ldr q6, [x0, #112]
+; NO_SVE-NEXT:    ldr q7, [x0, #96]
+; NO_SVE-NEXT:    ldr q16, [x1, #16]
+; NO_SVE-NEXT:    ldr q17, [x1]
+; NO_SVE-NEXT:    ldr q18, [x1, #48]
+; NO_SVE-NEXT:    ldr q19, [x1, #32]
+; NO_SVE-NEXT:    bif v0.16b, v16.16b, v21.16b
+; NO_SVE-NEXT:    ldr q20, [x1, #80]
+; NO_SVE-NEXT:    bif v1.16b, v17.16b, v21.16b
+; NO_SVE-NEXT:    ldr q16, [x1, #64]
+; NO_SVE-NEXT:    bif v2.16b, v18.16b, v21.16b
+; NO_SVE-NEXT:    ldr q17, [x1, #112]
+; NO_SVE-NEXT:    bif v3.16b, v19.16b, v21.16b
+; NO_SVE-NEXT:    ldr q18, [x1, #96]
+; NO_SVE-NEXT:    bif v4.16b, v20.16b, v21.16b
+; NO_SVE-NEXT:    stp q1, q0, [x0]
+; NO_SVE-NEXT:    mov v0.16b, v21.16b
+; NO_SVE-NEXT:    mov v1.16b, v21.16b
+; NO_SVE-NEXT:    stp q3, q2, [x0, #32]
+; NO_SVE-NEXT:    mov v2.16b, v21.16b
+; NO_SVE-NEXT:    bsl v0.16b, v5.16b, v16.16b
+; NO_SVE-NEXT:    bsl v1.16b, v6.16b, v17.16b
+; NO_SVE-NEXT:    bsl v2.16b, v7.16b, v18.16b
+; NO_SVE-NEXT:    stp q0, q4, [x0, #64]
+; NO_SVE-NEXT:    stp q2, q1, [x0, #96]
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_1024-LABEL: select_v64f16:
 ; VBITS_GE_1024:       // %bb.0:
 ; VBITS_GE_1024-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_1024-NEXT:    ptrue p0.h, vl64
 ; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    ptrue p1.h
 ; VBITS_GE_1024-NEXT:    mov z2.h, w8
 ; VBITS_GE_1024-NEXT:    and z2.h, z2.h, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.h, p1/z, z2.h, #0
 ; VBITS_GE_1024-NEXT:    sel z0.h, p1, z0.h, z1.h
 ; VBITS_GE_1024-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    ret
   %op1 = load volatile <64 x half>, <64 x half>* %a
   %op2 = load volatile <64 x half>, <64 x half>* %b
   %sel = select i1 %mask, <64 x half> %op1, <64 x half> %op2
   store <64 x half> %sel, <64 x half>* %a
   ret void
 }
 
 define void @select_v128f16(<128 x half>* %a, <128 x half>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v128f16:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    stp d11, d10, [sp, #-32]! // 16-byte Folded Spill
+; NO_SVE-NEXT:    .cfi_def_cfa_offset 32
+; NO_SVE-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
+; NO_SVE-NEXT:    .cfi_offset b8, -8
+; NO_SVE-NEXT:    .cfi_offset b9, -16
+; NO_SVE-NEXT:    .cfi_offset b10, -24
+; NO_SVE-NEXT:    .cfi_offset b11, -32
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #240]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0, #224]
+; NO_SVE-NEXT:    ldr q2, [x0, #208]
+; NO_SVE-NEXT:    ldr q3, [x0, #192]
+; NO_SVE-NEXT:    ldr q4, [x0, #176]
+; NO_SVE-NEXT:    dup v8.8h, w8
+; NO_SVE-NEXT:    ldr q5, [x0, #160]
+; NO_SVE-NEXT:    ldr q6, [x0, #144]
+; NO_SVE-NEXT:    ldr q7, [x0, #128]
+; NO_SVE-NEXT:    ldr q16, [x0, #112]
+; NO_SVE-NEXT:    ldr q17, [x0, #96]
+; NO_SVE-NEXT:    ldr q18, [x0, #80]
+; NO_SVE-NEXT:    ldr q19, [x0, #64]
+; NO_SVE-NEXT:    ldr q20, [x0, #48]
+; NO_SVE-NEXT:    ldr q21, [x0, #32]
+; NO_SVE-NEXT:    ldr q22, [x0, #16]
+; NO_SVE-NEXT:    ldr q23, [x0]
+; NO_SVE-NEXT:    ldr q24, [x1, #240]
+; NO_SVE-NEXT:    ldr q25, [x1, #224]
+; NO_SVE-NEXT:    ldr q26, [x1, #208]
+; NO_SVE-NEXT:    ldr q27, [x1, #192]
+; NO_SVE-NEXT:    bif v0.16b, v24.16b, v8.16b
+; NO_SVE-NEXT:    ldr q28, [x1, #176]
+; NO_SVE-NEXT:    bif v1.16b, v25.16b, v8.16b
+; NO_SVE-NEXT:    ldr q29, [x1, #160]
+; NO_SVE-NEXT:    bif v2.16b, v26.16b, v8.16b
+; NO_SVE-NEXT:    ldr q30, [x1, #144]
+; NO_SVE-NEXT:    bif v3.16b, v27.16b, v8.16b
+; NO_SVE-NEXT:    ldr q31, [x1, #128]
+; NO_SVE-NEXT:    ldr q9, [x1, #112]
+; NO_SVE-NEXT:    ldr q10, [x1, #96]
+; NO_SVE-NEXT:    bif v4.16b, v28.16b, v8.16b
+; NO_SVE-NEXT:    ldr q28, [x1, #80]
+; NO_SVE-NEXT:    ldr q24, [x1, #64]
+; NO_SVE-NEXT:    ldr q25, [x1, #48]
+; NO_SVE-NEXT:    ldr q26, [x1, #32]
+; NO_SVE-NEXT:    ldr q27, [x1, #16]
+; NO_SVE-NEXT:    ldr q11, [x1]
+; NO_SVE-NEXT:    stp q1, q0, [x0, #224]
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    stp q3, q2, [x0, #192]
+; NO_SVE-NEXT:    mov v1.16b, v8.16b
+; NO_SVE-NEXT:    mov v2.16b, v8.16b
+; NO_SVE-NEXT:    bsl v0.16b, v5.16b, v29.16b
+; NO_SVE-NEXT:    bsl v1.16b, v6.16b, v30.16b
+; NO_SVE-NEXT:    bsl v2.16b, v7.16b, v31.16b
+; NO_SVE-NEXT:    mov v3.16b, v8.16b
+; NO_SVE-NEXT:    stp q0, q4, [x0, #160]
+; NO_SVE-NEXT:    mov v4.16b, v8.16b
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    stp q2, q1, [x0, #128]
+; NO_SVE-NEXT:    mov v1.16b, v8.16b
+; NO_SVE-NEXT:    bsl v3.16b, v16.16b, v9.16b
+; NO_SVE-NEXT:    bsl v4.16b, v17.16b, v10.16b
+; NO_SVE-NEXT:    bsl v0.16b, v18.16b, v28.16b
+; NO_SVE-NEXT:    bsl v1.16b, v19.16b, v24.16b
+; NO_SVE-NEXT:    mov v2.16b, v8.16b
+; NO_SVE-NEXT:    stp q4, q3, [x0, #96]
+; NO_SVE-NEXT:    mov v3.16b, v8.16b
+; NO_SVE-NEXT:    mov v4.16b, v8.16b
+; NO_SVE-NEXT:    stp q1, q0, [x0, #64]
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    bsl v2.16b, v20.16b, v25.16b
+; NO_SVE-NEXT:    bsl v3.16b, v21.16b, v26.16b
+; NO_SVE-NEXT:    bsl v4.16b, v22.16b, v27.16b
+; NO_SVE-NEXT:    bsl v0.16b, v23.16b, v11.16b
+; NO_SVE-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
+; NO_SVE-NEXT:    stp q3, q2, [x0, #32]
+; NO_SVE-NEXT:    stp q0, q4, [x0]
+; NO_SVE-NEXT:    ldp d11, d10, [sp], #32 // 16-byte Folded Reload
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_2048-LABEL: select_v128f16:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_2048-NEXT:    ptrue p0.h, vl128
 ; VBITS_GE_2048-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    ptrue p1.h
 ; VBITS_GE_2048-NEXT:    mov z2.h, w8
 ; VBITS_GE_2048-NEXT:    and z2.h, z2.h, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.h, p1/z, z2.h, #0
 ; VBITS_GE_2048-NEXT:    sel z0.h, p1, z0.h, z1.h
 ; VBITS_GE_2048-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    ret
   %op1 = load volatile <128 x half>, <128 x half>* %a
   %op2 = load volatile <128 x half>, <128 x half>* %b
   %sel = select i1 %mask, <128 x half> %op1, <128 x half> %op2
   store <128 x half> %sel, <128 x half>* %a
   ret void
 }
 
 ; Don't use SVE for 64-bit vectors.
 define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v2f32:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w0, #0x1
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    dup v2.2s, w8
+; NO_SVE-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    csetm w8, ne
 ; CHECK-NEXT:    dup v2.2s, w8
 ; CHECK-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-NEXT:    ret
   %sel = select i1 %mask, <2 x float> %op1, <2 x float> %op2
   ret <2 x float> %sel
 }
 
 ; Don't use SVE for 128-bit vectors.
 define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v4f32:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w0, #0x1
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    dup v2.4s, w8
+; NO_SVE-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v4f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    csetm w8, ne
 ; CHECK-NEXT:    dup v2.4s, w8
 ; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    ret
   %sel = select i1 %mask, <4 x float> %op1, <4 x float> %op2
   ret <4 x float> %sel
 }
 
 define void @select_v8f32(<8 x float>* %a, <8 x float>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v8f32:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0, #16]
+; NO_SVE-NEXT:    ldr q2, [x1]
+; NO_SVE-NEXT:    ldr q3, [x1, #16]
+; NO_SVE-NEXT:    dup v4.4s, w8
+; NO_SVE-NEXT:    bif v0.16b, v2.16b, v4.16b
+; NO_SVE-NEXT:    bif v1.16b, v3.16b, v4.16b
+; NO_SVE-NEXT:    stp q0, q1, [x0]
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v8f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w2, #0x1
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; CHECK-NEXT:    ptrue p1.s
 ; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    and z2.s, z2.s, #0x1
 ; CHECK-NEXT:    cmpne p1.s, p1/z, z2.s, #0
 ; CHECK-NEXT:    sel z0.s, p1, z0.s, z1.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load volatile <8 x float>, <8 x float>* %a
   %op2 = load volatile <8 x float>, <8 x float>* %b
   %sel = select i1 %mask, <8 x float> %op1, <8 x float> %op2
   store <8 x float> %sel, <8 x float>* %a
   ret void
 }
 
 define void @select_v16f32(<16 x float>* %a, <16 x float>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v16f32:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #48]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0]
+; NO_SVE-NEXT:    ldr q2, [x0, #16]
+; NO_SVE-NEXT:    ldr q3, [x0, #32]
+; NO_SVE-NEXT:    ldr q4, [x1, #48]
+; NO_SVE-NEXT:    dup v6.4s, w8
+; NO_SVE-NEXT:    ldr q5, [x1]
+; NO_SVE-NEXT:    ldr q7, [x1, #16]
+; NO_SVE-NEXT:    ldr q16, [x1, #32]
+; NO_SVE-NEXT:    bif v1.16b, v5.16b, v6.16b
+; NO_SVE-NEXT:    bif v2.16b, v7.16b, v6.16b
+; NO_SVE-NEXT:    bif v0.16b, v4.16b, v6.16b
+; NO_SVE-NEXT:    bif v3.16b, v16.16b, v6.16b
+; NO_SVE-NEXT:    stp q1, q2, [x0]
+; NO_SVE-NEXT:    stp q3, q0, [x0, #32]
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_512-LABEL: select_v16f32:
 ; VBITS_GE_512:       // %bb.0:
 ; VBITS_GE_512-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
 ; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    ptrue p1.s
 ; VBITS_GE_512-NEXT:    mov z2.s, w8
 ; VBITS_GE_512-NEXT:    and z2.s, z2.s, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.s, p1/z, z2.s, #0
 ; VBITS_GE_512-NEXT:    sel z0.s, p1, z0.s, z1.s
 ; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_512-NEXT:    ret
   %op1 = load volatile <16 x float>, <16 x float>* %a
   %op2 = load volatile <16 x float>, <16 x float>* %b
   %sel = select i1 %mask, <16 x float> %op1, <16 x float> %op2
   store <16 x float> %sel, <16 x float>* %a
   ret void
 }
 
 define void @select_v32f32(<32 x float>* %a, <32 x float>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v32f32:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #16]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0]
+; NO_SVE-NEXT:    ldr q2, [x0, #48]
+; NO_SVE-NEXT:    ldr q3, [x0, #32]
+; NO_SVE-NEXT:    ldr q4, [x0, #80]
+; NO_SVE-NEXT:    dup v21.4s, w8
+; NO_SVE-NEXT:    ldr q5, [x0, #64]
+; NO_SVE-NEXT:    ldr q6, [x0, #112]
+; NO_SVE-NEXT:    ldr q7, [x0, #96]
+; NO_SVE-NEXT:    ldr q16, [x1, #16]
+; NO_SVE-NEXT:    ldr q17, [x1]
+; NO_SVE-NEXT:    ldr q18, [x1, #48]
+; NO_SVE-NEXT:    ldr q19, [x1, #32]
+; NO_SVE-NEXT:    bif v0.16b, v16.16b, v21.16b
+; NO_SVE-NEXT:    ldr q20, [x1, #80]
+; NO_SVE-NEXT:    bif v1.16b, v17.16b, v21.16b
+; NO_SVE-NEXT:    ldr q16, [x1, #64]
+; NO_SVE-NEXT:    bif v2.16b, v18.16b, v21.16b
+; NO_SVE-NEXT:    ldr q17, [x1, #112]
+; NO_SVE-NEXT:    bif v3.16b, v19.16b, v21.16b
+; NO_SVE-NEXT:    ldr q18, [x1, #96]
+; NO_SVE-NEXT:    bif v4.16b, v20.16b, v21.16b
+; NO_SVE-NEXT:    stp q1, q0, [x0]
+; NO_SVE-NEXT:    mov v0.16b, v21.16b
+; NO_SVE-NEXT:    mov v1.16b, v21.16b
+; NO_SVE-NEXT:    stp q3, q2, [x0, #32]
+; NO_SVE-NEXT:    mov v2.16b, v21.16b
+; NO_SVE-NEXT:    bsl v0.16b, v5.16b, v16.16b
+; NO_SVE-NEXT:    bsl v1.16b, v6.16b, v17.16b
+; NO_SVE-NEXT:    bsl v2.16b, v7.16b, v18.16b
+; NO_SVE-NEXT:    stp q0, q4, [x0, #64]
+; NO_SVE-NEXT:    stp q2, q1, [x0, #96]
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_1024-LABEL: select_v32f32:
 ; VBITS_GE_1024:       // %bb.0:
 ; VBITS_GE_1024-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_1024-NEXT:    ptrue p0.s, vl32
 ; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    ptrue p1.s
 ; VBITS_GE_1024-NEXT:    mov z2.s, w8
 ; VBITS_GE_1024-NEXT:    and z2.s, z2.s, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.s, p1/z, z2.s, #0
 ; VBITS_GE_1024-NEXT:    sel z0.s, p1, z0.s, z1.s
 ; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    ret
   %op1 = load volatile <32 x float>, <32 x float>* %a
   %op2 = load volatile <32 x float>, <32 x float>* %b
   %sel = select i1 %mask, <32 x float> %op1, <32 x float> %op2
   store <32 x float> %sel, <32 x float>* %a
   ret void
 }
 
 define void @select_v64f32(<64 x float>* %a, <64 x float>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v64f32:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    stp d11, d10, [sp, #-32]! // 16-byte Folded Spill
+; NO_SVE-NEXT:    .cfi_def_cfa_offset 32
+; NO_SVE-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
+; NO_SVE-NEXT:    .cfi_offset b8, -8
+; NO_SVE-NEXT:    .cfi_offset b9, -16
+; NO_SVE-NEXT:    .cfi_offset b10, -24
+; NO_SVE-NEXT:    .cfi_offset b11, -32
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #240]
+; NO_SVE-NEXT:    csetm w8, ne
+; NO_SVE-NEXT:    ldr q1, [x0, #224]
+; NO_SVE-NEXT:    ldr q2, [x0, #208]
+; NO_SVE-NEXT:    ldr q3, [x0, #192]
+; NO_SVE-NEXT:    ldr q4, [x0, #176]
+; NO_SVE-NEXT:    dup v8.4s, w8
+; NO_SVE-NEXT:    ldr q5, [x0, #160]
+; NO_SVE-NEXT:    ldr q6, [x0, #144]
+; NO_SVE-NEXT:    ldr q7, [x0, #128]
+; NO_SVE-NEXT:    ldr q16, [x0, #112]
+; NO_SVE-NEXT:    ldr q17, [x0, #96]
+; NO_SVE-NEXT:    ldr q18, [x0, #80]
+; NO_SVE-NEXT:    ldr q19, [x0, #64]
+; NO_SVE-NEXT:    ldr q20, [x0, #48]
+; NO_SVE-NEXT:    ldr q21, [x0, #32]
+; NO_SVE-NEXT:    ldr q22, [x0, #16]
+; NO_SVE-NEXT:    ldr q23, [x0]
+; NO_SVE-NEXT:    ldr q24, [x1, #240]
+; NO_SVE-NEXT:    ldr q25, [x1, #224]
+; NO_SVE-NEXT:    ldr q26, [x1, #208]
+; NO_SVE-NEXT:    ldr q27, [x1, #192]
+; NO_SVE-NEXT:    bif v0.16b, v24.16b, v8.16b
+; NO_SVE-NEXT:    ldr q28, [x1, #176]
+; NO_SVE-NEXT:    bif v1.16b, v25.16b, v8.16b
+; NO_SVE-NEXT:    ldr q29, [x1, #160]
+; NO_SVE-NEXT:    bif v2.16b, v26.16b, v8.16b
+; NO_SVE-NEXT:    ldr q30, [x1, #144]
+; NO_SVE-NEXT:    bif v3.16b, v27.16b, v8.16b
+; NO_SVE-NEXT:    ldr q31, [x1, #128]
+; NO_SVE-NEXT:    ldr q9, [x1, #112]
+; NO_SVE-NEXT:    ldr q10, [x1, #96]
+; NO_SVE-NEXT:    bif v4.16b, v28.16b, v8.16b
+; NO_SVE-NEXT:    ldr q28, [x1, #80]
+; NO_SVE-NEXT:    ldr q24, [x1, #64]
+; NO_SVE-NEXT:    ldr q25, [x1, #48]
+; NO_SVE-NEXT:    ldr q26, [x1, #32]
+; NO_SVE-NEXT:    ldr q27, [x1, #16]
+; NO_SVE-NEXT:    ldr q11, [x1]
+; NO_SVE-NEXT:    stp q1, q0, [x0, #224]
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    stp q3, q2, [x0, #192]
+; NO_SVE-NEXT:    mov v1.16b, v8.16b
+; NO_SVE-NEXT:    mov v2.16b, v8.16b
+; NO_SVE-NEXT:    bsl v0.16b, v5.16b, v29.16b
+; NO_SVE-NEXT:    bsl v1.16b, v6.16b, v30.16b
+; NO_SVE-NEXT:    bsl v2.16b, v7.16b, v31.16b
+; NO_SVE-NEXT:    mov v3.16b, v8.16b
+; NO_SVE-NEXT:    stp q0, q4, [x0, #160]
+; NO_SVE-NEXT:    mov v4.16b, v8.16b
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    stp q2, q1, [x0, #128]
+; NO_SVE-NEXT:    mov v1.16b, v8.16b
+; NO_SVE-NEXT:    bsl v3.16b, v16.16b, v9.16b
+; NO_SVE-NEXT:    bsl v4.16b, v17.16b, v10.16b
+; NO_SVE-NEXT:    bsl v0.16b, v18.16b, v28.16b
+; NO_SVE-NEXT:    bsl v1.16b, v19.16b, v24.16b
+; NO_SVE-NEXT:    mov v2.16b, v8.16b
+; NO_SVE-NEXT:    stp q4, q3, [x0, #96]
+; NO_SVE-NEXT:    mov v3.16b, v8.16b
+; NO_SVE-NEXT:    mov v4.16b, v8.16b
+; NO_SVE-NEXT:    stp q1, q0, [x0, #64]
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    bsl v2.16b, v20.16b, v25.16b
+; NO_SVE-NEXT:    bsl v3.16b, v21.16b, v26.16b
+; NO_SVE-NEXT:    bsl v4.16b, v22.16b, v27.16b
+; NO_SVE-NEXT:    bsl v0.16b, v23.16b, v11.16b
+; NO_SVE-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
+; NO_SVE-NEXT:    stp q3, q2, [x0, #32]
+; NO_SVE-NEXT:    stp q0, q4, [x0]
+; NO_SVE-NEXT:    ldp d11, d10, [sp], #32 // 16-byte Folded Reload
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_2048-LABEL: select_v64f32:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_2048-NEXT:    ptrue p0.s, vl64
 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1w { z1.s }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    ptrue p1.s
 ; VBITS_GE_2048-NEXT:    mov z2.s, w8
 ; VBITS_GE_2048-NEXT:    and z2.s, z2.s, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.s, p1/z, z2.s, #0
 ; VBITS_GE_2048-NEXT:    sel z0.s, p1, z0.s, z1.s
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    ret
   %op1 = load volatile <64 x float>, <64 x float>* %a
   %op2 = load volatile <64 x float>, <64 x float>* %b
   %sel = select i1 %mask, <64 x float> %op1, <64 x float> %op2
   store <64 x float> %sel, <64 x float>* %a
   ret void
 }
 
 ; Don't use SVE for 64-bit vectors.
 define <1 x double> @select_v1f64(<1 x double> %op1, <1 x double> %op2, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v1f64:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w0, #0x1
+; NO_SVE-NEXT:    csetm x8, ne
+; NO_SVE-NEXT:    fmov d2, x8
+; NO_SVE-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    csetm x8, ne
 ; CHECK-NEXT:    fmov d2, x8
 ; CHECK-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-NEXT:    ret
   %sel = select i1 %mask, <1 x double> %op1, <1 x double> %op2
   ret <1 x double> %sel
 }
 
 ; Don't use SVE for 128-bit vectors.
 define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v2f64:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w0, #0x1
+; NO_SVE-NEXT:    csetm x8, ne
+; NO_SVE-NEXT:    dup v2.2d, x8
+; NO_SVE-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v2f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    csetm x8, ne
 ; CHECK-NEXT:    dup v2.2d, x8
 ; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    ret
   %sel = select i1 %mask, <2 x double> %op1, <2 x double> %op2
   ret <2 x double> %sel
 }
 
 define void @select_v4f64(<4 x double>* %a, <4 x double>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v4f64:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0]
+; NO_SVE-NEXT:    csetm x8, ne
+; NO_SVE-NEXT:    ldr q1, [x0, #16]
+; NO_SVE-NEXT:    ldr q2, [x1]
+; NO_SVE-NEXT:    ldr q3, [x1, #16]
+; NO_SVE-NEXT:    dup v4.2d, x8
+; NO_SVE-NEXT:    bif v0.16b, v2.16b, v4.16b
+; NO_SVE-NEXT:    bif v1.16b, v3.16b, v4.16b
+; NO_SVE-NEXT:    stp q0, q1, [x0]
+; NO_SVE-NEXT:    ret
+;
 ; CHECK-LABEL: select_v4f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w2, #0x1
 ; CHECK-NEXT:    ptrue p0.d, vl4
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; CHECK-NEXT:    ptrue p1.d
 ; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    and z2.d, z2.d, #0x1
 ; CHECK-NEXT:    cmpne p1.d, p1/z, z2.d, #0
 ; CHECK-NEXT:    sel z0.d, p1, z0.d, z1.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load volatile <4 x double>, <4 x double>* %a
   %op2 = load volatile <4 x double>, <4 x double>* %b
   %sel = select i1 %mask, <4 x double> %op1, <4 x double> %op2
   store <4 x double> %sel, <4 x double>* %a
   ret void
 }
 
 define void @select_v8f64(<8 x double>* %a, <8 x double>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v8f64:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #48]
+; NO_SVE-NEXT:    csetm x8, ne
+; NO_SVE-NEXT:    ldr q1, [x0]
+; NO_SVE-NEXT:    ldr q2, [x0, #16]
+; NO_SVE-NEXT:    ldr q3, [x0, #32]
+; NO_SVE-NEXT:    ldr q4, [x1, #48]
+; NO_SVE-NEXT:    dup v6.2d, x8
+; NO_SVE-NEXT:    ldr q5, [x1]
+; NO_SVE-NEXT:    ldr q7, [x1, #16]
+; NO_SVE-NEXT:    ldr q16, [x1, #32]
+; NO_SVE-NEXT:    bif v1.16b, v5.16b, v6.16b
+; NO_SVE-NEXT:    bif v2.16b, v7.16b, v6.16b
+; NO_SVE-NEXT:    bif v0.16b, v4.16b, v6.16b
+; NO_SVE-NEXT:    bif v3.16b, v16.16b, v6.16b
+; NO_SVE-NEXT:    stp q1, q2, [x0]
+; NO_SVE-NEXT:    stp q3, q0, [x0, #32]
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_512-LABEL: select_v8f64:
 ; VBITS_GE_512:       // %bb.0:
 ; VBITS_GE_512-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
 ; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    ptrue p1.d
 ; VBITS_GE_512-NEXT:    mov z2.d, x8
 ; VBITS_GE_512-NEXT:    and z2.d, z2.d, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.d, p1/z, z2.d, #0
 ; VBITS_GE_512-NEXT:    sel z0.d, p1, z0.d, z1.d
 ; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_512-NEXT:    ret
   %op1 = load volatile <8 x double>, <8 x double>* %a
   %op2 = load volatile <8 x double>, <8 x double>* %b
   %sel = select i1 %mask, <8 x double> %op1, <8 x double> %op2
   store <8 x double> %sel, <8 x double>* %a
   ret void
 }
 
 define void @select_v16f64(<16 x double>* %a, <16 x double>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v16f64:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #16]
+; NO_SVE-NEXT:    csetm x8, ne
+; NO_SVE-NEXT:    ldr q1, [x0]
+; NO_SVE-NEXT:    ldr q2, [x0, #48]
+; NO_SVE-NEXT:    ldr q3, [x0, #32]
+; NO_SVE-NEXT:    ldr q4, [x0, #80]
+; NO_SVE-NEXT:    dup v21.2d, x8
+; NO_SVE-NEXT:    ldr q5, [x0, #64]
+; NO_SVE-NEXT:    ldr q6, [x0, #112]
+; NO_SVE-NEXT:    ldr q7, [x0, #96]
+; NO_SVE-NEXT:    ldr q16, [x1, #16]
+; NO_SVE-NEXT:    ldr q17, [x1]
+; NO_SVE-NEXT:    ldr q18, [x1, #48]
+; NO_SVE-NEXT:    ldr q19, [x1, #32]
+; NO_SVE-NEXT:    bif v0.16b, v16.16b, v21.16b
+; NO_SVE-NEXT:    ldr q20, [x1, #80]
+; NO_SVE-NEXT:    bif v1.16b, v17.16b, v21.16b
+; NO_SVE-NEXT:    ldr q16, [x1, #64]
+; NO_SVE-NEXT:    bif v2.16b, v18.16b, v21.16b
+; NO_SVE-NEXT:    ldr q17, [x1, #112]
+; NO_SVE-NEXT:    bif v3.16b, v19.16b, v21.16b
+; NO_SVE-NEXT:    ldr q18, [x1, #96]
+; NO_SVE-NEXT:    bif v4.16b, v20.16b, v21.16b
+; NO_SVE-NEXT:    stp q1, q0, [x0]
+; NO_SVE-NEXT:    mov v0.16b, v21.16b
+; NO_SVE-NEXT:    mov v1.16b, v21.16b
+; NO_SVE-NEXT:    stp q3, q2, [x0, #32]
+; NO_SVE-NEXT:    mov v2.16b, v21.16b
+; NO_SVE-NEXT:    bsl v0.16b, v5.16b, v16.16b
+; NO_SVE-NEXT:    bsl v1.16b, v6.16b, v17.16b
+; NO_SVE-NEXT:    bsl v2.16b, v7.16b, v18.16b
+; NO_SVE-NEXT:    stp q0, q4, [x0, #64]
+; NO_SVE-NEXT:    stp q2, q1, [x0, #96]
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_1024-LABEL: select_v16f64:
 ; VBITS_GE_1024:       // %bb.0:
 ; VBITS_GE_1024-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_1024-NEXT:    ptrue p0.d, vl16
 ; VBITS_GE_1024-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    ptrue p1.d
 ; VBITS_GE_1024-NEXT:    mov z2.d, x8
 ; VBITS_GE_1024-NEXT:    and z2.d, z2.d, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.d, p1/z, z2.d, #0
 ; VBITS_GE_1024-NEXT:    sel z0.d, p1, z0.d, z1.d
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    ret
   %op1 = load volatile <16 x double>, <16 x double>* %a
   %op2 = load volatile <16 x double>, <16 x double>* %b
   %sel = select i1 %mask, <16 x double> %op1, <16 x double> %op2
   store <16 x double> %sel, <16 x double>* %a
   ret void
 }
 
 define void @select_v32f64(<32 x double>* %a, <32 x double>* %b, i1 %mask) #0 {
+; NO_SVE-LABEL: select_v32f64:
+; NO_SVE:       // %bb.0:
+; NO_SVE-NEXT:    stp d11, d10, [sp, #-32]! // 16-byte Folded Spill
+; NO_SVE-NEXT:    .cfi_def_cfa_offset 32
+; NO_SVE-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
+; NO_SVE-NEXT:    .cfi_offset b8, -8
+; NO_SVE-NEXT:    .cfi_offset b9, -16
+; NO_SVE-NEXT:    .cfi_offset b10, -24
+; NO_SVE-NEXT:    .cfi_offset b11, -32
+; NO_SVE-NEXT:    tst w2, #0x1
+; NO_SVE-NEXT:    ldr q0, [x0, #240]
+; NO_SVE-NEXT:    csetm x8, ne
+; NO_SVE-NEXT:    ldr q1, [x0, #224]
+; NO_SVE-NEXT:    ldr q2, [x0, #208]
+; NO_SVE-NEXT:    ldr q3, [x0, #192]
+; NO_SVE-NEXT:    ldr q4, [x0, #176]
+; NO_SVE-NEXT:    dup v8.2d, x8
+; NO_SVE-NEXT:    ldr q5, [x0, #160]
+; NO_SVE-NEXT:    ldr q6, [x0, #144]
+; NO_SVE-NEXT:    ldr q7, [x0, #128]
+; NO_SVE-NEXT:    ldr q16, [x0, #112]
+; NO_SVE-NEXT:    ldr q17, [x0, #96]
+; NO_SVE-NEXT:    ldr q18, [x0, #80]
+; NO_SVE-NEXT:    ldr q19, [x0, #64]
+; NO_SVE-NEXT:    ldr q20, [x0, #48]
+; NO_SVE-NEXT:    ldr q21, [x0, #32]
+; NO_SVE-NEXT:    ldr q22, [x0, #16]
+; NO_SVE-NEXT:    ldr q23, [x0]
+; NO_SVE-NEXT:    ldr q24, [x1, #240]
+; NO_SVE-NEXT:    ldr q25, [x1, #224]
+; NO_SVE-NEXT:    ldr q26, [x1, #208]
+; NO_SVE-NEXT:    ldr q27, [x1, #192]
+; NO_SVE-NEXT:    bif v0.16b, v24.16b, v8.16b
+; NO_SVE-NEXT:    ldr q28, [x1, #176]
+; NO_SVE-NEXT:    bif v1.16b, v25.16b, v8.16b
+; NO_SVE-NEXT:    ldr q29, [x1, #160]
+; NO_SVE-NEXT:    bif v2.16b, v26.16b, v8.16b
+; NO_SVE-NEXT:    ldr q30, [x1, #144]
+; NO_SVE-NEXT:    bif v3.16b, v27.16b, v8.16b
+; NO_SVE-NEXT:    ldr q31, [x1, #128]
+; NO_SVE-NEXT:    ldr q9, [x1, #112]
+; NO_SVE-NEXT:    ldr q10, [x1, #96]
+; NO_SVE-NEXT:    bif v4.16b, v28.16b, v8.16b
+; NO_SVE-NEXT:    ldr q28, [x1, #80]
+; NO_SVE-NEXT:    ldr q24, [x1, #64]
+; NO_SVE-NEXT:    ldr q25, [x1, #48]
+; NO_SVE-NEXT:    ldr q26, [x1, #32]
+; NO_SVE-NEXT:    ldr q27, [x1, #16]
+; NO_SVE-NEXT:    ldr q11, [x1]
+; NO_SVE-NEXT:    stp q1, q0, [x0, #224]
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    stp q3, q2, [x0, #192]
+; NO_SVE-NEXT:    mov v1.16b, v8.16b
+; NO_SVE-NEXT:    mov v2.16b, v8.16b
+; NO_SVE-NEXT:    bsl v0.16b, v5.16b, v29.16b
+; NO_SVE-NEXT:    bsl v1.16b, v6.16b, v30.16b
+; NO_SVE-NEXT:    bsl v2.16b, v7.16b, v31.16b
+; NO_SVE-NEXT:    mov v3.16b, v8.16b
+; NO_SVE-NEXT:    stp q0, q4, [x0, #160]
+; NO_SVE-NEXT:    mov v4.16b, v8.16b
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    stp q2, q1, [x0, #128]
+; NO_SVE-NEXT:    mov v1.16b, v8.16b
+; NO_SVE-NEXT:    bsl v3.16b, v16.16b, v9.16b
+; NO_SVE-NEXT:    bsl v4.16b, v17.16b, v10.16b
+; NO_SVE-NEXT:    bsl v0.16b, v18.16b, v28.16b
+; NO_SVE-NEXT:    bsl v1.16b, v19.16b, v24.16b
+; NO_SVE-NEXT:    mov v2.16b, v8.16b
+; NO_SVE-NEXT:    stp q4, q3, [x0, #96]
+; NO_SVE-NEXT:    mov v3.16b, v8.16b
+; NO_SVE-NEXT:    mov v4.16b, v8.16b
+; NO_SVE-NEXT:    stp q1, q0, [x0, #64]
+; NO_SVE-NEXT:    mov v0.16b, v8.16b
+; NO_SVE-NEXT:    bsl v2.16b, v20.16b, v25.16b
+; NO_SVE-NEXT:    bsl v3.16b, v21.16b, v26.16b
+; NO_SVE-NEXT:    bsl v4.16b, v22.16b, v27.16b
+; NO_SVE-NEXT:    bsl v0.16b, v23.16b, v11.16b
+; NO_SVE-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
+; NO_SVE-NEXT:    stp q3, q2, [x0, #32]
+; NO_SVE-NEXT:    stp q0, q4, [x0]
+; NO_SVE-NEXT:    ldp d11, d10, [sp], #32 // 16-byte Folded Reload
+; NO_SVE-NEXT:    ret
+;
 ; VBITS_GE_2048-LABEL: select_v32f64:
 ; VBITS_GE_2048:       // %bb.0:
 ; VBITS_GE_2048-NEXT:    and w8, w2, #0x1
 ; VBITS_GE_2048-NEXT:    ptrue p0.d, vl32
 ; VBITS_GE_2048-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    ptrue p1.d
 ; VBITS_GE_2048-NEXT:    mov z2.d, x8

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-vselect.ll

; VBITS_GE_2048-NEXT:    ret
   %op1 = load <32 x double>, <32 x double>* %a
   %op2 = load <32 x double>, <32 x double>* %b
   %mask = fcmp oeq <32 x double> %op1, %op2
   %sel = select <32 x i1> %mask, <32 x double> %op1, <32 x double> %op2
   store <32 x double> %sel, <32 x double>* %a
   ret void
 }
 
-attributes #0 = { "target-features"="+sve" }
+attributes #0 = { "target-features"="+sve" uwtable }

llvm/test/CodeGen/AArch64/sve-fixed-length-int-vselect.ll

 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [sp]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; CHECK-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.b, z0.b, #0x1
 ; CHECK-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; CHECK-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:   .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i8>, <32 x i8>* %a
   %op2 = load <32 x i8>, <32 x i8>* %b
   %sel = select <32 x i1> %mask, <32 x i8> %op1, <32 x i8> %op2
   store <32 x i8> %sel, <32 x i8>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1b { z0.b }, p0/z, [sp]
 ; VBITS_GE_512-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.b, z0.b, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; VBITS_GE_512-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; VBITS_GE_512-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x i8>, <64 x i8>* %a
   %op2 = load <64 x i8>, <64 x i8>* %b
   %sel = select <64 x i1> %mask, <64 x i8> %op1, <64 x i8> %op2
   store <64 x i8> %sel, <64 x i8>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1b { z0.b }, p0/z, [sp]
 ; VBITS_GE_1024-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.b, z0.b, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; VBITS_GE_1024-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; VBITS_GE_1024-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <128 x i1>, <128 x i1>* %c
   %op1 = load <128 x i8>, <128 x i8>* %a
   %op2 = load <128 x i8>, <128 x i8>* %b
   %sel = select <128 x i1> %mask, <128 x i8> %op1, <128 x i8> %op2
   store <128 x i8> %sel, <128 x i8>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1b { z0.b }, p0/z, [sp]
 ; VBITS_GE_2048-NEXT:    ld1b { z1.b }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1b { z2.b }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.b, z0.b, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.b, p1/z, z0.b, #0
 ; VBITS_GE_2048-NEXT:    sel z0.b, p1, z1.b, z2.b
 ; VBITS_GE_2048-NEXT:    st1b { z0.b }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <256 x i1>, <256 x i1>* %c
   %op1 = load <256 x i8>, <256 x i8>* %a
   %op2 = load <256 x i8>, <256 x i8>* %b
   %sel = select <256 x i1> %mask, <256 x i8> %op1, <256 x i8> %op2
   store <256 x i8> %sel, <256 x i8>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [sp]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.h, z0.h, #0x1
 ; CHECK-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; CHECK-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x i16>, <16 x i16>* %a
   %op2 = load <16 x i16>, <16 x i16>* %b
   %sel = select <16 x i1> %mask, <16 x i16> %op1, <16 x i16> %op2
   store <16 x i16> %sel, <16 x i16>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [sp]
 ; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_512-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i16>, <32 x i16>* %a
   %op2 = load <32 x i16>, <32 x i16>* %b
   %sel = select <32 x i1> %mask, <32 x i16> %op1, <32 x i16> %op2
   store <32 x i16> %sel, <32 x i16>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1h { z0.h }, p0/z, [sp]
 ; VBITS_GE_1024-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_1024-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_1024-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x i16>, <64 x i16>* %a
   %op2 = load <64 x i16>, <64 x i16>* %b
   %sel = select <64 x i1> %mask, <64 x i16> %op1, <64 x i16> %op2
   store <64 x i16> %sel, <64 x i16>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1h { z0.h }, p0/z, [sp]
 ; VBITS_GE_2048-NEXT:    ld1h { z1.h }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1h { z2.h }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.h, z0.h, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_2048-NEXT:    sel z0.h, p1, z1.h, z2.h
 ; VBITS_GE_2048-NEXT:    st1h { z0.h }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <128 x i1>, <128 x i1>* %c
   %op1 = load <128 x i16>, <128 x i16>* %a
   %op2 = load <128 x i16>, <128 x i16>* %b
   %sel = select <128 x i1> %mask, <128 x i16> %op1, <128 x i16> %op2
   store <128 x i16> %sel, <128 x i16>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.s, z0.s, #0x1
 ; CHECK-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; CHECK-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <8 x i1>, <8 x i1>* %c
   %op1 = load <8 x i32>, <8 x i32>* %a
   %op2 = load <8 x i32>, <8 x i32>* %b
   %sel = select <8 x i1> %mask, <8 x i32> %op1, <8 x i32> %op2
   store <8 x i32> %sel, <8 x i32>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [sp]
 ; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_512-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x i32>, <16 x i32>* %a
   %op2 = load <16 x i32>, <16 x i32>* %b
   %sel = select <16 x i1> %mask, <16 x i32> %op1, <16 x i32> %op2
   store <16 x i32> %sel, <16 x i32>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1w { z0.s }, p0/z, [sp]
 ; VBITS_GE_1024-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_1024-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_1024-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i32>, <32 x i32>* %a
   %op2 = load <32 x i32>, <32 x i32>* %b
   %sel = select <32 x i1> %mask, <32 x i32> %op1, <32 x i32> %op2
   store <32 x i32> %sel, <32 x i32>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1w { z0.s }, p0/z, [sp]
 ; VBITS_GE_2048-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1w { z2.s }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.s, z0.s, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.s, p1/z, z0.s, #0
 ; VBITS_GE_2048-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <64 x i1>, <64 x i1>* %c
   %op1 = load <64 x i32>, <64 x i32>* %a
   %op2 = load <64 x i32>, <64 x i32>* %b
   %sel = select <64 x i1> %mask, <64 x i32> %op1, <64 x i32> %op2
   store <64 x i32> %sel, <64 x i32>* %a
   ret void
 }

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; CHECK-NEXT:    and z0.d, z0.d, #0x1
 ; CHECK-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %mask = load <4 x i1>, <4 x i1>* %c
   %op1 = load <4 x i64>, <4 x i64>* %a
   %op2 = load <4 x i64>, <4 x i64>* %b
   %sel = select <4 x i1> %mask, <4 x i64> %op1, <4 x i64> %op2
   store <4 x i64> %sel, <4 x i64>* %a
   ret void
 }

 ; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_512-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_512-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_512-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_512-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_512-NEXT:    mov sp, x29
+; VBITS_GE_512-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_512-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_512-NEXT:    .cfi_restore w30
+; VBITS_GE_512-NEXT:    .cfi_restore w29
 ; VBITS_GE_512-NEXT:    ret
   %mask = load <8 x i1>, <8 x i1>* %c
   %op1 = load <8 x i64>, <8 x i64>* %a
   %op2 = load <8 x i64>, <8 x i64>* %b
   %sel = select <8 x i1> %mask, <8 x i64> %op1, <8 x i64> %op2
   store <8 x i64> %sel, <8 x i64>* %a
   ret void
 }

 ; VBITS_GE_1024-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; VBITS_GE_1024-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_1024-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_1024-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_1024-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_1024-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_1024-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_1024-NEXT:    mov sp, x29
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_1024-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_1024-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_1024-NEXT:    .cfi_restore w30
+; VBITS_GE_1024-NEXT:    .cfi_restore w29
 ; VBITS_GE_1024-NEXT:    ret
   %mask = load <16 x i1>, <16 x i1>* %c
   %op1 = load <16 x i64>, <16 x i64>* %a
   %op2 = load <16 x i64>, <16 x i64>* %b
   %sel = select <16 x i1> %mask, <16 x i64> %op1, <16 x i64> %op2
   store <16 x i64> %sel, <16 x i64>* %a
   ret void
 }

 ; VBITS_GE_2048-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; VBITS_GE_2048-NEXT:    ld1d { z1.d }, p0/z, [x0]
 ; VBITS_GE_2048-NEXT:    ld1d { z2.d }, p0/z, [x1]
 ; VBITS_GE_2048-NEXT:    and z0.d, z0.d, #0x1
 ; VBITS_GE_2048-NEXT:    cmpne p1.d, p1/z, z0.d, #0
 ; VBITS_GE_2048-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_2048-NEXT:    mov sp, x29
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa wsp, 16
 ; VBITS_GE_2048-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; VBITS_GE_2048-NEXT:    .cfi_def_cfa_offset 0
+; VBITS_GE_2048-NEXT:    .cfi_restore w30
+; VBITS_GE_2048-NEXT:    .cfi_restore w29
 ; VBITS_GE_2048-NEXT:    ret
   %mask = load <32 x i1>, <32 x i1>* %c
   %op1 = load <32 x i64>, <32 x i64>* %a
   %op2 = load <32 x i64>, <32 x i64>* %b
   %sel = select <32 x i1> %mask, <32 x i64> %op1, <32 x i64> %op2
   store <32 x i64> %sel, <32 x i64>* %a
   ret void
 }
 
-attributes #0 = { "target-features"="+sve" }
+attributes #0 = { "target-features"="+sve" uwtable }

llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle.ll

 ; CHECK-NEXT:    mov z2.d, z1.d[1]
 ; CHECK-NEXT:    stp d1, d2, [sp, #16]
 ; CHECK-NEXT:    mov z1.d, z0.d[3]
 ; CHECK-NEXT:    mov z0.d, z0.d[2]
 ; CHECK-NEXT:    stp d0, d1, [sp]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %op1 = load <4 x double>, <4 x double>* %a
   %op2 = load <4 x double>, <4 x double>* %b
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   store <4 x double> %ret, <4 x double>* %a
   ret void
 }
 
-attributes #0 = { "target-features"="+sve" }
+attributes #0 = { "target-features"="+sve" uwtable }

llvm/test/CodeGen/AArch64/sve-insert-element.ll

 ; CHECK-NEXT:    mov z0.b, p2/m, w0
 ; CHECK-NEXT:    and z0.b, z0.b, #0x1
 ; CHECK-NEXT:    cmpne p0.b, p1/z, z0.b, #0
 ; CHECK-NEXT:    ret
   %res = insertelement <vscale x 16 x i1> %val, i1 %elt, i32 %idx
   ret <vscale x 16 x i1> %res
 }
 
-define <vscale x 32 x i1> @test_predicate_insert_32xi1(<vscale x 32 x i1> %val, i1 %elt, i32 %idx) {
+define <vscale x 32 x i1> @test_predicate_insert_32xi1(<vscale x 32 x i1> %val, i1 %elt, i32 %idx) uwtable {
 ; CHECK-LABEL: test_predicate_insert_32xi1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 ; CHECK-NEXT:    mov x8, #-1

 ; CHECK-NEXT:    strb w0, [x9, x8]
 ; CHECK-NEXT:    ld1b { z0.b }, p1/z, [sp]
 ; CHECK-NEXT:    ld1b { z1.b }, p1/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    and z0.b, z0.b, #0x1
 ; CHECK-NEXT:    and z1.b, z1.b, #0x1
 ; CHECK-NEXT:    cmpne p0.b, p1/z, z0.b, #0
 ; CHECK-NEXT:    cmpne p1.b, p1/z, z1.b, #0
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = insertelement <vscale x 32 x i1> %val, i1 %elt, i32 %idx
   ret <vscale x 32 x i1> %res
 }

llvm/test/CodeGen/AArch64/sve-insert-vector.ll

 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0, #5, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0, #4, mul vl]
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 8 x i64> %sv0, i64 8)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
-define void @insert_v2i64_nxv16i64(<2 x i64> %sv0, <2 x i64> %sv1, <vscale x 16 x i64>* %out) {
+define void @insert_v2i64_nxv16i64(<2 x i64> %sv0, <2 x i64> %sv1, <vscale x 16 x i64>* %out) uwtable {
 ; CHECK-LABEL: insert_v2i64_nxv16i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    addvl sp, sp, #-4
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
 ; CHECK-NEXT:    str q1, [sp, #32]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [sp, #2, mul vl]
 ; CHECK-NEXT:    ld1d { z3.d }, p0/z, [sp, #3, mul vl]
 ; CHECK-NEXT:    st1d { z3.d }, p0, [x0, #3, mul vl]
 ; CHECK-NEXT:    st1d { z2.d }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %v0 = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> undef, <2 x i64> %sv0, i64 0)
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> %v0, <2 x i64> %sv1, i64 4)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
 define void @insert_v2i64_nxv16i64_lo0(<2 x i64>* %psv, <vscale x 16 x i64>* %out) {
 ; CHECK-LABEL: insert_v2i64_nxv16i64_lo0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
 ; CHECK-NEXT:    ret
   %sv = load <2 x i64>, <2 x i64>* %psv
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> undef, <2 x i64> %sv, i64 0)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
-define void @insert_v2i64_nxv16i64_lo2(<2 x i64>* %psv, <vscale x 16 x i64>* %out) {
+define void @insert_v2i64_nxv16i64_lo2(<2 x i64>* %psv, <vscale x 16 x i64>* %out) uwtable {
 ; CHECK-LABEL: insert_v2i64_nxv16i64_lo2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    str q0, [sp, #16]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x1, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %sv = load <2 x i64>, <2 x i64>* %psv
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> undef, <2 x i64> %sv, i64 2)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
 

llvm/test/CodeGen/AArch64/sve-ldnf1.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=prologepilog -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
 #
 # Test that prologepilog works for each of the LDNF1 instructions for stack-based objects.
 #
 --- |
-  define void @testcase_positive_offset() {
+  define void @testcase_positive_offset() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset 63 * readsize
     ret void
   }
-  define void @testcase_negative_offset() {
+  define void @testcase_negative_offset() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset 63 * readsize
     ret void
   }
 
-  define void @testcase_positive_offset_out_of_range() {
+  define void @testcase_positive_offset_out_of_range() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset 64 * readsize
     ret void
   }
-  define void @testcase_negative_offset_out_of_range() {
+  define void @testcase_negative_offset_out_of_range() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset -1 * readsize
     ret void
   }
 ...
 ---
 name:            testcase_positive_offset
bb.0 (%ir-block.0):
     ; CHECK-NEXT: renamable $z0 = LDNF1H_D_IMM renamable $p0, $sp, 7 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNF1SH_S_IMM renamable $p0, $sp, 7 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNF1SH_D_IMM renamable $p0, $sp, 7 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNF1W_IMM renamable $p0, $sp, 7 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: renamable $z0 = LDNF1W_D_IMM renamable $p0, $sp, 7 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, $sp, 7 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: renamable $z0 = LDNF1D_IMM renamable $p0, $sp, 7 :: (load (s64) from %ir.object)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_S_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_D_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_H_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_S_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_D_IMM renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
bb.0 (%ir-block.0):
     ; CHECK-NEXT: renamable $z0 = LDNF1H_D_IMM renamable $p0, $sp, -8 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNF1SH_S_IMM renamable $p0, $sp, -8 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNF1SH_D_IMM renamable $p0, $sp, -8 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNF1W_IMM renamable $p0, $sp, -8 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: renamable $z0 = LDNF1W_D_IMM renamable $p0, $sp, -8 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, $sp, -8 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: renamable $z0 = LDNF1D_IMM renamable $p0, $sp, -8 :: (load (s64) from %ir.object)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_S_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_D_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_H_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_S_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_D_IMM renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
bb.0 (%ir-block.0):
     ; CHECK-NEXT: renamable $z0 = LDNF1SH_D_IMM renamable $p0, killed $x8, 7 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 1
     ; CHECK-NEXT: renamable $z0 = LDNF1W_IMM renamable $p0, killed $x8, 7 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDPL_XXI $sp, 4
     ; CHECK-NEXT: renamable $z0 = LDNF1W_D_IMM renamable $p0, killed $x8, 7 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDPL_XXI $sp, 4
     ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, killed $x8, 7 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_S_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_D_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_H_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_S_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_D_IMM renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
bb.0 (%ir-block.0):
     ; CHECK-NEXT: renamable $z0 = LDNF1SH_D_IMM renamable $p0, killed $x8, -8 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, -1
     ; CHECK-NEXT: renamable $z0 = LDNF1W_IMM renamable $p0, killed $x8, -8 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDPL_XXI $sp, -4
     ; CHECK-NEXT: renamable $z0 = LDNF1W_D_IMM renamable $p0, killed $x8, -8 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDPL_XXI $sp, -4
     ; CHECK-NEXT: renamable $z0 = LDNF1SW_D_IMM renamable $p0, killed $x8, -8 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNF1B_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_H_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_S_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1B_D_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_H_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_S_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNF1SB_D_IMM renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)

llvm/test/CodeGen/AArch64/sve-ldstnt1.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=prologepilog -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
 #
 # Test that prologepilog works for each of the LDNT1/STNT1 instructions for stack-based objects.
 #
 --- |
-  define void @testcase_positive_offset() {
+  define void @testcase_positive_offset() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset 7 * readsize
     ret void
   }
-  define void @testcase_negative_offset() {
+  define void @testcase_negative_offset() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset -8 * readsize
     ret void
   }
 
-  define void @testcase_positive_offset_out_of_range() {
+  define void @testcase_positive_offset_out_of_range() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset 8 * readsize
     ret void
   }
-  define void @testcase_negative_offset_out_of_range() {
+  define void @testcase_negative_offset_out_of_range() uwtable {
     %dummy = alloca <vscale x 2 x i64>, align 8
     %object = alloca <vscale x 2 x i64>, align 8
     ; Reads from %object at offset -9 * readsize
     ret void
   }
 ...
 ---
 name:            testcase_positive_offset
bb.0 (%ir-block.0):
     ; CHECK-NEXT: renamable $z0 = LDNT1H_ZRI renamable $p0, $sp, 7 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNT1W_ZRI renamable $p0, $sp, 7 :: (load (s32) from %ir.object, align 8)
     ; CHECK-NEXT: renamable $z0 = LDNT1D_ZRI renamable $p0, $sp, 7 :: (load (s64) from %ir.object)
     ; CHECK-NEXT: STNT1B_ZRI renamable $z0, renamable $p0, $sp, 7 :: (store (s8) into %ir.object, align 8)
     ; CHECK-NEXT: STNT1H_ZRI renamable $z0, renamable $p0, $sp, 7 :: (store (s16) into %ir.object, align 8)
     ; CHECK-NEXT: STNT1W_ZRI renamable $z0, renamable $p0, $sp, 7 :: (store (s32) into %ir.object, align 8)
     ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, $sp, 7 :: (store (s64) into %ir.object)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, 7 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, 7 :: (load 2 from %ir.object, align 2)
     renamable $z0 = LDNT1W_ZRI renamable $p0, %stack.1.object, 7 :: (load 4 from %ir.object, align 8)
     renamable $z0 = LDNT1D_ZRI renamable $p0, %stack.1.object, 7 :: (load 8 from %ir.object, align 8)
     STNT1B_ZRI renamable $z0, renamable $p0, %stack.1.object, 7 :: (store 1 into %ir.object, align 8)
     STNT1H_ZRI renamable $z0, renamable $p0, %stack.1.object, 7 :: (store 2 into %ir.object, align 8)
     STNT1W_ZRI renamable $z0, renamable $p0, %stack.1.object, 7 :: (store 4 into %ir.object, align 8)
bb.0 (%ir-block.0):
     ; CHECK-NEXT: renamable $z0 = LDNT1H_ZRI renamable $p0, $sp, -8 :: (load (s16) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNT1W_ZRI renamable $p0, $sp, -8 :: (load (s32) from %ir.object)
     ; CHECK-NEXT: renamable $z0 = LDNT1D_ZRI renamable $p0, $sp, -8 :: (load (s64) from %ir.object)
     ; CHECK-NEXT: STNT1B_ZRI renamable $z0, renamable $p0, $sp, -8 :: (store (s8) into %ir.object, align 8)
     ; CHECK-NEXT: STNT1H_ZRI renamable $z0, renamable $p0, $sp, -8 :: (store (s16) into %ir.object, align 8)
     ; CHECK-NEXT: STNT1W_ZRI renamable $z0, renamable $p0, $sp, -8 :: (store (s32) into %ir.object, align 8)
     ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, $sp, -8 :: (store (s64) into %ir.object)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, -8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, -8 :: (load 2 from %ir.object, align 2)
     renamable $z0 = LDNT1W_ZRI renamable $p0, %stack.1.object, -8 :: (load 4 from %ir.object, align 4)
     renamable $z0 = LDNT1D_ZRI renamable $p0, %stack.1.object, -8 :: (load 8 from %ir.object, align 8)
     STNT1B_ZRI renamable $z0, renamable $p0, %stack.1.object, -8 :: (store 1 into %ir.object, align 8)
     STNT1H_ZRI renamable $z0, renamable $p0, %stack.1.object, -8 :: (store 2 into %ir.object, align 8)
     STNT1W_ZRI renamable $z0, renamable $p0, %stack.1.object, -8 :: (store 4 into %ir.object, align 8)
bb.0 (%ir-block.0):
     ; CHECK-NEXT: STNT1B_ZRI renamable $z0, renamable $p0, killed $x8, 7 :: (store (s8) into %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 1
     ; CHECK-NEXT: STNT1H_ZRI renamable $z0, renamable $p0, killed $x8, 7 :: (store (s16) into %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 1
     ; CHECK-NEXT: STNT1W_ZRI renamable $z0, renamable $p0, killed $x8, 7 :: (store (s32) into %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 1
     ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, killed $x8, 7 :: (store (s64) into %ir.object)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, 8 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, 8 :: (load 2 from %ir.object, align 2)
     renamable $z0 = LDNT1W_ZRI renamable $p0, %stack.1.object, 8 :: (load 4 from %ir.object, align 4)
     renamable $z0 = LDNT1D_ZRI renamable $p0, %stack.1.object, 8 :: (load 8 from %ir.object, align 8)
     STNT1B_ZRI renamable $z0, renamable $p0, %stack.1.object, 8 :: (store 1 into %ir.object, align 8)
     STNT1H_ZRI renamable $z0, renamable $p0, %stack.1.object, 8 :: (store 2 into %ir.object, align 8)
     STNT1W_ZRI renamable $z0, renamable $p0, %stack.1.object, 8 :: (store 4 into %ir.object, align 8)
bb.0 (%ir-block.0):
     ; CHECK-NEXT: STNT1B_ZRI renamable $z0, renamable $p0, killed $x8, -8 :: (store (s8) into %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, -1
     ; CHECK-NEXT: STNT1H_ZRI renamable $z0, renamable $p0, killed $x8, -8 :: (store (s16) into %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, -1
     ; CHECK-NEXT: STNT1W_ZRI renamable $z0, renamable $p0, killed $x8, -8 :: (store (s32) into %ir.object, align 8)
     ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, -1
     ; CHECK-NEXT: STNT1D_ZRI renamable $z0, renamable $p0, killed $x8, -8 :: (store (s64) into %ir.object)
     ; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 4
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $wsp, 16
     ; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.2)
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
     ; CHECK-NEXT: RET_ReallyLR implicit $z0
     renamable $z0 = LDNT1B_ZRI renamable $p0, %stack.1.object, -9 :: (load 1 from %ir.object, align 2)
     renamable $z0 = LDNT1H_ZRI renamable $p0, %stack.1.object, -9 :: (load 2 from %ir.object, align 2)
     renamable $z0 = LDNT1W_ZRI renamable $p0, %stack.1.object, -9 :: (load 4 from %ir.object, align 4)
     renamable $z0 = LDNT1D_ZRI renamable $p0, %stack.1.object, -9 :: (load 8 from %ir.object, align 8)
     STNT1B_ZRI renamable $z0, renamable $p0, %stack.1.object, -9 :: (store 1 into %ir.object, align 8)
     STNT1H_ZRI renamable $z0, renamable $p0, %stack.1.object, -9 :: (store 2 into %ir.object, align 8)
     STNT1W_ZRI renamable $z0, renamable $p0, %stack.1.object, -9 :: (store 4 into %ir.object, align 8)
     STNT1D_ZRI renamable $z0, renamable $p0, %stack.1.object, -9 :: (store 8 into %ir.object, align 8)
     RET_ReallyLR implicit $z0
 ...

llvm/test/CodeGen/AArch64/sve-pred-arith.ll

 ; CHECK-NEXT:    ret
   %res = add <vscale x 2 x i1> %a, %b
   ret <vscale x 2 x i1> %res;
 }
 
 
 ; ILLEGAL ADDS
 
-define aarch64_sve_vector_pcs <vscale x 64 x i1> @add_nxv64i1(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b) {
+define aarch64_sve_vector_pcs <vscale x 64 x i1> @add_nxv64i1(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b) uwtable {
 ; CHECK-LABEL: add_nxv64i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill

 ; CHECK-NEXT:    eor p2.b, p8/z, p2.b, p7.b
 ; CHECK-NEXT:    eor p3.b, p8/z, p3.b, p6.b
 ; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = add <vscale x 64 x i1> %a, %b
   ret <vscale x 64 x i1> %res;
 }
 
 
 ; LEGAL SUBS
 
; CHECK-NEXT:    ret
   %res = sub <vscale x 2 x i1> %a, %b
   ret <vscale x 2 x i1> %res;
 }
 
 
 ; ILLEGAL SUBGS
 
 
-define aarch64_sve_vector_pcs <vscale x 64 x i1> @sub_nxv64i1(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b) {
+define aarch64_sve_vector_pcs <vscale x 64 x i1> @sub_nxv64i1(<vscale x 64 x i1> %a, <vscale x 64 x i1> %b) uwtable {
 ; CHECK-LABEL: sub_nxv64i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill

 ; CHECK-NEXT:    eor p2.b, p8/z, p2.b, p7.b
 ; CHECK-NEXT:    eor p3.b, p8/z, p3.b, p6.b
 ; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %res = sub <vscale x 64 x i1> %a, %b
   ret <vscale x 64 x i1> %res;
 }

llvm/test/CodeGen/AArch64/sve-varargs.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
 
 declare i32 @sve_printf(i8*, <vscale x 4 x i32>, ...)
 
 @.str_1 = internal constant [6 x i8] c"boo!\0A\00"
 
-define void @foo(<vscale x 4 x i32> %x) {
+define void @foo(<vscale x 4 x i32> %x) uwtable {
 ; CHECK-LABEL: foo:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x0, .str_1
 ; CHECK-NEXT:    add x0, x0, :lo12:.str_1
 ; CHECK-NEXT:    bl sve_printf
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
   %f = getelementptr [6 x i8], [6 x i8]* @.str_1, i64 0, i64 0
   call i32 (i8*, <vscale x 4 x i32>, ...) @sve_printf(i8* %f, <vscale x 4 x i32> %x)
   ret void
 }

llvm/test/CodeGen/AArch64/swifttail-call.ll

 ; COMMON-NEXT: // %bb.
 
   musttail call swifttailcc void @callee_stack0()
   ret void
 
 ; COMMON-NEXT: b callee_stack0
 }
 
-define swifttailcc void @caller_to0_from8([8 x i64], i64) {
+define swifttailcc void @caller_to0_from8([8 x i64], i64) #0 {
 ; COMMON-LABEL: caller_to0_from8:
 
   musttail call swifttailcc void @callee_stack0()
   ret void
 
 ; COMMON: add sp, sp, #16
+; COMMON-NEXT:  .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack0
 }
 
-define swifttailcc void @caller_to8_from0() {
+define swifttailcc void @caller_to8_from0() #0 {
 ; COMMON-LABEL: caller_to8_from0:
 
 ; Key point is that the "42" should go #16 below incoming stack
 ; pointer (we didn't have arg space to reuse).
   musttail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #-16]!
 ; COMMON-NEXT: b callee_stack8
 }
 
-define swifttailcc void @caller_to8_from8([8 x i64], i64 %a) {
+define swifttailcc void @caller_to8_from8([8 x i64], i64 %a) #0 {
 ; COMMON-LABEL: caller_to8_from8:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at SP on entry.
   musttail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp]
 ; COMMON-NEXT: b callee_stack8
 }
 
-define swifttailcc void @caller_to16_from8([8 x i64], i64 %a) {
+define swifttailcc void @caller_to16_from8([8 x i64], i64 %a) #0 {
 ; COMMON-LABEL: caller_to16_from8:
 ; COMMON-NOT: sub sp,
 
 ; Important point is that the call reuses the "dead" argument space
 ; above %a on the stack. If it tries to go below incoming-SP then the
 ; callee will not deallocate the space, even in swifttailcc.
   musttail call swifttailcc void @callee_stack16([8 x i64] undef, i64 42, i64 2)
 
 ; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
 ; COMMON-NEXT: b callee_stack16
   ret void
 }
 
 
-define swifttailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) {
+define swifttailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) #0 {
 ; COMMON-LABEL: caller_to8_from24:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at #16 above SP on entry.
   musttail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack8
 }
 
 
-define swifttailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
+define swifttailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) #0 {
 ; COMMON-LABEL: caller_to16_from16:
 ; COMMON-NOT: sub sp,
 
 ; Here we want to make sure that both loads happen before the stores:
 ; otherwise either %a or %b will be wrongly clobbered.
   musttail call swifttailcc void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
   ret void
 
; COMMON-NEXT: // %bb.
   ret void
 
 ; COMMON: bl callee_stack0
 ; COMMON: ret
 }
 
 ; Weakly-referenced extern functions cannot be tail-called, as AAELF does
 ; not define the behaviour of branch instructions to undefined weak symbols.
-define swifttailcc void @caller_weak() {
+define swifttailcc void @caller_weak() #0 {
 ; COMMON-LABEL: caller_weak:
 ; COMMON: bl callee_weak
   tail call void @callee_weak()
   ret void
 }
 
 declare { [2 x float] } @get_vec2()
 
-define { [3 x float] } @test_add_elem() {
+define { [3 x float] } @test_add_elem() #0 {
 ; SDAG-LABEL: test_add_elem:
 ; SDAG: bl get_vec2
 ; SDAG: fmov s2, #1.0
 ; SDAG: ret
 ; GISEL-LABEL: test_add_elem:
 ; GISEL: str  x30, [sp, #-16]!
 ; GISEL: bl get_vec2
 ; GISEL: fmov s2, #1.0
 ; GISEL: ldr  x30, [sp], #16
 ; GISEL: ret
 
   %call = tail call { [2 x float] } @get_vec2()
   %arr = extractvalue { [2 x float] } %call, 0
   %arr.0 = extractvalue [2 x float] %arr, 0
   %arr.1 = extractvalue [2 x float] %arr, 1
 
   %res.0 = insertvalue { [3 x float] } undef, float %arr.0, 0, 0
   %res.01 = insertvalue { [3 x float] } %res.0, float %arr.1, 0, 1
   %res.012 = insertvalue { [3 x float] } %res.01, float 1.000000e+00, 0, 2
   ret { [3 x float] } %res.012
 }
 
 declare double @get_double()
-define { double, [2 x double] } @test_mismatched_insert() {
+define { double, [2 x double] } @test_mismatched_insert() #0 {
 ; COMMON-LABEL: test_mismatched_insert:
 ; COMMON: bl get_double
 ; COMMON: bl get_double
 ; COMMON: bl get_double
 ; COMMON: ret
 
   %val0 = call double @get_double()
   %val1 = call double @get_double()
   %val2 = tail call double @get_double()
 
   %res.0 = insertvalue { double, [2 x double] } undef, double %val0, 0
   %res.01 = insertvalue { double, [2 x double] } %res.0, double %val1, 1, 0
   %res.012 = insertvalue { double, [2 x double] } %res.01, double %val2, 1, 1
 
   ret { double, [2 x double] } %res.012
 }
 
-define void @fromC_totail() {
+define void @fromC_totail() #0 {
 ; COMMON-LABEL: fromC_totail:
 ; COMMON: sub sp, sp, #48
 
 ; COMMON-NOT: sub sp,
 ; COMMON: mov w[[TMP:[0-9]+]], #42
 ; COMMON: str x[[TMP]], [sp]
 ; COMMON: bl callee_stack8
   ; We must reset the stack to where it was before the call by undoing its extra stack pop.
 ; COMMON: str x[[TMP]], [sp, #-16]!
 ; COMMON: bl callee_stack8
 ; COMMON: sub sp, sp, #16
 
   call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 }
 
-define void @fromC_totail_noreservedframe(i32 %len) {
+define void @fromC_totail_noreservedframe(i32 %len) #0 {
 ; COMMON-LABEL: fromC_totail_noreservedframe:
 ; COMMON: stp x29, x30, [sp, #-48]!
 
 ; COMMON: mov w[[TMP:[0-9]+]], #42
   ; Note stack is subtracted here to allocate space for arg
 ; COMMON: str x[[TMP]], [sp, #-16]!
 ; COMMON: bl callee_stack8
   ; And here.
 ; COMMON: str x[[TMP]], [sp, #-16]!
 ; COMMON: bl callee_stack8
   ; But not restored here because callee_stack8 did that for us.
 ; COMMON-NOT: sub sp,
 
   ; Variable sized allocation prevents reserving frame at start of function so each call must allocate any stack space it needs.
   %var = alloca i32, i32 %len
 
   call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 }
 
 declare void @Ccallee_stack8([8 x i64], i64)
 
-define swifttailcc void @fromtail_toC() {
+define swifttailcc void @fromtail_toC() #0 {
 ; COMMON-LABEL: fromtail_toC:
 ; COMMON: sub sp, sp, #32
 
 ; COMMON-NOT: sub sp,
 ; COMMON: mov w[[TMP:[0-9]+]], #42
 ; COMMON: str x[[TMP]], [sp]
 ; COMMON: bl Ccallee_stack8
   ; C callees will return with the stack exactly where we left it, so we mustn't try to fix anything.
 ; COMMON-NOT: add sp,
 ; COMMON-NOT: sub sp,
 ; COMMON: str x[[TMP]], [sp]{{$}}
 ; COMMON: bl Ccallee_stack8
 ; COMMON-NOT: sub sp,
 
 
   call void @Ccallee_stack8([8 x i64] undef, i64 42)
   call void @Ccallee_stack8([8 x i64] undef, i64 42)
   ret void
 }
 
 declare swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself %closure)
-define swiftcc i8* @CallSwiftSelf(i8* swiftself %closure, i8* %context) {
+define swiftcc i8* @CallSwiftSelf(i8* swiftself %closure, i8* %context) #0 {
 ; CHECK-LABEL: CallSwiftSelf:
 ; CHECK: stp x20
   ;call void asm "","~{r13}"() ; We get a push r13 but why not with the call
   ; below?
   %res = call swifttailcc i8* @SwiftSelf(i8 * swiftasync %context, i8* swiftself %closure)
   ret i8* %res
 }
+
+attributes #0 = { uwtable }
 No newline at end of file

llvm/test/CodeGen/AArch64/tail-call.ll

 ; COMMON-NEXT: // %bb.
 
   tail call fastcc void @callee_stack0()
   ret void
 
 ; COMMON-NEXT: b callee_stack0
 }
 
-define fastcc void @caller_to0_from8([8 x i64], i64) {
+define fastcc void @caller_to0_from8([8 x i64], i64) #0 {
 ; COMMON-LABEL: caller_to0_from8:
 
   tail call fastcc void @callee_stack0()
   ret void
 
 ; COMMON: add sp, sp, #16
+; COMMON: .cfi_def_cfa_offset  -16
 ; COMMON-NEXT: b callee_stack0
 }
 
-define fastcc void @caller_to8_from0() {
+define fastcc void @caller_to8_from0() #0 {
 ; COMMON-LABEL: caller_to8_from0:
 
 ; Key point is that the "42" should go #16 below incoming stack
 ; pointer (we didn't have arg space to reuse).
   tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #-16]!
 ; COMMON-NEXT: b callee_stack8
 }
 
-define fastcc void @caller_to8_from8([8 x i64], i64 %a) {
+define fastcc void @caller_to8_from8([8 x i64], i64 %a) #0 {
 ; COMMON-LABEL: caller_to8_from8:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at SP on entry.
   tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp]
 ; COMMON-NEXT: b callee_stack8
 }
 
-define fastcc void @caller_to16_from8([8 x i64], i64 %a) {
+define fastcc void @caller_to16_from8([8 x i64], i64 %a) #0 {
 ; COMMON-LABEL: caller_to16_from8:
 ; COMMON-NOT: sub sp,
 
 ; Important point is that the call reuses the "dead" argument space
 ; above %a on the stack. If it tries to go below incoming-SP then the
 ; callee will not deallocate the space, even in fastcc.
   tail call fastcc void @callee_stack16([8 x i64] undef, i64 42, i64 2)
 
 ; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
 ; COMMON-NEXT: b callee_stack16
   ret void
 }
 
 
-define fastcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) {
+define fastcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) #0 {
 ; COMMON-LABEL: caller_to8_from24:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at #16 above SP on entry.
   tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON: .cfi_def_cfa_offset  -16
 ; COMMON-NEXT: b callee_stack8
 }
 
 
-define fastcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
+define fastcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) #0 {
 ; COMMON-LABEL: caller_to16_from16:
 ; COMMON-NOT: sub sp,
 
 ; Here we want to make sure that both loads happen before the stores:
 ; otherwise either %a or %b will be wrongly clobbered.
   tail call fastcc void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
   ret void
 
; COMMON-NEXT: // %bb.
   ret void
 
 ; COMMON: bl callee_stack0
 ; COMMON: ret
 }
 
 ; Weakly-referenced extern functions cannot be tail-called, as AAELF does
 ; not define the behaviour of branch instructions to undefined weak symbols.
-define fastcc void @caller_weak() {
+define fastcc void @caller_weak() #0 {
 ; COMMON-LABEL: caller_weak:
 ; COMMON: bl callee_weak
   tail call void @callee_weak()
   ret void
 }
 
 declare { [2 x float] } @get_vec2()
 
-define { [3 x float] } @test_add_elem() {
+define { [3 x float] } @test_add_elem() #0 {
 ; SDAG-LABEL: test_add_elem:
 ; SDAG: bl get_vec2
 ; SDAG: fmov s2, #1.0
 ; SDAG: ret
 ; GISEL-LABEL: test_add_elem:
 ; GISEL: str  x30, [sp, #-16]!
 ; GISEL: bl get_vec2
 ; GISEL: fmov s2, #1.0
 ; GISEL: ldr  x30, [sp], #16
 ; GISEL: ret
 
   %call = tail call { [2 x float] } @get_vec2()
   %arr = extractvalue { [2 x float] } %call, 0
   %arr.0 = extractvalue [2 x float] %arr, 0
   %arr.1 = extractvalue [2 x float] %arr, 1
 
   %res.0 = insertvalue { [3 x float] } undef, float %arr.0, 0, 0
   %res.01 = insertvalue { [3 x float] } %res.0, float %arr.1, 0, 1
   %res.012 = insertvalue { [3 x float] } %res.01, float 1.000000e+00, 0, 2
   ret { [3 x float] } %res.012
 }
 
 declare double @get_double()
-define { double, [2 x double] } @test_mismatched_insert() {
+define { double, [2 x double] } @test_mismatched_insert() #0 {
 ; COMMON-LABEL: test_mismatched_insert:
 ; COMMON: bl get_double
 ; COMMON: bl get_double
 ; COMMON: bl get_double
 ; COMMON: ret
 
   %val0 = call double @get_double()
   %val1 = call double @get_double()
   %val2 = tail call double @get_double()
 
   %res.0 = insertvalue { double, [2 x double] } undef, double %val0, 0
   %res.01 = insertvalue { double, [2 x double] } %res.0, double %val1, 1, 0
   %res.012 = insertvalue { double, [2 x double] } %res.01, double %val2, 1, 1
 
   ret { double, [2 x double] } %res.012
 }
+
+attributes #0 = { uwtable }
 No newline at end of file

llvm/test/CodeGen/AArch64/tailcc-tail-call.ll

 ; COMMON-NEXT: // %bb.
 
   tail call tailcc void @callee_stack0()
   ret void
 
 ; COMMON-NEXT: b callee_stack0
 }
 
-define tailcc void @caller_to0_from8([8 x i64], i64) {
+define tailcc void @caller_to0_from8([8 x i64], i64) #0 {
 ; COMMON-LABEL: caller_to0_from8:
 
   tail call tailcc void @callee_stack0()
   ret void
 
 ; COMMON: add sp, sp, #16
+; COMMON: .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack0
 }
 
-define tailcc void @caller_to8_from0() "frame-pointer"="all"{
+define tailcc void @caller_to8_from0() "frame-pointer"="all" uwtable {
 ; COMMON-LABEL: caller_to8_from0:
 
 ; Key point is that the "42" should go #16 below incoming stack
 ; pointer (we didn't have arg space to reuse).
   tail call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [x29, #16]
 ; COMMON: ldp x29, x30, [sp], #16
   ; If there is a sub here then the 42 will be briefly exposed to corruption
   ; from an interrupt if the kernel does not honour a red-zone, and a larger
   ; call could well overflow the red zone even if it is present.
 ; COMMON-NOT: sub sp,
+; COMMON-NEXT: .cfi_def_cfa_offset 16
+; COMMON-NEXT: .cfi_restore w30
+; COMMON-NEXT: .cfi_restore w29
 ; COMMON-NEXT: b callee_stack8
 }
 
-define tailcc void @caller_to8_from8([8 x i64], i64 %a) {
+define tailcc void @caller_to8_from8([8 x i64], i64 %a) #0 {
 ; COMMON-LABEL: caller_to8_from8:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at SP on entry.
   tail call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp]
 ; COMMON-NEXT: b callee_stack8
 }
 
-define tailcc void @caller_to16_from8([8 x i64], i64 %a) {
+define tailcc void @caller_to16_from8([8 x i64], i64 %a) #0 {
 ; COMMON-LABEL: caller_to16_from8:
 ; COMMON-NOT: sub sp,
 
 ; Important point is that the call reuses the "dead" argument space
 ; above %a on the stack. If it tries to go below incoming-SP then the
 ; callee will not deallocate the space, even in tailcc.
   tail call tailcc void @callee_stack16([8 x i64] undef, i64 42, i64 2)
 
 ; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
 ; COMMON-NEXT: b callee_stack16
   ret void
 }
 
 
-define tailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) {
+define tailcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) #0 {
 ; COMMON-LABEL: caller_to8_from24:
 ; COMMON-NOT: sub sp,
 
 ; Key point is that the "%a" should go where at #16 above SP on entry.
   tail call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 
 ; COMMON: str {{x[0-9]+}}, [sp, #16]!
+; COMMON-NEXT: .cfi_def_cfa_offset -16
 ; COMMON-NEXT: b callee_stack8
 }
 
 
-define tailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) {
+define tailcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) #0 {
 ; COMMON-LABEL: caller_to16_from16:
 ; COMMON-NOT: sub sp,
 
 ; Here we want to make sure that both loads happen before the stores:
 ; otherwise either %a or %b will be wrongly clobbered.
   tail call tailcc void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
   ret void
 
; COMMON-NEXT: // %bb.
   ret void
 
 ; COMMON: bl callee_stack0
 ; COMMON: ret
 }
 
 ; Weakly-referenced extern functions cannot be tail-called, as AAELF does
 ; not define the behaviour of branch instructions to undefined weak symbols.
-define tailcc void @caller_weak() {
+define tailcc void @caller_weak() #0 {
 ; COMMON-LABEL: caller_weak:
 ; COMMON: bl callee_weak
   tail call void @callee_weak()
   ret void
 }
 
 declare { [2 x float] } @get_vec2()
 
-define { [3 x float] } @test_add_elem() {
+define { [3 x float] } @test_add_elem() #0 {
 ; SDAG-LABEL: test_add_elem:
 ; SDAG: bl get_vec2
 ; SDAG: fmov s2, #1.0
 ; SDAG: ret
 ; GISEL-LABEL: test_add_elem:
 ; GISEL: str  x30, [sp, #-16]!
 ; GISEL: bl get_vec2
 ; GISEL: fmov s2, #1.0
 ; GISEL: ldr  x30, [sp], #16
 ; GISEL: ret
 
   %call = tail call { [2 x float] } @get_vec2()
   %arr = extractvalue { [2 x float] } %call, 0
   %arr.0 = extractvalue [2 x float] %arr, 0
   %arr.1 = extractvalue [2 x float] %arr, 1
 
   %res.0 = insertvalue { [3 x float] } undef, float %arr.0, 0, 0
   %res.01 = insertvalue { [3 x float] } %res.0, float %arr.1, 0, 1
   %res.012 = insertvalue { [3 x float] } %res.01, float 1.000000e+00, 0, 2
   ret { [3 x float] } %res.012
 }
 
 declare double @get_double()
-define { double, [2 x double] } @test_mismatched_insert() {
+define { double, [2 x double] } @test_mismatched_insert() #0 {
 ; COMMON-LABEL: test_mismatched_insert:
 ; COMMON: bl get_double
 ; COMMON: bl get_double
 ; COMMON: bl get_double
 ; COMMON: ret
 
   %val0 = call double @get_double()
   %val1 = call double @get_double()
   %val2 = tail call double @get_double()
 
   %res.0 = insertvalue { double, [2 x double] } undef, double %val0, 0
   %res.01 = insertvalue { double, [2 x double] } %res.0, double %val1, 1, 0
   %res.012 = insertvalue { double, [2 x double] } %res.01, double %val2, 1, 1
 
   ret { double, [2 x double] } %res.012
 }
 
-define void @fromC_totail() {
+define void @fromC_totail() #0 {
 ; COMMON-LABEL: fromC_totail:
 ; COMMON: sub sp, sp, #32
 
 ; COMMON-NOT: sub sp,
 ; COMMON: mov w[[TMP:[0-9]+]], #42
 ; COMMON: str x[[TMP]], [sp]
 ; COMMON: bl callee_stack8
   ; We must reset the stack to where it was before the call by undoing its extra stack pop.
 ; COMMON: str x[[TMP]], [sp, #-16]!
 ; COMMON: bl callee_stack8
 ; COMMON: sub sp, sp, #16
 
   call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 }
 
-define void @fromC_totail_noreservedframe(i32 %len) {
+define void @fromC_totail_noreservedframe(i32 %len) #0 {
 ; COMMON-LABEL: fromC_totail_noreservedframe:
 ; COMMON: stp x29, x30, [sp, #-32]!
 
 ; COMMON: mov w[[TMP:[0-9]+]], #42
   ; Note stack is subtracted here to allocate space for arg
 ; COMMON: str x[[TMP]], [sp, #-16]!
 ; COMMON: bl callee_stack8
   ; And here.
 ; COMMON: str x[[TMP]], [sp, #-16]!
 ; COMMON: bl callee_stack8
   ; But not restored here because callee_stack8 did that for us.
 ; COMMON-NOT: sub sp,
 
   ; Variable sized allocation prevents reserving frame at start of function so each call must allocate any stack space it needs.
   %var = alloca i32, i32 %len
 
   call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   call tailcc void @callee_stack8([8 x i64] undef, i64 42)
   ret void
 }
 
 declare void @Ccallee_stack8([8 x i64], i64)
 
-define tailcc void @fromtail_toC() {
+define tailcc void @fromtail_toC() #0 {
 ; COMMON-LABEL: fromtail_toC:
 ; COMMON: sub sp, sp, #32
 
 ; COMMON-NOT: sub sp,
 ; COMMON: mov w[[TMP:[0-9]+]], #42
 ; COMMON: str x[[TMP]], [sp]
 ; COMMON: bl Ccallee_stack8
   ; C callees will return with the stack exactly where we left it, so we mustn't try to fix anything.
 ; COMMON-NOT: add sp,
 ; COMMON-NOT: sub sp,
 ; COMMON: str x[[TMP]], [sp]{{$}}
 ; COMMON: bl Ccallee_stack8
 ; COMMON-NOT: sub sp,
 
 
   call void @Ccallee_stack8([8 x i64] undef, i64 42)
   call void @Ccallee_stack8([8 x i64] undef, i64 42)
   ret void
 }
+
+attributes #0 = { uwtable }
 No newline at end of file

llvm/test/CodeGen/AArch64/unwind-preserved.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -O0 -global-isel=0 -global-isel-abort=0 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -O0 -global-isel=1 -global-isel-abort=0 < %s | FileCheck %s --check-prefix=GISEL
 
 ; Test that z0 is saved/restored, as the unwinder may only retain the low 64bits (d0).
-define <vscale x 4 x i32> @invoke_callee_may_throw_sve(<vscale x 4 x i32> %v) personality i8 0 {
+define <vscale x 4 x i32> @invoke_callee_may_throw_sve(<vscale x 4 x i32> %v) uwtable personality i8 0 {
 ; CHECK-LABEL: invoke_callee_may_throw_sve:
 ; CHECK:       .Lfunc_begin0:
 ; CHECK-NEXT:    .cfi_startproc
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16

 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa0, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 160 * VG
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    bl may_throw_sve
 ; CHECK-NEXT:  .Ltmp1:
 ; CHECK-NEXT:    str z0, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    b .LBB0_1
 ; CHECK-NEXT:  .LBB0_1: // %.Lcontinue
 ; CHECK-NEXT:    ldr z0, [sp, #1, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; CHECK-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
+; CHECK-NEXT:    .cfi_restore z8
+; CHECK-NEXT:    .cfi_restore z9
+; CHECK-NEXT:    .cfi_restore z10
+; CHECK-NEXT:    .cfi_restore z11
+; CHECK-NEXT:    .cfi_restore z12
+; CHECK-NEXT:    .cfi_restore z13
+; CHECK-NEXT:    .cfi_restore z14
+; CHECK-NEXT:    .cfi_restore z15
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %.Lunwind
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:  .Ltmp2:
 ; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; CHECK-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
+; CHECK-NEXT:    .cfi_def_cfa wsp, 16
+; CHECK-NEXT:    .cfi_restore z8
+; CHECK-NEXT:    .cfi_restore z9
+; CHECK-NEXT:    .cfi_restore z10
+; CHECK-NEXT:    .cfi_restore z11
+; CHECK-NEXT:    .cfi_restore z12
+; CHECK-NEXT:    .cfi_restore z13
+; CHECK-NEXT:    .cfi_restore z14
+; CHECK-NEXT:    .cfi_restore z15
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: invoke_callee_may_throw_sve:
 ; GISEL:       .Lfunc_begin0:
 ; GISEL-NEXT:    .cfi_startproc
 ; GISEL-NEXT:  // %bb.0:
 ; GISEL-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; GISEL-NEXT:    .cfi_def_cfa_offset 16

 ; GISEL-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
 ; GISEL-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
 ; GISEL-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
 ; GISEL-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
 ; GISEL-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
 ; GISEL-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
 ; GISEL-NEXT:    addvl sp, sp, #-2
 ; GISEL-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xa0, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 160 * VG
+; GISEL-NEXT:    .cfi_remember_state
 ; GISEL-NEXT:    str z0, [sp] // 16-byte Folded Spill
 ; GISEL-NEXT:  .Ltmp0:
 ; GISEL-NEXT:    bl may_throw_sve
 ; GISEL-NEXT:  .Ltmp1:
 ; GISEL-NEXT:    str z0, [sp, #1, mul vl] // 16-byte Folded Spill
 ; GISEL-NEXT:    b .LBB0_1
 ; GISEL-NEXT:  .LBB0_1: // %.Lcontinue
 ; GISEL-NEXT:    ldr z0, [sp, #1, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #2
+; GISEL-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; GISEL-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; GISEL-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #18
+; GISEL-NEXT:    .cfi_def_cfa wsp, 16
+; GISEL-NEXT:    .cfi_restore z8
+; GISEL-NEXT:    .cfi_restore z9
+; GISEL-NEXT:    .cfi_restore z10
+; GISEL-NEXT:    .cfi_restore z11
+; GISEL-NEXT:    .cfi_restore z12
+; GISEL-NEXT:    .cfi_restore z13
+; GISEL-NEXT:    .cfi_restore z14
+; GISEL-NEXT:    .cfi_restore z15
 ; GISEL-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
 ; GISEL-NEXT:    ret
 ; GISEL-NEXT:  .LBB0_2: // %.Lunwind
+; GISEL-NEXT:    .cfi_restore_state
 ; GISEL-NEXT:  .Ltmp2:
 ; GISEL-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #2
+; GISEL-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; GISEL-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
 ; GISEL-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload

 ; GISEL-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
 ; GISEL-NEXT:    addvl sp, sp, #18
+; GISEL-NEXT:    .cfi_def_cfa wsp, 16
+; GISEL-NEXT:    .cfi_restore z8
+; GISEL-NEXT:    .cfi_restore z9
+; GISEL-NEXT:    .cfi_restore z10
+; GISEL-NEXT:    .cfi_restore z11
+; GISEL-NEXT:    .cfi_restore z12
+; GISEL-NEXT:    .cfi_restore z13
+; GISEL-NEXT:    .cfi_restore z14
+; GISEL-NEXT:    .cfi_restore z15
 ; GISEL-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
 ; GISEL-NEXT:    ret
   %result = invoke <vscale x 4 x i32> @may_throw_sve(<vscale x 4 x i32> %v) to label %.Lcontinue unwind label %.Lunwind
 .Lcontinue:
   ret <vscale x 4 x i32> %result
 .Lunwind:
   %lp = landingpad { i8*, i32 } cleanup
   ret <vscale x 4 x i32> %v;
 }
 
 declare <vscale x 4 x i32> @may_throw_sve(<vscale x 4 x i32> %v);
 
 
 ; Test that q0 is saved/restored, as the unwinder may only retain the low 64bits (d0).
-define aarch64_vector_pcs <4 x i32> @invoke_callee_may_throw_neon(<4 x i32> %v) personality i8 0 {
+define aarch64_vector_pcs <4 x i32> @invoke_callee_may_throw_neon(<4 x i32> %v) uwtable personality i8 0 {
 ; CHECK-LABEL: invoke_callee_may_throw_neon:
 ; CHECK:       .Lfunc_begin1:
 ; CHECK-NEXT:    .cfi_startproc
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #304
 ; CHECK-NEXT:    .cfi_def_cfa_offset 304
 ; CHECK-NEXT:    stp q23, q22, [sp, #32] // 32-byte Folded Spill
 ; CHECK-NEXT:    stp q21, q20, [sp, #64] // 32-byte Folded Spill

 ; CHECK-NEXT:    .cfi_offset b16, -160
 ; CHECK-NEXT:    .cfi_offset b17, -176
 ; CHECK-NEXT:    .cfi_offset b18, -192
 ; CHECK-NEXT:    .cfi_offset b19, -208
 ; CHECK-NEXT:    .cfi_offset b20, -224
 ; CHECK-NEXT:    .cfi_offset b21, -240
 ; CHECK-NEXT:    .cfi_offset b22, -256
 ; CHECK-NEXT:    .cfi_offset b23, -272
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:  .Ltmp3:
 ; CHECK-NEXT:    bl may_throw_neon
 ; CHECK-NEXT:  .Ltmp4:
 ; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    b .LBB1_1
 ; CHECK-NEXT:  .LBB1_1: // %.Lcontinue
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp q9, q8, [sp, #256] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #304
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
+; CHECK-NEXT:    .cfi_restore b16
+; CHECK-NEXT:    .cfi_restore b17
+; CHECK-NEXT:    .cfi_restore b18
+; CHECK-NEXT:    .cfi_restore b19
+; CHECK-NEXT:    .cfi_restore b20
+; CHECK-NEXT:    .cfi_restore b21
+; CHECK-NEXT:    .cfi_restore b22
+; CHECK-NEXT:    .cfi_restore b23
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB1_2: // %.Lunwind
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:  .Ltmp5:
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp q9, q8, [sp, #256] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; CHECK-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #304
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
+; CHECK-NEXT:    .cfi_restore b16
+; CHECK-NEXT:    .cfi_restore b17
+; CHECK-NEXT:    .cfi_restore b18
+; CHECK-NEXT:    .cfi_restore b19
+; CHECK-NEXT:    .cfi_restore b20
+; CHECK-NEXT:    .cfi_restore b21
+; CHECK-NEXT:    .cfi_restore b22
+; CHECK-NEXT:    .cfi_restore b23
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: invoke_callee_may_throw_neon:
 ; GISEL:       .Lfunc_begin1:
 ; GISEL-NEXT:    .cfi_startproc
 ; GISEL-NEXT:  // %bb.0:
 ; GISEL-NEXT:    sub sp, sp, #304
 ; GISEL-NEXT:    .cfi_def_cfa_offset 304

 ; GISEL-NEXT:    .cfi_offset b16, -160
 ; GISEL-NEXT:    .cfi_offset b17, -176
 ; GISEL-NEXT:    .cfi_offset b18, -192
 ; GISEL-NEXT:    .cfi_offset b19, -208
 ; GISEL-NEXT:    .cfi_offset b20, -224
 ; GISEL-NEXT:    .cfi_offset b21, -240
 ; GISEL-NEXT:    .cfi_offset b22, -256
 ; GISEL-NEXT:    .cfi_offset b23, -272
+; GISEL-NEXT:    .cfi_remember_state
 ; GISEL-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; GISEL-NEXT:  .Ltmp3:
 ; GISEL-NEXT:    bl may_throw_neon
 ; GISEL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; GISEL-NEXT:  .Ltmp4:
 ; GISEL-NEXT:    b .LBB1_1
 ; GISEL-NEXT:  .LBB1_1: // %.Lcontinue
 ; GISEL-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp q9, q8, [sp, #256] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; GISEL-NEXT:    add sp, sp, #304
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
+; GISEL-NEXT:    .cfi_restore b8
+; GISEL-NEXT:    .cfi_restore b9
+; GISEL-NEXT:    .cfi_restore b10
+; GISEL-NEXT:    .cfi_restore b11
+; GISEL-NEXT:    .cfi_restore b12
+; GISEL-NEXT:    .cfi_restore b13
+; GISEL-NEXT:    .cfi_restore b14
+; GISEL-NEXT:    .cfi_restore b15
+; GISEL-NEXT:    .cfi_restore b16
+; GISEL-NEXT:    .cfi_restore b17
+; GISEL-NEXT:    .cfi_restore b18
+; GISEL-NEXT:    .cfi_restore b19
+; GISEL-NEXT:    .cfi_restore b20
+; GISEL-NEXT:    .cfi_restore b21
+; GISEL-NEXT:    .cfi_restore b22
+; GISEL-NEXT:    .cfi_restore b23
 ; GISEL-NEXT:    ret
 ; GISEL-NEXT:  .LBB1_2: // %.Lunwind
+; GISEL-NEXT:    .cfi_restore_state
 ; GISEL-NEXT:  .Ltmp5:
 ; GISEL-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp q9, q8, [sp, #256] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q11, q10, [sp, #224] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q13, q12, [sp, #192] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q15, q14, [sp, #160] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q17, q16, [sp, #128] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q19, q18, [sp, #96] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q21, q20, [sp, #64] // 32-byte Folded Reload
 ; GISEL-NEXT:    ldp q23, q22, [sp, #32] // 32-byte Folded Reload
 ; GISEL-NEXT:    add sp, sp, #304
+; GISEL-NEXT:    .cfi_def_cfa_offset 0
+; GISEL-NEXT:    .cfi_restore w30
+; GISEL-NEXT:    .cfi_restore w29
+; GISEL-NEXT:    .cfi_restore b8
+; GISEL-NEXT:    .cfi_restore b9
+; GISEL-NEXT:    .cfi_restore b10
+; GISEL-NEXT:    .cfi_restore b11
+; GISEL-NEXT:    .cfi_restore b12
+; GISEL-NEXT:    .cfi_restore b13
+; GISEL-NEXT:    .cfi_restore b14
+; GISEL-NEXT:    .cfi_restore b15
+; GISEL-NEXT:    .cfi_restore b16
+; GISEL-NEXT:    .cfi_restore b17
+; GISEL-NEXT:    .cfi_restore b18
+; GISEL-NEXT:    .cfi_restore b19
+; GISEL-NEXT:    .cfi_restore b20
+; GISEL-NEXT:    .cfi_restore b21
+; GISEL-NEXT:    .cfi_restore b22
+; GISEL-NEXT:    .cfi_restore b23
 ; GISEL-NEXT:    ret
   %result = invoke aarch64_vector_pcs <4 x i32> @may_throw_neon(<4 x i32> %v) to label %.Lcontinue unwind label %.Lunwind
 .Lcontinue:
   ret <4 x i32> %result
 .Lunwind:
   %lp = landingpad { i8*, i32 } cleanup
   ret <4 x i32> %v;
 }
 
 declare aarch64_vector_pcs <4 x i32> @may_throw_neon(<4 x i32> %v);

llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll

 
 while_end:
   ret void
 }
 
 declare %struct_type* @foo()
 declare void @foo2()
 
-define void @test4(i32 %n) personality i32 (...)* @__FrameHandler {
+define void @test4(i32 %n) uwtable personality i32 (...)* @__FrameHandler {
 ; CHECK-LABEL: test4:
 ; CHECK:       .Lfunc_begin0:
 ; CHECK-NEXT:    .cfi_startproc
 ; CHECK-NEXT:    .cfi_personality 0, __FrameHandler
 ; CHECK-NEXT:    .cfi_lsda 0, .Lexception0
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
 ; CHECK-NEXT:    .cfi_offset w21, -24
 ; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    mov w20, wzr
 ; CHECK-NEXT:    mov w21, #40000
 ; CHECK-NEXT:  .LBB3_1: // %while_cond
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    bl foo
 ; CHECK-NEXT:  .Ltmp1:
 ; CHECK-NEXT:  // %bb.2: // %while_cond_x.split
 ; CHECK-NEXT:    // in Loop: Header=BB3_1 Depth=1
 ; CHECK-NEXT:    add x8, x0, x21
 ; CHECK-NEXT:    cmp w20, w19
 ; CHECK-NEXT:    str wzr, [x8]
 ; CHECK-NEXT:    b.ge .LBB3_4
 ; CHECK-NEXT:  // %bb.3: // %while_body
 ; CHECK-NEXT:    // in Loop: Header=BB3_1 Depth=1
 ; CHECK-NEXT:    add w9, w20, #1
 ; CHECK-NEXT:    stp w9, w20, [x8]
 ; CHECK-NEXT:    mov w20, w9
 ; CHECK-NEXT:    b .LBB3_1
 ; CHECK-NEXT:  .LBB3_4: // %while_end
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore w21
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB3_5: // %cleanup
+; CHECK-NEXT:    .cfi_restore_state
 ; CHECK-NEXT:  .Ltmp2:
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl foo2
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    bl _Unwind_Resume
 entry:
   br label %while_cond
 

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.generated.expected

 ; CHECK-NEXT:  .LBB0_3:
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    ldur w8, [x29, #-8]
 ; CHECK-NEXT:    cbnz w8, .LBB0_2
 ; CHECK-NEXT:  .LBB0_4:
 ; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:  .LBB0_5:
-; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
+; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-LABEL: main:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #48
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    stp x29, x30, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x29, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa w29, 16
 ; CHECK-NEXT:    .cfi_offset w30, -8
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    adrp x9, x
 ; CHECK-NEXT:    mov w10, #2
 ; CHECK-NEXT:    mov w11, #3
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    str w8, [x9, :lo12:x]
 ; CHECK-NEXT:    mov w9, #4
 ; CHECK-NEXT:    stp w8, wzr, [x29, #-8]
 ; CHECK-NEXT:    stur w10, [x29, #-12]
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
 ; CHECK-NEXT:    stp w10, w8, [x29, #-12]
-; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
+; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-LABEL: OUTLINED_FUNCTION_0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w9, #2
 ; CHECK-NEXT:    mov w10, #3
 ; CHECK-NEXT:    mov w11, #4
 ; CHECK-NEXT:    stp w9, w8, [x29, #-12]
 ; CHECK-NEXT:    stp w11, w10, [sp, #12]
 ; CHECK-NEXT:    ret

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/aarch64_generated_funcs.ll.nogenerated.expected

 ; CHECK-NEXT:  .LBB0_3:
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    ldur w8, [x29, #-8]
 ; CHECK-NEXT:    cbnz w8, .LBB0_2
 ; CHECK-NEXT:  .LBB0_4:
 ; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:  .LBB0_5:
-; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
+; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
   %4 = alloca i32, align 4
   %5 = alloca i32, align 4
   store i32 0, i32* %1, align 4
   store i32 0, i32* %2, align 4

 ; CHECK-NEXT:    str w8, [x9, :lo12:x]
 ; CHECK-NEXT:    mov w9, #4
 ; CHECK-NEXT:    stp w8, wzr, [x29, #-8]
 ; CHECK-NEXT:    stur w10, [x29, #-12]
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
 ; CHECK-NEXT:    stp w10, w8, [x29, #-12]
-; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    stp w9, w11, [sp, #12]
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
+; CHECK-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
   %1 = alloca i32, align 4
   %2 = alloca i32, align 4
   %3 = alloca i32, align 4
   %4 = alloca i32, align 4
   %5 = alloca i32, align 4
 
   store i32 0, i32* %1, align 4