[AArch64] Emit some .cfi_* for non-SVE cases in emitEpilogue

[MaskRay]

Summary

WIP.

Ideally .cfi_def_offset should be used instead. .cfi_adjust_cfa_offset is used for now.

There are many epilogue code paths where .cfi_* directives are missing.
They need separate fixes.

Something is not correct yet (e.g. addsub-constant-folding.ll)

[MaskRay]

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

1688

SVE callee save stack seems very difficult, so not in the scope of this patch.

llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll

279

This is incorrect.

1490

This is apparently incorrect.

[MaskRay]

fix some issues

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

 }
 
 // Convert callee-save register save/restore instruction to do stack pointer
 // decrement/increment to allocate/deallocate the callee-save stack area by
 // converting store/load to use pre/post increment version.
 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
-    bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) {
+    bool NeedsDwarfCFI, bool NeedsWinCFI, bool *HasWinCFI,
+    bool InProlog = true) {
   // Ignore instructions that do not operate on SP, i.e. shadow call stack
   // instructions and associated CFI instruction.
   while (MBBI->getOpcode() == AArch64::STRXpost ||
          MBBI->getOpcode() == AArch64::LDRXpre ||
          MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
     if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
       assert(MBBI->getOperand(0).getReg() != AArch64::SP);
     ++MBBI;
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
   assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
          "Unexpected base register in callee-save save/restore instruction!");
   assert(CSStackSizeInc % Scale == 0);
   MIB.addImm(CSStackSizeInc / (int)Scale);
 
   MIB.setMIFlags(MBBI->getFlags());
   MIB.setMemRefs(MBBI->memoperands());
 
+  if (NeedsDwarfCFI && !InProlog) {
+    unsigned CFIIndex = MBB.getParent()->addFrameInst(
+        MCCFIInstruction::createAdjustCfaOffset(nullptr, -CSStackSizeInc));
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex);
+  }
+
   // Generate a new SEH code that corresponds to the new instruction.
   if (NeedsWinCFI) {
     *HasWinCFI = true;
     InsertSEH(*MIB, *TII,
               InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy);
   }
 
   return std::prev(MBB.erase(MBBI));
 }
 
 // Fixup callee-save register save/restore instructions to take into account
 // combined SP bump by adding the local stack size to the stack offsets.
-static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
+static bool fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
                                               uint64_t LocalStackSize,
                                               bool NeedsWinCFI,
                                               bool *HasWinCFI) {
   if (AArch64InstrInfo::isSEHInstruction(MI))
-    return;
+    return false;
 
   unsigned Opc = MI.getOpcode();
 
   // Ignore instructions that do not operate on SP, i.e. shadow call stack
   // instructions and associated CFI instruction.
   if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
       Opc == AArch64::CFI_INSTRUCTION) {
     if (Opc != AArch64::CFI_INSTRUCTION)
       assert(MI.getOperand(0).getReg() != AArch64::SP);
-    return;
+    return false;
   }
 
   unsigned Scale;
   switch (Opc) {
   case AArch64::STPXi:
   case AArch64::STRXui:
   case AArch64::STPDi:
   case AArch64::STRDui:
static bool fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
   if (NeedsWinCFI) {
     *HasWinCFI = true;
     auto MBBI = std::next(MachineBasicBlock::iterator(MI));
     assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
     assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
            "Expecting a SEH instruction");
     fixupSEHOpcode(MBBI, LocalStackSize);
   }
+  return true;
 }
 
 static void adaptForLdStOpt(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator FirstSPPopI,
                             MachineBasicBlock::iterator LastPopI) {
   // Sometimes (when we restore in the same order as we save), we can end up
   // with code like this:
   //
emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
                     StackOffset::getFixed(-NumBytes), TII,
                     MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
     NumBytes = 0;
   } else if (HomPrologEpilog) {
     // Stack has been already adjusted.
     NumBytes -= PrologueSaveSize;
   } else if (PrologueSaveSize != 0) {
     MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
-        MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI);
+        MBB, MBBI, DL, TII, -PrologueSaveSize, false, NeedsWinCFI, &HasWinCFI);
     NumBytes -= PrologueSaveSize;
   }
   assert(NumBytes >= 0 && "Negative stack allocation size!?");
 
   // Move past the saves of the callee-saved registers, fixing up the offsets
   // and pre-inc if we decided to combine the callee-save and local stack
   // pointer bump above.
   MachineBasicBlock::iterator End = MBB.end();
void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
   if (MBB.end() != MBBI) {
     DL = MBBI->getDebugLoc();
     IsFunclet = isFuncletReturnInstr(*MBBI);
   }
 
   int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
                                : MFI.getStackSize();
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+  // TODO Also emit CFI when SVECalleeSavedStackSize != 0.
+  bool NeedsDwarfCFI = MF.getTarget().getTargetTriple().isOSBinFormatELF() &&
+                       MF.getFunction().needsUnwindTableEntry() &&
+                       AFI->getSVECalleeSavedStackSize() == 0;
 
   // All calls are tail calls in GHC calling conv, and functions have no
   // prologue/epilogue.
   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     return;
 
   // How much of the stack used by incoming arguments this function is expected
   // to restore in this particular epilogue.
if (!CombineSPBump && PrologueSaveSize != 0) {
     while (AArch64InstrInfo::isSEHInstruction(*Pop))
       Pop = std::prev(Pop);
     // Converting the last ldp to a post-index ldp is valid only if the last
     // ldp's offset is 0.
     const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
     // If the offset is 0 and the AfterCSR pop is not actually trying to
     // allocate more stack for arguments (in space that an untimely interrupt
     // may clobber), convert it to a post-index ldp.
-    if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0)
-      convertCalleeSaveRestoreToSPPrePostIncDec(
-          MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false);
-    else {
+    if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
+      convertCalleeSaveRestoreToSPPrePostIncDec(MBB, Pop, DL, TII,
+                                                PrologueSaveSize, NeedsDwarfCFI,
+                                                NeedsWinCFI, &HasWinCFI, false);
+    } else {
       // If not, make sure to emit an add after the last ldp.
       // We're doing this by transfering the size to be restored from the
       // adjustment *before* the CSR pops to the adjustment *after* the CSR
       // pops.
       AfterCSRPopSize += PrologueSaveSize;
     }
   }
 
+  bool EmitCfi = NeedsDwarfCFI && !hasFP(MF);
   // Move past the restores of the callee-saved registers.
   // If we plan on combining the sp bump of the local stack size and the callee
   // save stack size, we might need to adjust the CSR save and restore offsets.
   MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
   MachineBasicBlock::iterator Begin = MBB.begin();
   while (LastPopI != Begin) {
     --LastPopI;
     if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
         IsSVECalleeSave(LastPopI)) {
       ++LastPopI;
       break;
-    } else if (CombineSPBump)
-      fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
-                                        NeedsWinCFI, &HasWinCFI);
+    } else if (CombineSPBump) {
+      if (fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
+                                            NeedsWinCFI, &HasWinCFI) &&
+          EmitCfi) {
+        unsigned CFIIndex =
+            MF.addFrameInst(MCCFIInstruction::createAdjustCfaOffset(
+                nullptr, AFI->getLocalStackSize()));
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+            .addCFIIndex(CFIIndex);
+      }
+    }
   }
 
   if (MF.hasWinCFI()) {
     // If the prologue didn't contain any SEH opcodes and didn't set the
     // MF.hasWinCFI() flag, assume the epilogue won't either, and skip the
     // EpilogStart - to avoid generating CFI for functions that don't need it.
     // (And as we didn't generate any prologue at all, it would be asymmetrical
     // to the epilogue.) By the end of the function, we assert that

llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
 
 declare void @byval_i32(i32* byval(i32) %ptr)
 
 define void @call_byval_i32(i32* %incoming) {
 ; CHECK-LABEL: call_byval_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    str w8, [sp]
 ; CHECK-NEXT:    bl byval_i32
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   call void @byval_i32(i32* byval(i32) %incoming)
   ret void
 }
 
 declare void @byval_a64i32([64 x i32]* byval([64 x i32]) %ptr)
 

llvm/test/CodeGen/AArch64/addsub-constant-folding.ll

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    add w0, w19, #10
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = add i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_add_const_add_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    movi v0.4s, #10
 ; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = add <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = add <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    add w0, w19, #6
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_add_const_sub_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    movi v0.4s, #6
 ; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = add <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #-6
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = add i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 2, %t0
   ret i32 %t1
 }
 
 define <4 x i32> @vec_add_const_const_sub(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    mvni v0.4s, #5
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = add <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> <i32 2, i32 2, i32 2, i32 2>, %t0
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    sub w0, w19, #6
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = add i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_sub_const_add_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    mvni v0.4s, #5
 ; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = add <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    sub w0, w19, #10
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_sub_const_sub_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    movi v0.4s, #10
 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w0, #8
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #10
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = sub i32 %arg, 8
   call void @use(i32 %t0)
   %t1 = sub i32 2, %t0
   ret i32 %t1
 }
 
 define <4 x i32> @vec_sub_const_const_sub(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    movi v0.4s, #2
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> %arg, <i32 8, i32 8, i32 8, i32 8>
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> <i32 2, i32 2, i32 2, i32 2>, %t0
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w8, w0
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #10
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
   %t1 = add i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_const_sub_add_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    movi v0.4s, #10
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> <i32 8, i32 8, i32 8, i32 8>, %arg
   call void @vec_use(<4 x i32> %t0)
   %t1 = add <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    mov w19, w0
 ; CHECK-NEXT:    sub w0, w8, w0
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #6
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
   %t1 = sub i32 %t0, 2
   ret i32 %t1
 }
 
 define <4 x i32> @vec_const_sub_sub_const(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    movi v0.4s, #6
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> <i32 8, i32 8, i32 8, i32 8>, %arg
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> %t0, <i32 2, i32 2, i32 2, i32 2>
   ret <4 x i32> %t1
 }
 

 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w8, #8
 ; CHECK-NEXT:    sub w19, w8, w0
 ; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    bl use
 ; CHECK-NEXT:    mov w8, #2
 ; CHECK-NEXT:    sub w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %t0 = sub i32 8, %arg
   call void @use(i32 %t0)
   %t1 = sub i32 2, %t0
   ret i32 %t1
 }
 
 define <4 x i32> @vec_const_sub_const_sub(<4 x i32> %arg) {

 ; CHECK-NEXT:    movi v1.4s, #8
 ; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl vec_use
 ; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    movi v0.4s, #2
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %t0 = sub <4 x i32> <i32 8, i32 8, i32 8, i32 8>, %arg
   call void @vec_use(<4 x i32> %t0)
   %t1 = sub <4 x i32> <i32 2, i32 2, i32 2, i32 2>, %t0
   ret <4 x i32> %t1
 }
 

llvm/test/CodeGen/AArch64/arm64-atomic-128.ll

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_acq
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_adjust_cfa_offset -16
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_acq_rel
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_adjust_cfa_offset -16
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap_seqcst:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_rel
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_adjust_cfa_offset -16
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap_release:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

 ; OUTLINE-NEXT:    mov x1, x3
 ; OUTLINE-NEXT:    mov x8, x0
 ; OUTLINE-NEXT:    mov x0, x2
 ; OUTLINE-NEXT:    mov x2, x4
 ; OUTLINE-NEXT:    mov x3, x5
 ; OUTLINE-NEXT:    mov x4, x8
 ; OUTLINE-NEXT:    bl __aarch64_cas16_relax
 ; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    .cfi_adjust_cfa_offset -16
 ; OUTLINE-NEXT:    ret
 ;
 ; LSE-LABEL: val_compare_and_swap_monotonic:
 ; LSE:       // %bb.0:
 ; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
 ; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
 ; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3

llvm/test/CodeGen/AArch64/arm64-fp128.ll

 ; CHECK-NEXT:    bl __fixtfsi
 ; CHECK-NEXT:    adrp x8, var32
 ; CHECK-NEXT:    str w0, [x8, :lo12:var32]
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __fixtfdi
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    str x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %val = load fp128, fp128* @lhs, align 16
 
   %val32 = fptosi fp128 %val to i32
   store i32 %val32, i32* @var32
 
   %val64 = fptosi fp128 %val to i64

 ; CHECK-NEXT:    bl __fixunstfsi
 ; CHECK-NEXT:    adrp x8, var32
 ; CHECK-NEXT:    str w0, [x8, :lo12:var32]
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __fixunstfdi
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    str x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %val = load fp128, fp128* @lhs, align 16
 
   %val32 = fptoui fp128 %val to i32
   store i32 %val32, i32* @var32
 
   %val64 = fptoui fp128 %val to i64

 ; CHECK-NEXT:    bl __floatsitf
 ; CHECK-NEXT:    adrp x19, lhs
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    ldr x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    bl __floatditf
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 
   %src32 = load i32, i32* @var32
   %val32 = sitofp i32 %src32 to fp128
   store volatile fp128 %val32, fp128* @lhs
 
   %src64 = load i64, i64* @var64
   %val64 = sitofp i64 %src64 to fp128

 ; CHECK-NEXT:    bl __floatunsitf
 ; CHECK-NEXT:    adrp x19, lhs
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, var64
 ; CHECK-NEXT:    ldr x0, [x8, :lo12:var64]
 ; CHECK-NEXT:    bl __floatunditf
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 
   %src32 = load i32, i32* @var32
   %val32 = uitofp i32 %src32 to fp128
   store volatile fp128 %val32, fp128* @lhs
 
   %src64 = load i64, i64* @var64
   %val64 = uitofp i64 %src64 to fp128

 ; CHECK-NEXT:    adrp x8, lhs
 ; CHECK-NEXT:    ldr q0, [x8, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, rhs
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:rhs]
 ; CHECK-NEXT:    bl __letf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w0, le
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
 ; Technically, everything after the call to __letf2 is redundant, but we'll let
 ; LLVM have its fun for now.
   %val = fcmp ole fp128 %lhs, %rhs

 ; CHECK-NEXT:    adrp x8, lhs
 ; CHECK-NEXT:    ldr q0, [x8, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, rhs
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:rhs]
 ; CHECK-NEXT:    bl __letf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w0, gt
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
   %val = fcmp ugt fp128 %lhs, %rhs
 
   ret i1 %val

 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w19, eq
 ; CHECK-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
 ; CHECK-NEXT:    bl __unordtf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    cset w8, ne
 ; CHECK-NEXT:    orr w0, w8, w19
 ; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 32
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
   %val = fcmp ueq fp128 %lhs, %rhs
 

 ; CHECK-NEXT:    adrp x8, rhs
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:rhs]
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    b.ge .LBB11_2
 ; CHECK-NEXT:  // %bb.1: // %iftrue
 ; CHECK-NEXT:    mov w0, #42
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB11_2: // %iffalse
 ; CHECK-NEXT:    mov w0, #29
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 
   %lhs = load fp128, fp128* @lhs, align 16
   %rhs = load fp128, fp128* @rhs, align 16
 
   ; olt == !uge, which LLVM optimizes this to.
   %cond = fcmp olt fp128 %lhs, %rhs
   br i1 %cond, label %iftrue, label %iffalse

 ; CHECK-NEXT:    bl __trunctfsf2
 ; CHECK-NEXT:    adrp x8, varfloat
 ; CHECK-NEXT:    str s0, [x8, :lo12:varfloat]
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
 ; CHECK-NEXT:    adrp x8, vardouble
 ; CHECK-NEXT:    str d0, [x8, :lo12:vardouble]
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
 
   %val = load fp128, fp128* @lhs, align 16
 
   %half = fptrunc fp128 %val to half
   store half %half, half* @varhalf, align 2
 

 ; CHECK-NEXT:    ldr s0, [x8, :lo12:varfloat]
 ; CHECK-NEXT:    bl __extendsftf2
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    adrp x8, vardouble
 ; CHECK-NEXT:    ldr d0, [x8, :lo12:vardouble]
 ; CHECK-NEXT:    bl __extenddftf2
 ; CHECK-NEXT:    str q0, [x19, :lo12:lhs]
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 
   %val = load fp128, fp128* @lhs, align 16
 
   %half = load half, half* @varhalf
   %fromhalf = fpext half %half to fp128
   store volatile fp128 %fromhalf, fp128* @lhs, align 16
 

llvm/test/CodeGen/AArch64/cmp-select-sign.ll

 ; CHECK-NEXT:    sshr v2.4s, v0.4s, #31
 ; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    orr v2.4s, #1
 ; CHECK-NEXT:    xtn v0.4h, v0.4s
 ; CHECK-NEXT:    str q2, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl use_4xi1
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %c = icmp sgt <4 x i32> %a, <i32 -1, i32 -1, i32 -1, i32 -1>
   %res = select <4 x i1> %c, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
   call void @use_4xi1(<4 x i1> %c)
   ret <4 x i32> %res
 }
 

llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll

 ; CHECK-NEXT:    bl Update
 ; CHECK-NEXT:    sub x21, x21, #2
 ; CHECK-NEXT:    cmp x19, x21
 ; CHECK-NEXT:    b.lt .LBB6_1
 ; CHECK-NEXT:  .LBB6_3: // %while.end
 ; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -48
 ; CHECK-NEXT:    ret
 entry:
   %size = getelementptr inbounds %struct.Struct, %struct.Struct* %hdCall, i64 0, i32 0
   %0 = load i64, i64* %size, align 8
   br label %land.rhs
 
 land.rhs:
   %rp.06 = phi i64 [ %0, %entry ], [ %sub, %while.body ]

 ; CHECK-NEXT:  // %bb.6:
 ; CHECK-NEXT:    mov w0, #123
 ; CHECK-NEXT:    b .LBB7_8
 ; CHECK-NEXT:  .LBB7_7: // %if.end
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:  .LBB7_8: // %return
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, i32* @a, align 4
   %cmp4 = icmp slt i32 %0, -1
   br i1 %cmp4, label %while.body.preheader, label %while.end
 
 while.body.preheader:                             ; preds = %entry
   br label %while.body

 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
 ; CHECK-NEXT:    b.ne .LBB8_6
 ; CHECK-NEXT:  // %bb.5:
 ; CHECK-NEXT:    mov w0, #123
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB8_6: // %if.end
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, i32* @a, align 4
   %cmp4 = icmp slt i32 %0, 1
   br i1 %cmp4, label %while.body.preheader, label %while.end
 
 while.body.preheader:                             ; preds = %entry
   br label %while.body

 ; CHECK-NEXT:    fcmp d8, #0.0
 ; CHECK-NEXT:    fcsel d0, d8, d0, gt
 ; CHECK-NEXT:    fmov d1, #-2.00000000
 ; CHECK-NEXT:    bl woo
 ; CHECK-NEXT:    mov w0, #4
 ; CHECK-NEXT:  .LBB9_4: // %return
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
 ; CHECK-NEXT:    ret
 
 ; CHECK-LABEL-DAG: .LBB9_3
 
 entry:
   %cmp = icmp sgt i32 %argc, 1
   br i1 %cmp, label %land.lhs.true, label %if.end
 

 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    mov w8, #42
 ; CHECK-NEXT:    csinc w8, w8, wzr, gt
 ; CHECK-NEXT:    cmp w0, #2, lsl #12 // =8192
 ; CHECK-NEXT:    mov w9, #128
 ; CHECK-NEXT:    csel w0, w9, w8, ge
 ; CHECK-NEXT:    bl zoo
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 ; [...]
 
   %tst_low = icmp sgt i32 %in, 8191
   br i1 %tst_low, label %true, label %false
 
 true:
   call i32 @zoo(i32 128)

llvm/test/CodeGen/AArch64/cond-br-tuning.ll

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    and w19, w1, #0x6
 ; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    cbnz w19, .LBB9_2
 ; CHECK-NEXT:  // %bb.1: // %if.end
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB9_2: // %if.then
 ; CHECK-NEXT:    bl foo
 entry:
   %c = and i32 %a, 6
   call void @bar(i32 %c)
   %tobool = icmp eq i32 %c, 0
   br i1 %tobool, label %if.end, label %if.then
 
 if.then:
   tail call void @foo()
   unreachable
 
 if.end:
   ret void
 }

llvm/test/CodeGen/AArch64/csr-split.ll

 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x8, a
 ; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
 ; CHECK-NEXT:    cmp x8, x0
 ; CHECK-NEXT:    b.eq .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %if.end
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %if.then
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl callVoid
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    b callNonVoid
 ;
 ; CHECK-APPLE-LABEL: test1:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8

 ; CHECK-NEXT:    adrp x8, a
 ; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    cmp x8, x0
 ; CHECK-NEXT:    b.eq .LBB1_3
 ; CHECK-NEXT:  .LBB1_2: // %return
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB1_3: // %if.then2
 ; CHECK-NEXT:    bl callVoid
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    b callNonVoid
 ;
 ; CHECK-APPLE-LABEL: test2:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8

 ; CHECK-NEXT:    mov x20, x0
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    str x0, [x20]
 ; CHECK-NEXT:  .LBB2_2: // %land.end
 ; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-APPLE-LABEL: test3:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
 ; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
 ; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-APPLE-NEXT:    .cfi_offset w30, -8

llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll

 ; CHECK-NEXT:    mov w8, #2147483647
 ; CHECK-NEXT:    csel w19, w8, w19, gt
 ; CHECK-NEXT:    mov v1.16b, v0.16b
 ; CHECK-NEXT:    bl __unordtf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
     %x = call <1 x i32> @llvm.fptosi.sat.v1f128.v1i32(<1 x fp128> %f)
     ret <1 x i32> %x
 }
 
 define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-LABEL: test_signed_v2f128_v2i32:

 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    mov v0.s[1], w22
 ; CHECK-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 64
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 64
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 64
 ; CHECK-NEXT:    add sp, sp, #112
 ; CHECK-NEXT:    ret
     %x = call <2 x i32> @llvm.fptosi.sat.v2f128.v2i32(<2 x fp128> %f)
     ret <2 x i32> %x
 }
 
 define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-LABEL: test_signed_v3f128_v3i32:

 ; CHECK-NEXT:    bl __unordtf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[2], w8
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 80
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 80
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 80
 ; CHECK-NEXT:    add sp, sp, #128
 ; CHECK-NEXT:    ret
     %x = call <3 x i32> @llvm.fptosi.sat.v3f128.v3i32(<3 x fp128> %f)
     ret <3 x i32> %x
 }
 
 define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-LABEL: test_signed_v4f128_v4i32:

 ; CHECK-NEXT:    bl __unordtf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csel w8, wzr, w19, ne
 ; CHECK-NEXT:    ldp x20, x19, [sp, #128] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[3], w8
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 96
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 96
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 96
 ; CHECK-NEXT:    add sp, sp, #144
 ; CHECK-NEXT:    ret
     %x = call <4 x i32> @llvm.fptosi.sat.v4f128.v4i32(<4 x fp128> %f)
     ret <4 x i32> %x
 }
 
 ;
 ; FP16 to signed 32-bit -- Vector size variation

 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptosi.sat.v2f32.v2i100(<2 x float> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_signed_v2f32_v2i128(<2 x float> %f) {
 ; CHECK-LABEL: test_signed_v2f32_v2i128:

 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptosi.sat.v2f32.v2i128(<2 x float> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 2-Vector double to signed integer -- result size variation

 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptosi.sat.v2f64.v2i100(<2 x double> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_signed_v2f64_v2i128(<2 x double> %f) {
 ; CHECK-LABEL: test_signed_v2f64_v2i128:

 ; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptosi.sat.v2f64.v2i128(<2 x double> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 4-Vector half to signed integer -- result size variation

 ; CHECK-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #112
 ; CHECK-NEXT:    ret
     %x = call <4 x i100> @llvm.fptosi.sat.v4f16.v4i100(<4 x half> %f)
     ret <4 x i100> %x
 }
 
 define <4 x i128> @test_signed_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-LABEL: test_signed_v4f16_v4i128:

 ; CHECK-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #24] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d10, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #112
 ; CHECK-NEXT:    ret
     %x = call <4 x i128> @llvm.fptosi.sat.v4f16.v4i128(<4 x half> %f)
     ret <4 x i128> %x
 }
 

llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll

 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    cmp w19, #0
 ; CHECK-NEXT:    csel w19, wzr, w0, lt
 ; CHECK-NEXT:    bl __gttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
     %x = call <1 x i32> @llvm.fptoui.sat.v1f128.v1i32(<1 x fp128> %f)
     ret <1 x i32> %x
 }
 
 define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-LABEL: test_unsigned_v2f128_v2i32:

 ; CHECK-NEXT:    bl __gttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    fmov s0, w8
 ; CHECK-NEXT:    mov v0.s[1], w20
 ; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 64
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 64
 ; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
     %x = call <2 x i32> @llvm.fptoui.sat.v2f128.v2i32(<2 x fp128> %f)
     ret <2 x i32> %x
 }
 
 define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-LABEL: test_unsigned_v3f128_v3i32:

 ; CHECK-NEXT:    csel w19, wzr, w0, lt
 ; CHECK-NEXT:    bl __gttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[2], w8
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 80
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 80
 ; CHECK-NEXT:    add sp, sp, #112
 ; CHECK-NEXT:    ret
     %x = call <3 x i32> @llvm.fptoui.sat.v3f128.v3i32(<3 x fp128> %f)
     ret <3 x i32> %x
 }
 
 define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-LABEL: test_unsigned_v4f128_v4i32:

 ; CHECK-NEXT:    csel w19, wzr, w0, lt
 ; CHECK-NEXT:    bl __gttf2
 ; CHECK-NEXT:    cmp w0, #0
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv w8, w19, wzr, le
 ; CHECK-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov v0.s[3], w8
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 96
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 96
 ; CHECK-NEXT:    add sp, sp, #128
 ; CHECK-NEXT:    ret
     %x = call <4 x i32> @llvm.fptoui.sat.v4f128.v4i32(<4 x fp128> %f)
     ret <4 x i32> %x
 }
 
 ;
 ; FP16 to unsigned 32-bit -- Vector size variation

 ; CHECK-NEXT:    fcmp s0, s9
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    csel x1, x21, x9, gt
 ; CHECK-NEXT:    ldp x30, x21, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptoui.sat.v2f32.v2i100(<2 x float> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_unsigned_v2f32_v2i128(<2 x float> %f) {
 ; CHECK-LABEL: test_unsigned_v2f32_v2i128:

 ; CHECK-NEXT:    fcmp s0, s9
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv x1, x9, xzr, le
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptoui.sat.v2f32.v2i128(<2 x float> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 2-Vector double to unsigned integer -- result size variation

 ; CHECK-NEXT:    fcmp d0, d9
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    csel x1, x21, x9, gt
 ; CHECK-NEXT:    ldp x30, x21, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
     %x = call <2 x i100> @llvm.fptoui.sat.v2f64.v2i100(<2 x double> %f)
     ret <2 x i100> %x
 }
 
 define <2 x i128> @test_unsigned_v2f64_v2i128(<2 x double> %f) {
 ; CHECK-LABEL: test_unsigned_v2f64_v2i128:

 ; CHECK-NEXT:    fcmp d0, d9
 ; CHECK-NEXT:    csinv x8, x8, xzr, le
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv x1, x9, xzr, le
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
     %x = call <2 x i128> @llvm.fptoui.sat.v2f64.v2i128(<2 x double> %f)
     ret <2 x i128> %x
 }
 
 ;
 ; 4-Vector half to unsigned integer -- result size variation

 ; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x30, x25, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
     %x = call <4 x i100> @llvm.fptoui.sat.v4f16.v4i100(<4 x half> %f)
     ret <4 x i100> %x
 }
 
 define <4 x i128> @test_unsigned_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-LABEL: test_unsigned_v4f16_v4i128:

 ; CHECK-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    csinv x1, x9, xzr, le
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 16
 ; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
     %x = call <4 x i128> @llvm.fptoui.sat.v4f16.v4i128(<4 x half> %f)
     ret <4 x i128> %x
 }
 

llvm/test/CodeGen/AArch64/ldst-paired-aliasing.ll

 ; CHECK-NEXT:  // %bb.1: // %for.inc
 ; CHECK-NEXT:    bl f
 ; CHECK-NEXT:    b .LBB0_3
 ; CHECK-NEXT:  .LBB0_2: // %if.then
 ; CHECK-NEXT:    bl f2
 ; CHECK-NEXT:  .LBB0_3: // %common.ret
 ; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 96
 ; CHECK-NEXT:    add sp, sp, #112
 ; CHECK-NEXT:    ret
 
 
 for.body.lr.ph.i.i.i.i.i.i63:
   %b1 = alloca [10 x i32], align 16
   %x0 = bitcast [10 x i32]* %b1 to i8*
   %b2 = alloca [10 x i32], align 16

llvm/test/CodeGen/AArch64/local_vars.ll

 
 ; Dont't really care, but it would be a Bad Thing if this came after the epilogue.
 ; CHECK-WITHFP-ARM64: bl foo
 ; CHECK: bl foo
   call void @foo()
   ret void
 
 ; CHECK: ldr x30, [sp], #16
+; CHECK-NEXT: .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT: ret
 
 ; CHECK-WITHFP-ARM64: ldp x29, x30, [sp], #16
+; CHECK-WITHFP-ARM64-NEXT: .cfi_adjust_cfa_offset -16
 ; CHECK-WITHFP-ARM64-NEXT: ret
 }
 
 define void @stack_local() {
   %local_var = alloca i64
 ; CHECK-LABEL: stack_local:
 ; CHECK: sub sp, sp, #16
 

llvm/test/CodeGen/AArch64/logical_shifted_reg.ll

 ; CHECK-NEXT:    str w3, [x8]
 ; CHECK-NEXT:    str w4, [x8]
 ; CHECK-NEXT:    str w5, [x8]
 ; CHECK-NEXT:    str w6, [x8]
 ; CHECK-NEXT:    str w7, [x8]
 ; CHECK-NEXT:    str w19, [x8]
 ; CHECK-NEXT:    str w9, [x8]
 ; CHECK-NEXT:    ldr x19, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %val1 = load i32, i32* @var1_32
   %val2 = load i32, i32* @var2_32
 
   ; First check basic and/bic/or/orn/eor/eon patterns with no shift
   %neg_val2 = xor i32 -1, %val2
 
   %and_noshift = and i32 %val1, %val2

 ; CHECK-NEXT:    str x3, [x8]
 ; CHECK-NEXT:    str x4, [x8]
 ; CHECK-NEXT:    str x5, [x8]
 ; CHECK-NEXT:    str x6, [x8]
 ; CHECK-NEXT:    str x7, [x8]
 ; CHECK-NEXT:    str x19, [x8]
 ; CHECK-NEXT:    str x9, [x8]
 ; CHECK-NEXT:    ldr x19, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %val1 = load i64, i64* @var1_64
   %val2 = load i64, i64* @var2_64
 
   ; First check basic and/bic/or/orn/eor/eon patterns with no shift
   %neg_val2 = xor i64 -1, %val2
 
   %and_noshift = and i64 %val1, %val2

llvm/test/CodeGen/AArch64/machine-licm-sink-instr.ll

 ; CHECK-NEXT:    b.ne .LBB0_2
 ; CHECK-NEXT:    b .LBB0_4
 ; CHECK-NEXT:  .LBB0_3:
 ; CHECK-NEXT:    mov w20, w19
 ; CHECK-NEXT:  .LBB0_4: // %for.cond.cleanup
 ; CHECK-NEXT:    mov w0, w20
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
 ; CHECK-NEXT:    ret
 entry:
   %cmp63 = icmp sgt i32 %n, 0
   br i1 %cmp63, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:
   %0 = load i32, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @A, i64 0, i64 0), align 4
   br label %for.body

 ; CHECK-NEXT:    b.ne .LBB1_2
 ; CHECK-NEXT:    b .LBB1_4
 ; CHECK-NEXT:  .LBB1_3:
 ; CHECK-NEXT:    mov w21, w19
 ; CHECK-NEXT:  .LBB1_4: // %for.cond.cleanup
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov w0, w21
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
 ; CHECK-NEXT:    ret
 entry:
   %cmp63 = icmp sgt i32 %n, 0
   br i1 %cmp63, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:
   %0 = load i32, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @A, i64 0, i64 0), align 4
   %call0 = tail call i32 @_Z3usei(i32 %n)

 ; CHECK-NEXT:    b.ne .LBB2_2
 ; CHECK-NEXT:    b .LBB2_4
 ; CHECK-NEXT:  .LBB2_3:
 ; CHECK-NEXT:    mov w21, w19
 ; CHECK-NEXT:  .LBB2_4: // %for.cond.cleanup
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    mov w0, w21
 ; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
 ; CHECK-NEXT:    ret
 entry:
   %cmp63 = icmp sgt i32 %n, 0
   br i1 %cmp63, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:
   %0 = load i32, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @A, i64 0, i64 0), align 4
   store i32 42, i32* %store, align 4

llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -enable-machine-outliner -verify-machineinstrs | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-pc-linux-gnu"
 
 declare i32 @thunk_called_fn(i32, i32, i32, i32)
 
 define i32 @a() {
 ; CHECK-LABEL: a:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    bl [[OUTLINED_DIRECT:OUTLINED_FUNCTION_[0-9]+]]
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_1
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
   %cx = add i32 %call, 8
   ret i32 %cx
 }
 
 define i32 @b() {
 ; CHECK-LABEL: b:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    bl [[OUTLINED_DIRECT]]
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_1
 ; CHECK-NEXT:    add w0, w0, #88
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 @thunk_called_fn(i32 1, i32 2, i32 3, i32 4)
   %cx = add i32 %call, 88
   ret i32 %cx
 }
 
 define hidden i32 @c(i32 (i32, i32, i32, i32)* %fptr) {
 ; CHECK-LABEL: c:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    bl [[OUTLINED_INDIRECT:OUTLINED_FUNCTION_[0-9]+]]
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    add w0, w0, #8
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 8
   ret i32 %add
 }
 
 define hidden i32 @d(i32 (i32, i32, i32, i32)* %fptr) {
 ; CHECK-LABEL: d:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    bl [[OUTLINED_INDIRECT]]
+; CHECK-NEXT:    bl OUTLINED_FUNCTION_0
 ; CHECK-NEXT:    add w0, w0, #88
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
 entry:
   %call = tail call i32 %fptr(i32 1, i32 2, i32 3, i32 4)
   %add = add nsw i32 %call, 88
   ret i32 %add
 }
 
 ; CHECK: [[OUTLINED_INDIRECT]]:

llvm/test/CodeGen/AArch64/merge-store-dependency.ll

 ; A53-NEXT:    lsl w9, w9, #1
 ; A53-NEXT:    cmp w9, w8
 ; A53-NEXT:    b.le .LBB0_2
 ; A53-NEXT:  // %bb.3: // %while.end.i
 ; A53-NEXT:    bl foo
 ; A53-NEXT:    adrp x8, gv1
 ; A53-NEXT:    str x0, [x8, :lo12:gv1]
 ; A53-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; A53-NEXT:    .cfi_adjust_cfa_offset -16
 ; A53-NEXT:    ret
 ; A53-NEXT:  .LBB0_4: // %while.body.i.split
 ; A53-NEXT:    // =>This Inner Loop Header: Depth=1
 ; A53-NEXT:    b .LBB0_4
 entry:
   %0 = bitcast %struct1* %fde to i8*
   tail call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 40, i1 false)
   %state = getelementptr inbounds %struct1, %struct1* %fde, i64 0, i32 4

llvm/test/CodeGen/AArch64/optimize-cond-branch.ll

 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    cbz wzr, .LBB0_4
 ; CHECK-NEXT:  // %bb.2: // %b3
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    and w0, w8, #0x100
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    cbz w0, .LBB0_5
 ; CHECK-NEXT:  .LBB0_3: // %common.ret.sink.split
 ; CHECK-NEXT:    b extfunc
 ; CHECK-NEXT:  .LBB0_4: // %b2
 ; CHECK-NEXT:    bl extfunc
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    cbnz w0, .LBB0_3
 ; CHECK-NEXT:  .LBB0_5: // %common.ret
 ; CHECK-NEXT:    ret
   %c0 = icmp sgt i64 0, 0
   br i1 %c0, label %b1, label %b6
 
 b1:
   br i1 undef, label %b3, label %b2

llvm/test/CodeGen/AArch64/ragreedy-local-interval-cost.ll

 ; CHECK-NEXT:    stp q20, q19, [x8, #304]
 ; CHECK-NEXT:    stp q18, q17, [x8, #336]
 ; CHECK-NEXT:    stp q16, q7, [x8, #368]
 ; CHECK-NEXT:    stp q6, q5, [x8, #400]
 ; CHECK-NEXT:    stp q4, q3, [x8, #432]
 ; CHECK-NEXT:    stp q0, q2, [x8, #464]
 ; CHECK-NEXT:    str q1, [x8, #496]
 ; CHECK-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -64
 ; CHECK-NEXT:    ret
 entry:
   br label %for.cond1.preheader
 
 for.cond1.preheader:                              ; preds = %for.cond1.preheader, %entry
   %0 = phi <2 x i64> [ undef, %entry ], [ %118, %for.cond1.preheader ]
   %1 = phi <2 x i64> [ undef, %entry ], [ %116, %for.cond1.preheader ]
   %2 = phi <2 x i64> [ zeroinitializer, %entry ], [ %114, %for.cond1.preheader ]

llvm/test/CodeGen/AArch64/shift_minsize.ll

 ; CHECK-DARWIN-NOT: __ashlti3
 ; CHECK-DARWIN-NOT: __ashrti3
 
 define i64 @f0(i64 %val, i64 %amt) minsize optsize {
 ; CHECK-LABEL: f0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl x0, x0, x1
 ; CHECK-NEXT:    ret
+;
+; CHECK-DARWIN-LABEL: f0:
+; CHECK-DARWIN:       ; %bb.0:
+; CHECK-DARWIN-NEXT:    lsl x0, x0, x1
+; CHECK-DARWIN-NEXT:    ret
   %res = shl i64 %val, %amt
   ret i64 %res
 }
 
 define i32 @f1(i64 %x, i64 %y) minsize optsize {
 ; CHECK-LABEL: f1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl x0, x0, x1
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; CHECK-DARWIN-LABEL: f1:
+; CHECK-DARWIN:       ; %bb.0:
+; CHECK-DARWIN-NEXT:    lsl x0, x0, x1
+; CHECK-DARWIN-NEXT:    ; kill: def $w0 killed $w0 killed $x0
+; CHECK-DARWIN-NEXT:    ret
   %a = shl i64 %x, %y
   %b = trunc i64 %a to i32
   ret i32 %b
 }
 
 define i32 @f2(i64 %x, i64 %y) minsize optsize {
 ; CHECK-LABEL: f2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr x0, x0, x1
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; CHECK-DARWIN-LABEL: f2:
+; CHECK-DARWIN:       ; %bb.0:
+; CHECK-DARWIN-NEXT:    asr x0, x0, x1
+; CHECK-DARWIN-NEXT:    ; kill: def $w0 killed $w0 killed $x0
+; CHECK-DARWIN-NEXT:    ret
   %a = ashr i64 %x, %y
   %b = trunc i64 %a to i32
   ret i32 %b
 }
 
 define i32 @f3(i64 %x, i64 %y) minsize optsize {
 ; CHECK-LABEL: f3:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr x0, x0, x1
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
+;
+; CHECK-DARWIN-LABEL: f3:
+; CHECK-DARWIN:       ; %bb.0:
+; CHECK-DARWIN-NEXT:    lsr x0, x0, x1
+; CHECK-DARWIN-NEXT:    ; kill: def $w0 killed $w0 killed $x0
+; CHECK-DARWIN-NEXT:    ret
   %a = lshr i64 %x, %y
   %b = trunc i64 %a to i32
   ret i32 %b
 }
 
 define dso_local { i64, i64 } @shl128(i64 %x.coerce0, i64 %x.coerce1, i8 signext %y) minsize optsize {
 ; CHECK-LABEL: shl128:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w2, w2
 ; CHECK-NEXT:    bl __ashlti3
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
+;
+; CHECK-DARWIN-LABEL: shl128:
+; CHECK-DARWIN:       ; %bb.0: ; %entry
+; CHECK-DARWIN-NEXT:    mvn w8, w2
+; CHECK-DARWIN-NEXT:    lsr x9, x0, #1
+; CHECK-DARWIN-NEXT:    mov w10, w2
+; CHECK-DARWIN-NEXT:    lsr x8, x9, x8
+; CHECK-DARWIN-NEXT:    lsl x9, x1, x10
+; CHECK-DARWIN-NEXT:    lsl x11, x0, x10
+; CHECK-DARWIN-NEXT:    tst x10, #0x40
+; CHECK-DARWIN-NEXT:    orr x8, x9, x8
+; CHECK-DARWIN-NEXT:    csel x1, x11, x8, ne
+; CHECK-DARWIN-NEXT:    csel x0, xzr, x11, ne
+; CHECK-DARWIN-NEXT:    ret
 
 entry:
   %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
   %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
   %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
   %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
   %conv = sext i8 %y to i32
   %sh_prom = zext i32 %conv to i128

 ; CHECK-LABEL: ashr128:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w2, w2
 ; CHECK-NEXT:    bl __ashrti3
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
+;
+; CHECK-DARWIN-LABEL: ashr128:
+; CHECK-DARWIN:       ; %bb.0: ; %entry
+; CHECK-DARWIN-NEXT:    mov w8, w2
+; CHECK-DARWIN-NEXT:    mvn w9, w2
+; CHECK-DARWIN-NEXT:    lsl x10, x1, #1
+; CHECK-DARWIN-NEXT:    lsl x9, x10, x9
+; CHECK-DARWIN-NEXT:    lsr x11, x0, x8
+; CHECK-DARWIN-NEXT:    asr x10, x1, #63
+; CHECK-DARWIN-NEXT:    asr x12, x1, x8
+; CHECK-DARWIN-NEXT:    tst x8, #0x40
+; CHECK-DARWIN-NEXT:    orr x8, x9, x11
+; CHECK-DARWIN-NEXT:    csel x0, x12, x8, ne
+; CHECK-DARWIN-NEXT:    csel x1, x10, x12, ne
+; CHECK-DARWIN-NEXT:    ret
 entry:
   %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
   %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
   %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
   %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
   %conv = sext i8 %y to i32
   %sh_prom = zext i32 %conv to i128
   %shr = ashr i128 %x.sroa.0.0.insert.insert, %sh_prom

 ; CHECK-LABEL: lshr128:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov w2, w2
 ; CHECK-NEXT:    bl __lshrti3
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
+;
+; CHECK-DARWIN-LABEL: lshr128:
+; CHECK-DARWIN:       ; %bb.0: ; %entry
+; CHECK-DARWIN-NEXT:    mov w8, w2
+; CHECK-DARWIN-NEXT:    mvn w9, w2
+; CHECK-DARWIN-NEXT:    lsl x10, x1, #1
+; CHECK-DARWIN-NEXT:    lsr x11, x0, x8
+; CHECK-DARWIN-NEXT:    lsl x9, x10, x9
+; CHECK-DARWIN-NEXT:    lsr x10, x1, x8
+; CHECK-DARWIN-NEXT:    tst x8, #0x40
+; CHECK-DARWIN-NEXT:    orr x8, x9, x11
+; CHECK-DARWIN-NEXT:    csel x0, x10, x8, ne
+; CHECK-DARWIN-NEXT:    csel x1, xzr, x10, ne
+; CHECK-DARWIN-NEXT:    ret
 entry:
   %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
   %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
   %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
   %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
   %conv = sext i8 %y to i32
   %sh_prom = zext i32 %conv to i128
   %shr = lshr i128 %x.sroa.0.0.insert.insert, %sh_prom
   %retval.sroa.0.0.extract.trunc = trunc i128 %shr to i64
   %retval.sroa.2.0.extract.shift = lshr i128 %shr, 64
   %retval.sroa.2.0.extract.trunc = trunc i128 %retval.sroa.2.0.extract.shift to i64
   %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %retval.sroa.0.0.extract.trunc, 0
   %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.sroa.2.0.extract.trunc, 1
   ret { i64, i64 } %.fca.1.insert
 }

llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll

 ; CHECK-NEXT:    bl consume
 ; CHECK-NEXT:    b .LBB8_3
 ; CHECK-NEXT:  .LBB8_2:
 ; CHECK-NEXT:    mov w19, #1
 ; CHECK-NEXT:  .LBB8_3: // %common.ret
 ; CHECK-NEXT:    and w0, w19, #0x1
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
 ; CHECK-NEXT:    ret
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %a)]
   br i1 %external_cond, label %left, label %right
 
 left:
   %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 0, i32 0)
   %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)

llvm/test/CodeGen/AArch64/sve-insert-element.ll

 ; CHECK-NEXT:    st1b { z1.b }, p1, [sp]
 ; CHECK-NEXT:    strb w0, [x8, x9]
 ; CHECK-NEXT:    ld1b { z0.b }, p1/z, [sp]
 ; CHECK-NEXT:    ld1b { z1.b }, p1/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    and z0.b, z0.b, #0x1
 ; CHECK-NEXT:    and z1.b, z1.b, #0x1
 ; CHECK-NEXT:    cmpne p0.b, p1/z, z0.b, #0
 ; CHECK-NEXT:    cmpne p1.b, p1/z, z1.b, #0
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %res = insertelement <vscale x 32 x i1> %val, i1 %elt, i32 %idx
   ret <vscale x 32 x i1> %res
 }

llvm/test/CodeGen/AArch64/sve-insert-vector.ll

 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x8, #2, mul vl]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x8, #3, mul vl]
 ; CHECK-NEXT:    ld1d { z3.d }, p0/z, [sp]
 ; CHECK-NEXT:    st1d { z2.d }, p0, [x0, #3, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0, #1, mul vl]
 ; CHECK-NEXT:    st1d { z3.d }, p0, [x0]
-; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ret
   %v0 = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> undef, <2 x i64> %sv0, i64 0)
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> %v0, <2 x i64> %sv1, i64 4)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
 define void @insert_v2i64_nxv16i64_lo0(<2 x i64>* %psv, <vscale x 16 x i64>* %out) {

 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    str q0, [sp, #16]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [sp]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x1, #1, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x1]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %sv = load <2 x i64>, <2 x i64>* %psv
   %v = call <vscale x 16 x i64> @llvm.experimental.vector.insert.v2i64.nxv16i64(<vscale x 16 x i64> undef, <2 x i64> %sv, i64 2)
   store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
   ret void
 }
 
 

llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll

 ; CHECK-NEXT:    sub x10, x10, #1
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    cmp x9, x10
 ; CHECK-NEXT:    st1b { z1.b }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
 ; CHECK-NEXT:    csel x9, x9, x10, lo
 ; CHECK-NEXT:    ldrb w0, [x8, x9]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 32 x i8> %a, i32 %idx
   ret i8 %ext
 }
 
 define i16 @split_extract_16i16_idx(<vscale x 16 x i16> %a, i32 %idx) {
 ; CHECK-LABEL: split_extract_16i16_idx:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    rdvl x10, #1
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sxtw x9, w0
 ; CHECK-NEXT:    sub x10, x10, #1
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    cmp x9, x10
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
 ; CHECK-NEXT:    csel x9, x9, x10, lo
 ; CHECK-NEXT:    ldrh w0, [x8, x9, lsl #1]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 16 x i16> %a, i32 %idx
   ret i16 %ext
 }
 
 define i32 @split_extract_8i32_idx(<vscale x 8 x i32> %a, i32 %idx) {
 ; CHECK-LABEL: split_extract_8i32_idx:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    cnth x10
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sxtw x9, w0
 ; CHECK-NEXT:    sub x10, x10, #1
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    cmp x9, x10
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    csel x9, x9, x10, lo
 ; CHECK-NEXT:    ldr w0, [x8, x9, lsl #2]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 8 x i32> %a, i32 %idx
   ret i32 %ext
 }
 
 define i64 @split_extract_8i64_idx(<vscale x 8 x i64> %a, i32 %idx) {
 ; CHECK-LABEL: split_extract_8i64_idx:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    cmp x9, x10
 ; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #3, mul vl]
 ; CHECK-NEXT:    st1d { z2.d }, p0, [x8, #2, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
 ; CHECK-NEXT:    csel x9, x9, x10, lo
 ; CHECK-NEXT:    ldr x0, [x8, x9, lsl #3]
-; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 8 x i64> %a, i32 %idx
   ret i64 %ext
 }
 
 ; EXTRACT VECTOR ELT, CONSTANT IDX
 
 define i16 @promote_extract_4i16(<vscale x 4 x i16> %a) {

 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    mov w9, #128
 ; CHECK-NEXT:    cmp x10, #128
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
 ; CHECK-NEXT:    csel x9, x10, x9, lo
 ; CHECK-NEXT:    ldrh w0, [x8, x9, lsl #1]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 16 x i16> %a, i32 128
   ret i16 %ext
 }
 
 define i32 @split_extract_16i32(<vscale x 16 x i32> %a) {
 ; CHECK-LABEL: split_extract_16i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    cmp x10, x9
 ; CHECK-NEXT:    st1w { z3.s }, p0, [x8, #3, mul vl]
 ; CHECK-NEXT:    st1w { z2.s }, p0, [x8, #2, mul vl]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    csel x9, x10, x9, lo
 ; CHECK-NEXT:    ldr w0, [x8, x9, lsl #2]
-; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 16 x i32> %a, i32 100000
   ret i32 %ext
 }
 
 define i64 @split_extract_4i64(<vscale x 4 x i64> %a) {
 ; CHECK-LABEL: split_extract_4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    cntw x10
 ; CHECK-NEXT:    sub x10, x10, #1
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    mov w9, #10
 ; CHECK-NEXT:    cmp x10, #10
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
 ; CHECK-NEXT:    csel x9, x10, x9, lo
 ; CHECK-NEXT:    ldr x0, [x8, x9, lsl #3]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ext = extractelement <vscale x 4 x i64> %a, i32 10
   ret i64 %ext
 }

llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll

 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    csel x8, x1, x8, lo
 ; CHECK-NEXT:    mov x9, sp
 ; CHECK-NEXT:    st1b { z1.b }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
 ; CHECK-NEXT:    strb w0, [x9, x8]
 ; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x9, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [sp]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 32 x i8> %a, i8 %elt, i64 %idx
   ret <vscale x 32 x i8> %ins
 }
 
 define <vscale x 8 x float> @split_insert_8f32_idx(<vscale x 8 x float> %a, float %elt, i64 %idx) {
 ; CHECK-LABEL: split_insert_8f32_idx:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    cnth x8
 ; CHECK-NEXT:    sub x8, x8, #1
 ; CHECK-NEXT:    cmp x0, x8
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    csel x8, x0, x8, lo
 ; CHECK-NEXT:    mov x9, sp
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    str s2, [x9, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x9, #1, mul vl]
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 8 x float> %a, float %elt, i64 %idx
   ret <vscale x 8 x float> %ins
 }
 
 define <vscale x 8 x i64> @split_insert_8i64_idx(<vscale x 8 x i64> %a, i64 %elt, i64 %idx) {
 ; CHECK-LABEL: split_insert_8i64_idx:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    st1d { z2.d }, p0, [x9, #2, mul vl]
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x9, #1, mul vl]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
 ; CHECK-NEXT:    str x0, [x9, x8, lsl #3]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x9, #1, mul vl]
 ; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x9, #2, mul vl]
 ; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x9, #3, mul vl]
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
-; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 8 x i64> %a, i64 %elt, i64 %idx
   ret <vscale x 8 x i64> %ins
 }
 
 ; INSERT VECTOR ELT, CONSTANT IDX
 
 define <vscale x 4 x i16> @promote_insert_4i16(<vscale x 4 x i16> %a, i16 %elt) {

 ; CHECK-NEXT:    st1h { z2.h }, p0, [x8, #2, mul vl]
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
 ; CHECK-NEXT:    strh w0, [x8, x9, lsl #1]
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ld1h { z2.h }, p0/z, [x8, #2, mul vl]
 ; CHECK-NEXT:    ld1h { z3.h }, p0/z, [x8, #3, mul vl]
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [sp]
-; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #4
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 32 x i16> %a, i16 %elt, i64 128
   ret <vscale x 32 x i16> %ins
 }
 
 define <vscale x 8 x i32> @split_insert_8i32(<vscale x 8 x i32> %a, i32 %elt) {
 ; CHECK-LABEL: split_insert_8i32:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    csel x9, x10, x9, lo
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x8, #1, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
 ; CHECK-NEXT:    str w0, [x8, x9, lsl #2]
 ; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
-; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
+; CHECK-NEXT:    addvl sp, sp, #2
 ; CHECK-NEXT:    ret
   %ins = insertelement <vscale x 8 x i32> %a, i32 %elt, i64 1000000
   ret <vscale x 8 x i32> %ins
 }

llvm/test/CodeGen/AArch64/sve-varargs.ll

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x0, .str_1
 ; CHECK-NEXT:    add x0, x0, :lo12:.str_1
 ; CHECK-NEXT:    bl sve_printf
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-NEXT:    ret
   %f = getelementptr [6 x i8], [6 x i8]* @.str_1, i64 0, i64 0
   call i32 (i8*, <vscale x 4 x i32>, ...) @sve_printf(i8* %f, <vscale x 4 x i32> %x)
   ret void
 }