[PowerPC] Generate CFI directives when probing in prologue

[lkail]

Summary

Add missing CFI directives when probing in prologue if stack-clash-protection is enabled.

[jsji]

LGTM.

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

1375

Can we add comments here about excluding AIX.

1456

We should merge this two CFIs into one.

llvm/test/CodeGen/PowerPC/stack-clash-prologue.ll

58

These CFIs are not generated in this patch, it would be better if we can pre-commit the testcase changed due to removing of nounwind attr first, so that we can see the real changes.

[lkail]

Address @jsji 's comments.

[jsji]

LG, Thanks.

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
             .addCFIIndex(CFIIndex);
       }
     }
   }
 }
 
 void PPCFrameLowering::inlineStackProbe(MachineFunction &MF,
                                         MachineBasicBlock &PrologMBB) const {
-  // TODO: Generate CFI instructions.
   bool isPPC64 = Subtarget.isPPC64();
   const PPCTargetLowering &TLI = *Subtarget.getTargetLowering();
   const PPCInstrInfo &TII = *Subtarget.getInstrInfo();
   MachineFrameInfo &MFI = MF.getFrameInfo();
+  MachineModuleInfo &MMI = MF.getMMI();
+  const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
+  const bool needsCFI = MF.needsFrameMoves() && !Subtarget.isAIXABI();
   auto StackAllocMIPos = llvm::find_if(PrologMBB, [](MachineInstr &MI) {
     int Opc = MI.getOpcode();
     return Opc == PPC::PROBED_STACKALLOC_64 || Opc == PPC::PROBED_STACKALLOC_32;
   });
   if (StackAllocMIPos == PrologMBB.end())
     return;
   const BasicBlock *ProbedBB = PrologMBB.getBasicBlock();
   DebugLoc DL = PrologMBB.findDebugLoc(StackAllocMIPos);
   MachineInstr &MI = *StackAllocMIPos;
   int64_t NegFrameSize = MI.getOperand(2).getImm();
   int64_t NegProbeSize = -(int64_t)TLI.getStackProbeSize(MF);
   assert(isInt<32>(NegProbeSize) && "Unhandled probe size");
   int64_t NumBlocks = NegFrameSize / NegProbeSize;
   int64_t NegResidualSize = NegFrameSize % NegProbeSize;
   Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
   Register ScratchReg = MI.getOperand(0).getReg();
   Register FPReg = MI.getOperand(1).getReg();
   const PPCRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
   bool HasBP = RegInfo->hasBasePointer(MF);
   Align MaxAlign = MFI.getMaxAlign();
   // Initialize current frame pointer.
   const MCInstrDesc &CopyInst = TII.get(isPPC64 ? PPC::OR8 : PPC::OR);
   BuildMI(PrologMBB, {MI}, DL, CopyInst, FPReg).addReg(SPReg).addReg(SPReg);
+  // Subroutines to generate .cfi_* directives.
+  auto buildDefCFAReg = [&](MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MBBI, Register Reg) {
+    unsigned RegNum = MRI->getDwarfRegNum(Reg, true);
+    unsigned CFIIndex = MF.addFrameInst(
+        MCCFIInstruction::createDefCfaRegister(nullptr, RegNum));
+    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex);
+  };
+  auto buildCFAOffset = [&](MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MBBI, int Offset) {
+    unsigned CFIIndex = MBB.getParent()->addFrameInst(
+        MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
+    BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex);
+  };
   // Subroutine to determine if we can use the Imm as part of d-form.
   auto CanUseDForm = [](int64_t Imm) { return isInt<16>(Imm) && Imm % 4 == 0; };
   // Subroutine to materialize the Imm into TempReg.
   auto MaterializeImm = [&](MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MBBI, int64_t Imm,
                             Register &TempReg) {
     assert(isInt<32>(Imm) && "Unhandled imm");
     if (isInt<16>(Imm))
       BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::LI8 : PPC::LI), TempReg)
           .addImm(Imm);
     else {
       BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::LIS8 : PPC::LIS), TempReg)
           .addImm(Imm >> 16);
       BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::ORI8 : PPC::ORI), TempReg)
           .addReg(TempReg)
           .addImm(Imm & 0xFFFF);
     }
   };
   // Subroutine to store frame pointer and decrease stack pointer by probe size.
   auto allocateAndProbe = [&](MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator MBBI, int64_t NegSize,
-                              Register NegSizeReg, bool UseDForm) {
+                              Register NegSizeReg, bool UseDForm,
+                              bool EmitCFI) {
     if (UseDForm)
       BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::STDU : PPC::STWU), SPReg)
           .addReg(FPReg)
           .addImm(NegSize)
           .addReg(SPReg);
     else
       BuildMI(MBB, MBBI, DL, TII.get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg)
           .addReg(FPReg)
           .addReg(SPReg)
           .addReg(NegSizeReg);
+    if (needsCFI && EmitCFI) {
+      // Presume using FPReg to calculate CFA.
+      buildCFAOffset(MBB, MBBI, 0);
+    }
   };
+  // Use FPReg to calculate CFA.
+  buildDefCFAReg(PrologMBB, {MI}, FPReg);
   // For case HasBP && MaxAlign > 1, we have to align the SP by performing
   // SP = SP - SP % MaxAlign.
   if (HasBP && MaxAlign > 1) {
     if (isPPC64)
       BuildMI(PrologMBB, {MI}, DL, TII.get(PPC::RLDICL), ScratchReg)
           .addReg(FPReg)
           .addImm(0)
           .addImm(64 - Log2(MaxAlign));
     else
       BuildMI(PrologMBB, {MI}, DL, TII.get(PPC::RLWINM), ScratchReg)
           .addReg(FPReg)
           .addImm(0)
           .addImm(32 - Log2(MaxAlign))
           .addImm(31);
     BuildMI(PrologMBB, {MI}, DL, TII.get(isPPC64 ? PPC::STDUX : PPC::STWUX),
             SPReg)
         .addReg(FPReg)
         .addReg(SPReg)
         .addReg(ScratchReg);
   }
   // Probe residual part.
   if (NegResidualSize) {
     bool ResidualUseDForm = CanUseDForm(NegResidualSize);
     if (!ResidualUseDForm)
       MaterializeImm(PrologMBB, {MI}, NegResidualSize, ScratchReg);
     allocateAndProbe(PrologMBB, {MI}, NegResidualSize, ScratchReg,
-                     ResidualUseDForm);
+                     ResidualUseDForm, true);
   }
   bool UseDForm = CanUseDForm(NegProbeSize);
   // If number of blocks is small, just probe them directly.
   if (NumBlocks < 3) {
     if (!UseDForm)
       MaterializeImm(PrologMBB, {MI}, NegProbeSize, ScratchReg);
     for (int i = 0; i < NumBlocks; ++i)
-      allocateAndProbe(PrologMBB, {MI}, NegProbeSize, ScratchReg, UseDForm);
+      allocateAndProbe(PrologMBB, {MI}, NegProbeSize, ScratchReg, UseDForm,
+                       true);
+    // Restore using SPReg to calculate CFA.
+    buildDefCFAReg(PrologMBB, {MI}, SPReg);
   } else {
     // Since CTR is a volatile register and current shrinkwrap implementation
     // won't choose an MBB in a loop as the PrologMBB, it's safe to synthesize a
     // CTR loop to probe.
     // Calculate trip count and stores it in CTRReg.
     MaterializeImm(PrologMBB, {MI}, NumBlocks, ScratchReg);
     BuildMI(PrologMBB, {MI}, DL, TII.get(isPPC64 ? PPC::MTCTR8 : PPC::MTCTR))
         .addReg(ScratchReg, RegState::Kill);
     if (!UseDForm)
       MaterializeImm(PrologMBB, {MI}, NegProbeSize, ScratchReg);
     // Create MBBs of the loop.
     MachineFunction::iterator MBBInsertPoint =
         std::next(PrologMBB.getIterator());
     MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(ProbedBB);
     MF.insert(MBBInsertPoint, LoopMBB);
     MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(ProbedBB);
     MF.insert(MBBInsertPoint, ExitMBB);
     // Synthesize the loop body.
     allocateAndProbe(*LoopMBB, LoopMBB->end(), NegProbeSize, ScratchReg,
-                     UseDForm);
+                     UseDForm, false);
     BuildMI(LoopMBB, DL, TII.get(isPPC64 ? PPC::BDNZ8 : PPC::BDNZ))
         .addMBB(LoopMBB);
     LoopMBB->addSuccessor(ExitMBB);
     LoopMBB->addSuccessor(LoopMBB);
     // Synthesize the exit MBB.
     ExitMBB->splice(ExitMBB->end(), &PrologMBB,
                     std::next(MachineBasicBlock::iterator(MI)),
                     PrologMBB.end());
     ExitMBB->transferSuccessorsAndUpdatePHIs(&PrologMBB);
     PrologMBB.addSuccessor(LoopMBB);
+    // Restore using SPReg to calculate CFA.
+    buildDefCFAReg(*ExitMBB, ExitMBB->begin(), SPReg);
     // Update liveins.
     recomputeLiveIns(*LoopMBB);
     recomputeLiveIns(*ExitMBB);
   }
   ++NumPrologProbed;
   MI.eraseFromParent();
 }
 

llvm/test/CodeGen/PowerPC/stack-clash-prologue.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
 ; RUN:   -mtriple=powerpc64le-linux-gnu < %s | FileCheck \
 ; RUN:   -check-prefix=CHECK-LE %s
 ; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
 ; RUN:   -mtriple=powerpc64-linux-gnu < %s | FileCheck \
 ; RUN:   -check-prefix=CHECK-BE %s
 ; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs \
 ; RUN:   -mtriple=powerpc-linux-gnu < %s | FileCheck \
 ; RUN:   -check-prefix=CHECK-32 %s
 
 ; Free probe
-define i8 @f0() #0 nounwind {
+define i8 @f0() #0 {
 ; CHECK-LE-LABEL: f0:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, -64(r1)
 ; CHECK-LE-NEXT:    lbz r3, -64(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f0:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, -64(r1)
 ; CHECK-BE-NEXT:    lbz r3, -64(r1)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f0:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    stwu r1, -80(r1)
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    lbz r3, 16(r1)
 ; CHECK-32-NEXT:    addi r1, r1, 80
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 64
   %b = getelementptr inbounds i8, i8* %a, i64 63
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
-define i8 @f1() #0 "stack-probe-size"="0" nounwind {
+define i8 @f1() #0 "stack-probe-size"="0" {
 ; CHECK-LE-LABEL: f1:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-LE-NEXT:    li r0, 259
 ; CHECK-LE-NEXT:    mtctr r0
 ; CHECK-LE-NEXT:  .LBB1_1: # %entry
 ; CHECK-LE-NEXT:    #
 ; CHECK-LE-NEXT:    stdu r12, -16(r1)
 ; CHECK-LE-NEXT:    bdnz .LBB1_1
 ; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 4144
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, 48(r1)
 ; CHECK-LE-NEXT:    lbz r3, 48(r1)
 ; CHECK-LE-NEXT:    addi r1, r1, 4144
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f1:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-BE-NEXT:    li r0, 260
 ; CHECK-BE-NEXT:    mtctr r0
 ; CHECK-BE-NEXT:  .LBB1_1: # %entry
 ; CHECK-BE-NEXT:    #
 ; CHECK-BE-NEXT:    stdu r12, -16(r1)
 ; CHECK-BE-NEXT:    bdnz .LBB1_1
 ; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 4160
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, 64(r1)
 ; CHECK-BE-NEXT:    lbz r3, 64(r1)
 ; CHECK-BE-NEXT:    addi r1, r1, 4160
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f1:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-32-NEXT:    li r0, 257
 ; CHECK-32-NEXT:    mtctr r0
 ; CHECK-32-NEXT:  .LBB1_1: # %entry
 ; CHECK-32-NEXT:    #
 ; CHECK-32-NEXT:    stwu r12, -16(r1)
 ; CHECK-32-NEXT:    bdnz .LBB1_1
 ; CHECK-32-NEXT:  # %bb.2: # %entry
-; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    .cfi_def_cfa_register r1
 ; CHECK-32-NEXT:    sub r0, r1, r12
-; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 4112
+; CHECK-32-NEXT:    li r3, 3
+; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    lbz r3, 16(r1)
 ; CHECK-32-NEXT:    addi r1, r1, 4112
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 4096
   %b = getelementptr inbounds i8, i8* %a, i64 63
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
-define i8 @f2() #0 nounwind {
+define i8 @f2() #0 {
 ; CHECK-LE-LABEL: f2:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-LE-NEXT:    li r0, 16
 ; CHECK-LE-NEXT:    mtctr r0
 ; CHECK-LE-NEXT:  .LBB2_1: # %entry
 ; CHECK-LE-NEXT:    #
 ; CHECK-LE-NEXT:    stdu r12, -4096(r1)
 ; CHECK-LE-NEXT:    bdnz .LBB2_1
 ; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 65584
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, 48(r1)
 ; CHECK-LE-NEXT:    lbz r3, 48(r1)
 ; CHECK-LE-NEXT:    ld r1, 0(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f2:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-BE-NEXT:    li r0, 16
 ; CHECK-BE-NEXT:    mtctr r0
 ; CHECK-BE-NEXT:  .LBB2_1: # %entry
 ; CHECK-BE-NEXT:    #
 ; CHECK-BE-NEXT:    stdu r12, -4096(r1)
 ; CHECK-BE-NEXT:    bdnz .LBB2_1
 ; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 65600
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, 64(r1)
 ; CHECK-BE-NEXT:    lbz r3, 64(r1)
 ; CHECK-BE-NEXT:    ld r1, 0(r1)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f2:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-32-NEXT:    li r0, 16
 ; CHECK-32-NEXT:    mtctr r0
 ; CHECK-32-NEXT:  .LBB2_1: # %entry
 ; CHECK-32-NEXT:    #
 ; CHECK-32-NEXT:    stwu r12, -4096(r1)
 ; CHECK-32-NEXT:    bdnz .LBB2_1
 ; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    .cfi_def_cfa_register r1
 ; CHECK-32-NEXT:    sub r0, r1, r12
-; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 65552
+; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    mr r0, r31
 ; CHECK-32-NEXT:    lbz r3, 16(r1)
 ; CHECK-32-NEXT:    lwz r31, 0(r1)
 ; CHECK-32-NEXT:    mr r1, r31
 ; CHECK-32-NEXT:    mr r31, r0
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 65536
   %b = getelementptr inbounds i8, i8* %a, i64 63
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
-define i8 @f3() #0 "stack-probe-size"="32768" nounwind {
+define i8 @f3() #0 "stack-probe-size"="32768" {
 ; CHECK-LE-LABEL: f3:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-LE-NEXT:    stdu r12, -32768(r1)
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-LE-NEXT:    stdu r12, -32768(r1)
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 65584
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, 48(r1)
 ; CHECK-LE-NEXT:    lbz r3, 48(r1)
 ; CHECK-LE-NEXT:    ld r1, 0(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f3:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-BE-NEXT:    stdu r12, -32768(r1)
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-BE-NEXT:    stdu r12, -32768(r1)
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 65600
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, 64(r1)
 ; CHECK-BE-NEXT:    lbz r3, 64(r1)
 ; CHECK-BE-NEXT:    ld r1, 0(r1)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f3:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-32-NEXT:    stwu r12, -32768(r1)
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-32-NEXT:    stwu r12, -32768(r1)
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-32-NEXT:    .cfi_def_cfa_register r1
 ; CHECK-32-NEXT:    sub r0, r1, r12
-; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 65552
+; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    mr r0, r31
 ; CHECK-32-NEXT:    lbz r3, 16(r1)
 ; CHECK-32-NEXT:    lwz r31, 0(r1)
 ; CHECK-32-NEXT:    mr r1, r31
 ; CHECK-32-NEXT:    mr r31, r0
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 65536
   %b = getelementptr inbounds i8, i8* %a, i64 63
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
 ; Same as f2, but without protection.
-define i8 @f4() nounwind {
+define i8 @f4() {
 ; CHECK-LE-LABEL: f4:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE-NEXT:    lis r0, -2
 ; CHECK-LE-NEXT:    ori r0, r0, 65488
 ; CHECK-LE-NEXT:    stdux r1, r1, r0
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 65584
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, 48(r1)
 ; CHECK-LE-NEXT:    lbz r3, 48(r1)
 ; CHECK-LE-NEXT:    ld r1, 0(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f4:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    lis r0, -2
 ; CHECK-BE-NEXT:    ori r0, r0, 65472
 ; CHECK-BE-NEXT:    stdux r1, r1, r0
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 65600
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, 64(r1)
 ; CHECK-BE-NEXT:    lbz r3, 64(r1)
 ; CHECK-BE-NEXT:    ld r1, 0(r1)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f4:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    lis r0, -2
 ; CHECK-32-NEXT:    ori r0, r0, 65520
 ; CHECK-32-NEXT:    stwux r1, r1, r0
-; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 65552
+; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    mr r0, r31
 ; CHECK-32-NEXT:    lbz r3, 16(r1)
 ; CHECK-32-NEXT:    lwz r31, 0(r1)
 ; CHECK-32-NEXT:    mr r1, r31
 ; CHECK-32-NEXT:    mr r31, r0
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 65536
   %b = getelementptr inbounds i8, i8* %a, i64 63
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
-define i8 @f5() #0 "stack-probe-size"="65536" nounwind {
+define i8 @f5() #0 "stack-probe-size"="65536" {
 ; CHECK-LE-LABEL: f5:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-LE-NEXT:    li r0, 16
 ; CHECK-LE-NEXT:    mtctr r0
 ; CHECK-LE-NEXT:    lis r0, -1
 ; CHECK-LE-NEXT:    nop
 ; CHECK-LE-NEXT:  .LBB5_1: # %entry
 ; CHECK-LE-NEXT:    #
 ; CHECK-LE-NEXT:    stdux r12, r1, r0
 ; CHECK-LE-NEXT:    bdnz .LBB5_1
 ; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 1048624
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, 48(r1)
 ; CHECK-LE-NEXT:    lbz r3, 48(r1)
 ; CHECK-LE-NEXT:    ld r1, 0(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f5:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-BE-NEXT:    li r0, 16
 ; CHECK-BE-NEXT:    mtctr r0
 ; CHECK-BE-NEXT:    lis r0, -1
 ; CHECK-BE-NEXT:    nop
 ; CHECK-BE-NEXT:  .LBB5_1: # %entry
 ; CHECK-BE-NEXT:    #
 ; CHECK-BE-NEXT:    stdux r12, r1, r0
 ; CHECK-BE-NEXT:    bdnz .LBB5_1
 ; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 1048640
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, 64(r1)
 ; CHECK-BE-NEXT:    lbz r3, 64(r1)
 ; CHECK-BE-NEXT:    ld r1, 0(r1)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f5:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-32-NEXT:    li r0, 16
 ; CHECK-32-NEXT:    mtctr r0
 ; CHECK-32-NEXT:    lis r0, -1
 ; CHECK-32-NEXT:    nop
 ; CHECK-32-NEXT:  .LBB5_1: # %entry
 ; CHECK-32-NEXT:    #
 ; CHECK-32-NEXT:    stwux r12, r1, r0
 ; CHECK-32-NEXT:    bdnz .LBB5_1
 ; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    .cfi_def_cfa_register r1
 ; CHECK-32-NEXT:    sub r0, r1, r12
-; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 1048592
+; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    mr r0, r31
 ; CHECK-32-NEXT:    lbz r3, 16(r1)
 ; CHECK-32-NEXT:    lwz r31, 0(r1)
 ; CHECK-32-NEXT:    mr r1, r31
 ; CHECK-32-NEXT:    mr r31, r0
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 1048576
   %b = getelementptr inbounds i8, i8* %a, i64 63
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
-define i8 @f6() #0 nounwind {
+define i8 @f6() #0 {
 ; CHECK-LE-LABEL: f6:
 ; CHECK-LE:       # %bb.0: # %entry
 ; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-LE-NEXT:    stdu r12, -48(r1)
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-LE-NEXT:    lis r0, 4
 ; CHECK-LE-NEXT:    nop
 ; CHECK-LE-NEXT:    mtctr r0
 ; CHECK-LE-NEXT:  .LBB6_1: # %entry
 ; CHECK-LE-NEXT:    #
 ; CHECK-LE-NEXT:    stdu r12, -4096(r1)
 ; CHECK-LE-NEXT:    bdnz .LBB6_1
 ; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 1073741872
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, 48(r1)
 ; CHECK-LE-NEXT:    lbz r3, 48(r1)
 ; CHECK-LE-NEXT:    ld r1, 0(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f6:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-BE-NEXT:    stdu r12, -64(r1)
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-BE-NEXT:    lis r0, 4
 ; CHECK-BE-NEXT:    nop
 ; CHECK-BE-NEXT:    mtctr r0
 ; CHECK-BE-NEXT:  .LBB6_1: # %entry
 ; CHECK-BE-NEXT:    #
 ; CHECK-BE-NEXT:    stdu r12, -4096(r1)
 ; CHECK-BE-NEXT:    bdnz .LBB6_1
 ; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 1073741888
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, 64(r1)
 ; CHECK-BE-NEXT:    lbz r3, 64(r1)
 ; CHECK-BE-NEXT:    ld r1, 0(r1)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f6:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    .cfi_def_cfa_register r12
 ; CHECK-32-NEXT:    stwu r12, -16(r1)
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-32-NEXT:    lis r0, 4
 ; CHECK-32-NEXT:    nop
 ; CHECK-32-NEXT:    mtctr r0
 ; CHECK-32-NEXT:  .LBB6_1: # %entry
 ; CHECK-32-NEXT:    #
 ; CHECK-32-NEXT:    stwu r12, -4096(r1)
 ; CHECK-32-NEXT:    bdnz .LBB6_1
 ; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    .cfi_def_cfa_register r1
 ; CHECK-32-NEXT:    sub r0, r1, r12
-; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 1073741840
+; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    stb r3, 16(r1)
 ; CHECK-32-NEXT:    mr r0, r31
 ; CHECK-32-NEXT:    lbz r3, 16(r1)
 ; CHECK-32-NEXT:    lwz r31, 0(r1)
 ; CHECK-32-NEXT:    mr r1, r31
 ; CHECK-32-NEXT:    mr r31, r0
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 1073741824
   %b = getelementptr inbounds i8, i8* %a, i64 63
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
-define i8 @f7() #0 "stack-probe-size"="65536" nounwind {
+define i8 @f7() #0 "stack-probe-size"="65536" {
 ; CHECK-LE-LABEL: f7:
 ; CHECK-LE:       # %bb.0: # %entry
-; CHECK-LE-NEXT:    lis r0, -1
 ; CHECK-LE-NEXT:    mr r12, r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r12
+; CHECK-LE-NEXT:    lis r0, -1
 ; CHECK-LE-NEXT:    ori r0, r0, 13776
 ; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-LE-NEXT:    li r0, 15258
 ; CHECK-LE-NEXT:    mtctr r0
 ; CHECK-LE-NEXT:    lis r0, -1
 ; CHECK-LE-NEXT:    nop
 ; CHECK-LE-NEXT:  .LBB7_1: # %entry
 ; CHECK-LE-NEXT:    #
 ; CHECK-LE-NEXT:    stdux r12, r1, r0
 ; CHECK-LE-NEXT:    bdnz .LBB7_1
 ; CHECK-LE-NEXT:  # %bb.2: # %entry
+; CHECK-LE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-LE-NEXT:    .cfi_def_cfa_offset 1000000048
 ; CHECK-LE-NEXT:    li r3, 3
 ; CHECK-LE-NEXT:    stb r3, 41(r1)
 ; CHECK-LE-NEXT:    lbz r3, 41(r1)
 ; CHECK-LE-NEXT:    ld r1, 0(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: f7:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lis r0, -1
 ; CHECK-BE-NEXT:    mr r12, r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r12
+; CHECK-BE-NEXT:    lis r0, -1
 ; CHECK-BE-NEXT:    ori r0, r0, 13760
 ; CHECK-BE-NEXT:    stdux r12, r1, r0
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-BE-NEXT:    li r0, 15258
 ; CHECK-BE-NEXT:    mtctr r0
 ; CHECK-BE-NEXT:    lis r0, -1
 ; CHECK-BE-NEXT:    nop
 ; CHECK-BE-NEXT:  .LBB7_1: # %entry
 ; CHECK-BE-NEXT:    #
 ; CHECK-BE-NEXT:    stdux r12, r1, r0
 ; CHECK-BE-NEXT:    bdnz .LBB7_1
 ; CHECK-BE-NEXT:  # %bb.2: # %entry
+; CHECK-BE-NEXT:    .cfi_def_cfa_register r1
+; CHECK-BE-NEXT:    .cfi_def_cfa_offset 1000000064
 ; CHECK-BE-NEXT:    li r3, 3
 ; CHECK-BE-NEXT:    stb r3, 57(r1)
 ; CHECK-BE-NEXT:    lbz r3, 57(r1)
 ; CHECK-BE-NEXT:    ld r1, 0(r1)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-32-LABEL: f7:
 ; CHECK-32:       # %bb.0: # %entry
-; CHECK-32-NEXT:    lis r0, -1
 ; CHECK-32-NEXT:    mr r12, r1
+; CHECK-32-NEXT:    .cfi_def_cfa_register r12
+; CHECK-32-NEXT:    lis r0, -1
 ; CHECK-32-NEXT:    ori r0, r0, 13808
 ; CHECK-32-NEXT:    stwux r12, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-32-NEXT:    li r0, 15258
 ; CHECK-32-NEXT:    mtctr r0
 ; CHECK-32-NEXT:    lis r0, -1
 ; CHECK-32-NEXT:    nop
 ; CHECK-32-NEXT:  .LBB7_1: # %entry
 ; CHECK-32-NEXT:    #
 ; CHECK-32-NEXT:    stwux r12, r1, r0
 ; CHECK-32-NEXT:    bdnz .LBB7_1
 ; CHECK-32-NEXT:  # %bb.2: # %entry
+; CHECK-32-NEXT:    .cfi_def_cfa_register r1
 ; CHECK-32-NEXT:    sub r0, r1, r12
-; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    sub r0, r1, r0
+; CHECK-32-NEXT:    .cfi_def_cfa_offset 1000000016
+; CHECK-32-NEXT:    li r3, 3
 ; CHECK-32-NEXT:    stb r3, 9(r1)
 ; CHECK-32-NEXT:    mr r0, r31
 ; CHECK-32-NEXT:    lbz r3, 9(r1)
 ; CHECK-32-NEXT:    lwz r31, 0(r1)
 ; CHECK-32-NEXT:    mr r1, r31
 ; CHECK-32-NEXT:    mr r31, r0
 ; CHECK-32-NEXT:    blr
 entry:
   %a = alloca i8, i64 1000000007
   %b = getelementptr inbounds i8, i8* %a, i64 101
   store volatile i8 3, i8* %a
   %c = load volatile i8, i8* %a
   ret i8 %c
 }
 
 attributes #0 = { "probe-stack"="inline-asm" }