[PowerPC] Move TOC save to prologue when profitable

[nemanjai]

Summary

The indirect call sequence on PPC requires that the TOC base register be saved prior to the indirect call and restored after the call since the indirect call may branch to a global entry point in another DSO which will update the TOC base. Over the last couple of years, we have improved this to:

• be able to hoist TOC saves from loops (with changes to MachineLICM)
• avoid multiple saves when one dominates the other[s]

However, it is still possible to have multiple TOC saves dynamically in the execution path if there is no dominance relationship between them.

This patch moves the TOC save to the prologue when one of the TOC saves is in a block that post-dominates entry (i.e. it cannot be avoided) or if it is in a block that is hotter than entry.

[hfinkel]

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

447

You've introduced this function, which contains two lines of code, and then make a Boolean variable in several later places with the same name. I recommend just moving this code into its caller below and eliminating any potential confusion.

1314

Need a MustSaveTOC check here?

llvm/lib/Target/PowerPC/PPCMIPeephole.cpp

229

Why the MFI.hasVarSizedObjects() check?

[nemanjai]

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

447

Will do. I was a little on the fence about the value of this function as well.

1314

Yup, this is an omission. I'll fix it.

llvm/lib/Target/PowerPC/PPCMIPeephole.cpp

229

It is actually not necessary as this function won't be called when processing functions that have dynamic objects.

[nemanjai]

Remove unnecessary function and fix up conditions for the transformation.

[hfinkel]

One question below, but otherwise, LGTM.

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

1878–1880

The point of this is that, if we have R2 here, we'll hit the llvm_unreachable below? Can you write an assert instead?

[nemanjai]

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

1878–1880

That's a good point. I'll change it on the commit.

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

static bool MustSaveLR(const MachineFunction &MF, unsigned LR) {
   // We need a save/restore of LR if there is any def of LR (which is
   // defined by calls, including the PIC setup sequence), or if there is
   // some use of the LR stack slot (e.g. for builtin_return_address).
   // (LR comes in 32 and 64 bit versions.)
   MachineRegisterInfo::def_iterator RI = MF.getRegInfo().def_begin(LR);
   return RI !=MF.getRegInfo().def_end() || MFI->isLRStoreRequired();
 }
 
+static bool MustSaveTOC(const MachineFunction &MF) {
+  const PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
+  return FI->mustSaveTOC();
+}
+
 /// determineFrameLayoutAndUpdate - Determine the size of the frame and maximum
 /// call frame size. Update the MachineFunction object with the stack size.
 unsigned
 PPCFrameLowering::determineFrameLayoutAndUpdate(MachineFunction &MF,
                                                 bool UseEstimate) const {
   unsigned NewMaxCallFrameSize = 0;
   unsigned FrameSize = determineFrameLayout(MF, UseEstimate,
                                             &NewMaxCallFrameSize);
PPCFrameLowering::determineFrameLayout(const MachineFunction &MF,
 
   const PPCRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
 
   unsigned LR = RegInfo->getRARegister();
   bool DisableRedZone = MF.getFunction().hasFnAttribute(Attribute::NoRedZone);
   bool CanUseRedZone = !MFI.hasVarSizedObjects() && // No dynamic alloca.
                        !MFI.adjustsStack() &&       // No calls.
                        !MustSaveLR(MF, LR) &&       // No need to save LR.
+                       !MustSaveTOC(MF) &&          // No need to save TOC.
                        !RegInfo->hasBasePointer(MF); // No special alignment.
 
   // Note: for PPC32 SVR4ABI (Non-DarwinABI), we can still generate stackless
   // code if all local vars are reg-allocated.
   bool FitsInRedZone = FrameSize <= Subtarget.getRedZoneSize();
 
   // Check whether we can skip adjusting the stack pointer (by using red zone)
   if (!DisableRedZone && CanUseRedZone && FitsInRedZone) {
if (!isInt<32>(NegFrameSize))
     llvm_unreachable("Unhandled stack size!");
 
   if (MFI.isFrameAddressTaken())
     replaceFPWithRealFP(MF);
 
   // Check if the link register (LR) must be saved.
   PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
   bool MustSaveLR = FI->mustSaveLR();
+  bool MustSaveTOC = FI->mustSaveTOC();
   const SmallVectorImpl<unsigned> &MustSaveCRs = FI->getMustSaveCRs();
   bool MustSaveCR = !MustSaveCRs.empty();
   // Do we have a frame pointer and/or base pointer for this function?
   bool HasFP = hasFP(MF);
   bool HasBP = RegInfo->hasBasePointer(MF);
   bool HasRedZone = isPPC64 || !isSVR4ABI;
 
   unsigned SPReg       = isPPC64 ? PPC::X1  : PPC::R1;
   unsigned BPReg       = RegInfo->getBaseRegister(MF);
   unsigned FPReg       = isPPC64 ? PPC::X31 : PPC::R31;
   unsigned LRReg       = isPPC64 ? PPC::LR8 : PPC::LR;
+  unsigned TOCReg      = isPPC64 ? PPC::X2 :  PPC::R2;
   unsigned ScratchReg  = 0;
   unsigned TempReg     = isPPC64 ? PPC::X12 : PPC::R12; // another scratch reg
   //  ...(R12/X12 is volatile in both Darwin & SVR4, & can't be a function arg.)
   const MCInstrDesc& MFLRInst = TII.get(isPPC64 ? PPC::MFLR8
                                                 : PPC::MFLR );
   const MCInstrDesc& StoreInst = TII.get(isPPC64 ? PPC::STD
                                                  : PPC::STW );
   const MCInstrDesc& StoreUpdtInst = TII.get(isPPC64 ? PPC::STDU
BuildMI(MBB, MBBI, dl, OrImmInst, ScratchReg)
       .addImm(NegFrameSize & 0xFFFF);
     BuildMI(MBB, MBBI, dl, StoreUpdtIdxInst, SPReg)
       .addReg(SPReg, RegState::Kill)
       .addReg(SPReg)
       .addReg(ScratchReg);
     HasSTUX = true;
   }
 
+  // Save the TOC register after the stack pointer update if a prologue TOC
+  // save is required for the function.
+  if (MustSaveTOC) {
+    assert(isELFv2ABI && "TOC saves in the prologue only supported on ELFv2");
+    BuildMI(MBB, StackUpdateLoc, dl, TII.get(PPC::STD))
+      .addReg(TOCReg, getKillRegState(true))
+      .addImm(TOCSaveOffset)
+      .addReg(SPReg);
+  }
+
   if (!HasRedZone) {
     assert(!isPPC64 && "A red zone is always available on PPC64");
     if (HasSTUX) {
       // The negated frame size is in ScratchReg, and the SPReg has been
       // decremented by the frame size: SPReg = old SPReg + ScratchReg.
       // Since FPOffset, PBPOffset, etc. are relative to the beginning of
       // the stack frame (i.e. the old SP), ideally, we would put the old
       // SP into a register and use it as the base for the stores. The
for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
       unsigned Reg = CSI[I].getReg();
       if (Reg == PPC::LR || Reg == PPC::LR8 || Reg == PPC::RM) continue;
 
       // This is a bit of a hack: CR2LT, CR2GT, CR2EQ and CR2UN are just
       // subregisters of CR2. We just need to emit a move of CR2.
       if (PPC::CRBITRCRegClass.contains(Reg))
         continue;
 
+      if (Reg == PPC::X2 || Reg == PPC::R2)
+        continue;
+
       // For SVR4, don't emit a move for the CR spill slot if we haven't
       // spilled CRs.
       if (isSVR4ABI && (PPC::CR2 <= Reg && Reg <= PPC::CR4)
           && !MustSaveCR)
         continue;
 
       // For 64-bit SVR4 when we have spilled CRs, the spill location
       // is SP+8, not a frame-relative slot.
if (CSI.empty() && !needsFP(MF)) {
     return;
   }
 
   unsigned MinGPR = PPC::R31;
   unsigned MinG8R = PPC::X31;
   unsigned MinFPR = PPC::F31;
   unsigned MinVR = Subtarget.hasSPE() ? PPC::S31 : PPC::V31;
 
+  PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
   bool HasGPSaveArea = false;
   bool HasG8SaveArea = false;
   bool HasFPSaveArea = false;
   bool HasVRSAVESaveArea = false;
   bool HasVRSaveArea = false;
+  bool MustSaveTOC = FI->mustSaveTOC();
 
   SmallVector<CalleeSavedInfo, 18> GPRegs;
   SmallVector<CalleeSavedInfo, 18> G8Regs;
   SmallVector<CalleeSavedInfo, 18> FPRegs;
   SmallVector<CalleeSavedInfo, 18> VRegs;
 
   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     unsigned Reg = CSI[i].getReg();
-    if (PPC::GPRCRegClass.contains(Reg) ||
-        PPC::SPE4RCRegClass.contains(Reg)) {
+    bool IgnoreR2 = MustSaveTOC && (Reg == PPC::X2 || Reg == PPC::R2);
+    if (!IgnoreR2 && (PPC::GPRCRegClass.contains(Reg) ||
+        PPC::SPE4RCRegClass.contains(Reg))) {
       HasGPSaveArea = true;
 
       GPRegs.push_back(CSI[i]);
 
       if (Reg < MinGPR) {
         MinGPR = Reg;
       }
-    } else if (PPC::G8RCRegClass.contains(Reg)) {
+    } else if (!IgnoreR2 && PPC::G8RCRegClass.contains(Reg)) {
       HasG8SaveArea = true;
 
       G8Regs.push_back(CSI[i]);
 
       if (Reg < MinG8R) {
         MinG8R = Reg;
       }
     } else if (PPC::F8RCRegClass.contains(Reg)) {
PPCFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
 
   // Currently, this function only handles SVR4 32- and 64-bit ABIs.
   // Return false otherwise to maintain pre-existing behavior.
   if (!Subtarget.isSVR4ABI())
     return false;
 
   MachineFunction *MF = MBB.getParent();
   const PPCInstrInfo &TII = *Subtarget.getInstrInfo();
+  PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();
+  bool MustSaveTOC = FI->mustSaveTOC();
   DebugLoc DL;
   bool CRSpilled = false;
   MachineInstrBuilder CRMIB;
 
   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     unsigned Reg = CSI[i].getReg();
     // Only Darwin actually uses the VRSAVE register, but it can still appear
     // here if, for example, @llvm.eh.unwind.init() is used.  If we're not on
for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     if (!IsLiveIn)
        MBB.addLiveIn(Reg);
 
     if (CRSpilled && IsCRField) {
       CRMIB.addReg(Reg, RegState::ImplicitKill);
       continue;
     }
 
+    // The actual spill will happen in the prologue.
+    if ((Reg == PPC::X2 || Reg == PPC::R2) && MustSaveTOC)
+      continue;
+
     // Insert the spill to the stack frame.
     if (IsCRField) {
       PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
       if (Subtarget.isPPC64()) {
         // The actual spill will happen at the start of the prologue.
         FuncInfo->addMustSaveCR(Reg);
       } else {
         CRSpilled = true;
PPCFrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
 
   // Currently, this function only handles SVR4 32- and 64-bit ABIs.
   // Return false otherwise to maintain pre-existing behavior.
   if (!Subtarget.isSVR4ABI())
     return false;
 
   MachineFunction *MF = MBB.getParent();
   const PPCInstrInfo &TII = *Subtarget.getInstrInfo();
+  PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();
+  bool MustSaveTOC = FI->mustSaveTOC();
   bool CR2Spilled = false;
   bool CR3Spilled = false;
   bool CR4Spilled = false;
   unsigned CSIIndex = 0;
 
   // Initialize insertion-point logic; we will be restoring in reverse
   // order of spill.
   MachineBasicBlock::iterator I = MI, BeforeI = I;
   bool AtStart = I == MBB.begin();
 
   if (!AtStart)
     --BeforeI;
 
   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     unsigned Reg = CSI[i].getReg();
 
     // Only Darwin actually uses the VRSAVE register, but it can still appear
     // here if, for example, @llvm.eh.unwind.init() is used.  If we're not on
     // Darwin, ignore it.
     if (Reg == PPC::VRSAVE && !Subtarget.isDarwinABI())
       continue;
 
+    if ((Reg == PPC::X2 || Reg == PPC::R2) && MustSaveTOC)
+      continue;
+
     if (Reg == PPC::CR2) {
       CR2Spilled = true;
       // The spill slot is associated only with CR2, which is the
       // first nonvolatile spilled.  Save it here.
       CSIIndex = i;
       continue;
     } else if (Reg == PPC::CR3) {
       CR3Spilled = true;

llvm/lib/Target/PowerPC/PPCMIPeephole.cpp

 // removal, and it would miss cleanups made possible following VSX
 // swap removal.
 //
 //===---------------------------------------------------------------------===//
 
 #include "PPC.h"
 #include "PPCInstrBuilder.h"
 #include "PPCInstrInfo.h"
+#include "PPCMachineFunctionInfo.h"
 #include "PPCTargetMachine.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Support/Debug.h"
 #include "MCTargetDesc/PPCPredicates.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "ppc-mi-peepholes"
 
 STATISTIC(RemoveTOCSave, "Number of TOC saves removed");
 STATISTIC(MultiTOCSaves,
           "Number of functions with multiple TOC saves that must be kept");
+STATISTIC(NumTOCSavesInPrologue, "Number of TOC saves placed in the prologue");
 STATISTIC(NumEliminatedSExt, "Number of eliminated sign-extensions");
 STATISTIC(NumEliminatedZExt, "Number of eliminated zero-extensions");
 STATISTIC(NumOptADDLIs, "Number of optimized ADD instruction fed by LI");
 STATISTIC(NumConvertedToImmediateForm,
           "Number of instructions converted to their immediate form");
 STATISTIC(NumFunctionsEnteredInMIPeephole,
           "Number of functions entered in PPC MI Peepholes");
 STATISTIC(NumFixedPointIterations,
struct PPCMIPeephole : public MachineFunctionPass {
   MachineRegisterInfo *MRI;
 
   PPCMIPeephole() : MachineFunctionPass(ID) {
     initializePPCMIPeepholePass(*PassRegistry::getPassRegistry());
   }
 
 private:
   MachineDominatorTree *MDT;
+  MachinePostDominatorTree *MPDT;
+  MachineBlockFrequencyInfo *MBFI;
+  uint64_t EntryFreq;
 
   // Initialize class variables.
   void initialize(MachineFunction &MFParm);
 
   // Perform peepholes.
   bool simplifyCode(void);
 
   // Perform peepholes.
   bool eliminateRedundantCompare(void);
   bool eliminateRedundantTOCSaves(std::map<MachineInstr *, bool> &TOCSaves);
   void emitRLDICWhenLoweringJumpTables(MachineInstr &MI);
   void UpdateTOCSaves(std::map<MachineInstr *, bool> &TOCSaves,
                       MachineInstr *MI);
 
 public:
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<MachineDominatorTree>();
+    AU.addRequired<MachinePostDominatorTree>();
+    AU.addRequired<MachineBlockFrequencyInfo>();
     AU.addPreserved<MachineDominatorTree>();
+    AU.addPreserved<MachinePostDominatorTree>();
+    AU.addPreserved<MachineBlockFrequencyInfo>();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
   // Main entry point for this pass.
   bool runOnMachineFunction(MachineFunction &MF) override {
     if (skipFunction(MF.getFunction()))
       return false;
     initialize(MF);
     return simplifyCode();
   }
 };
 
 // Initialize class variables.
 void PPCMIPeephole::initialize(MachineFunction &MFParm) {
   MF = &MFParm;
   MRI = &MF->getRegInfo();
   MDT = &getAnalysis<MachineDominatorTree>();
+  MPDT = &getAnalysis<MachinePostDominatorTree>();
+  MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
+  EntryFreq = MBFI->getEntryFreq();
   TII = MF->getSubtarget<PPCSubtarget>().getInstrInfo();
   LLVM_DEBUG(dbgs() << "*** PowerPC MI peephole pass ***\n\n");
   LLVM_DEBUG(MF->dump());
 }
 
 static MachineInstr *getVRegDefOrNull(MachineOperand *Op,
                                       MachineRegisterInfo *MRI) {
   assert(Op && "Invalid Operand!");

 // Each time a new TOC save is encountered, it checks if any of the existing
 // ones are dominated by the new one. If so, it marks the existing one as
 // redundant by setting it's entry in the map as false. It then adds the new
 // instruction to the map with either true or false depending on if any
 // existing instructions dominated the new one.
 void PPCMIPeephole::UpdateTOCSaves(
   std::map<MachineInstr *, bool> &TOCSaves, MachineInstr *MI) {
   assert(TII->isTOCSaveMI(*MI) && "Expecting a TOC save instruction here");
+  assert(MF->getSubtarget<PPCSubtarget>().isELFv2ABI() &&
+         "TOC-save removal only supported on ELFv2");
+  PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();
+  MachineFrameInfo &MFI = MF->getFrameInfo();
+
+  MachineBasicBlock *Entry = &MF->front();
+  uint64_t CurrBlockFreq = MBFI->getBlockFreq(MI->getParent()).getFrequency();
+
+  // If the block in which the TOC save resides is in a block that
+  // post-dominates Entry, or a block that is hotter than entry (keep in mind
+  // that early MachineLICM has already run so the TOC save won't be hoisted)
+  // we can just do the save in the prologue.
+  if (!MFI.hasVarSizedObjects() &&
+      (CurrBlockFreq > EntryFreq || MPDT->dominates(MI->getParent(), Entry)))
+    FI->setMustSaveTOC(true);
+
+  // If we are saving the TOC in the prologue, all the TOC saves can be removed
+  // from the code.
+  if (FI->mustSaveTOC()) {
+    for (auto &TOCSave : TOCSaves)
+      TOCSave.second = false;
+    // Add new instruction to map.
+    TOCSaves[MI] = false;
+    return;
+  }
+
   bool Keep = true;
   for (auto It = TOCSaves.begin(); It != TOCSaves.end(); It++ ) {
     MachineInstr *CurrInst = It->first;
     // If new instruction dominates an existing one, mark existing one as
     // redundant.
     if (It->second && MDT->dominates(MI, CurrInst))
       It->second = false;
     // Check if the new instruction is redundant.
for (MachineBasicBlock &MBB : *MF) {
     if (ToErase) {
       ToErase->eraseFromParent();
       ToErase = nullptr;
     }
   }
 
   // Eliminate all the TOC save instructions which are redundant.
   Simplified |= eliminateRedundantTOCSaves(TOCSaves);
+  PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();
+  if (FI->mustSaveTOC())
+    NumTOCSavesInPrologue++;
+
   // We try to eliminate redundant compare instruction.
   Simplified |= eliminateRedundantCompare();
 
   return Simplified;
 }
 
 // helper functions for eliminateRedundantCompare
 static bool isEqOrNe(MachineInstr *BI) {
void PPCMIPeephole::emitRLDICWhenLoweringJumpTables(MachineInstr &MI) {
   LLVM_DEBUG(MI.dump());
   NumRotatesCollapsed++;
 }
 
 } // end default namespace
 
 INITIALIZE_PASS_BEGIN(PPCMIPeephole, DEBUG_TYPE,
                       "PowerPC MI Peephole Optimization", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree)
 INITIALIZE_PASS_END(PPCMIPeephole, DEBUG_TYPE,
                     "PowerPC MI Peephole Optimization", false, false)
 
 char PPCMIPeephole::ID = 0;
 FunctionPass*
 llvm::createPPCMIPeepholePass() { return new PPCMIPeephole(); }
 

llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h

class PPCFunctionInfo : public MachineFunctionInfo {
   /// Frame index where the old PIC base pointer is stored.
   int PICBasePointerSaveIndex = 0;
 
   /// MustSaveLR - Indicates whether LR is defined (or clobbered) in the current
   /// function.  This is only valid after the initial scan of the function by
   /// PEI.
   bool MustSaveLR;
 
+  /// MustSaveTOC - Indicates that the TOC save needs to be performed in the
+  /// prologue of the function. This is typically the case when there are
+  /// indirect calls in the function and it is more profitable to save the
+  /// TOC pointer in the prologue than in the block(s) containing the call(s).
+  bool MustSaveTOC = false;
+
   /// Do we have to disable shrink-wrapping? This has to be set if we emit any
   /// instructions that clobber LR in the entry block because discovering this
   /// in PEI is too late (happens after shrink-wrapping);
   bool ShrinkWrapDisabled = false;
 
   /// Does this function have any stack spills.
   bool HasSpills = false;
 
public:
   /// MustSaveLR - This is set when the prolog/epilog inserter does its initial
   /// scan of the function. It is true if the LR/LR8 register is ever explicitly
   /// defined/clobbered in the machine function (e.g. by calls and movpctolr,
   /// which is used in PIC generation), or if the LR stack slot is explicitly
   /// referenced by builtin_return_address.
   void setMustSaveLR(bool U) { MustSaveLR = U; }
   bool mustSaveLR() const    { return MustSaveLR; }
 
+  void setMustSaveTOC(bool U) { MustSaveTOC = U; }
+  bool mustSaveTOC() const    { return MustSaveTOC; }
+
   /// We certainly don't want to shrink wrap functions if we've emitted a
   /// MovePCtoLR8 as that has to go into the entry, so the prologue definitely
   /// has to go into the entry block.
   void setShrinkWrapDisabled(bool U) { ShrinkWrapDisabled = U; }
   bool shrinkWrapDisabled() const { return ShrinkWrapDisabled; }
 
   void setHasSpills()      { HasSpills = true; }
   bool hasSpills() const   { return HasSpills; }

llvm/test/CodeGen/PowerPC/MCSE-caller-preserved-reg.ll

 ; CHECK-LABEL: _ZN2CC3funEv:
 ; CHECK:    mflr 0
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    .cfi_offset r30, -16
 ; CHECK-NEXT:    std 30, -16(1)
 ; CHECK-NEXT:    std 0, 16(1)
 ; CHECK-NEXT:    stdu 1, -48(1)
-; CHECK-NEXT:    ld 12, 0(3)
-; CHECK-NEXT:    mr 30, 3
 ; CHECK-NEXT:    std 2, 24(1)
+; CHECK-NEXT:    mr 30, 3
+; CHECK-NEXT:    ld 12, 0(3)
 ; CHECK-NEXT:    mtctr 12
 ; CHECK-NEXT:    bctrl
 ; CHECK-NEXT:    ld 2, 24(1)
 ; CHECK-NEXT:    addis 3, 2, _ZN2CC2ccE@got@tlsgd@ha
 ; CHECK-NEXT:    addi 3, 3, _ZN2CC2ccE@got@tlsgd@l
 ; CHECK-NEXT:    bl __tls_get_addr(_ZN2CC2ccE@tlsgd)
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    ld 4, 0(3)

llvm/test/CodeGen/PowerPC/tocSaveInPrologue.ll

 ; CHECK-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r0, 16(r1)
 ; CHECK-NEXT:    stdu r1, -64(r1)
 ; CHECK-NEXT:    mr r29, r5
 ; CHECK-NEXT:    cmpwi cr1, r4, 11
 ; CHECK-NEXT:    mr r30, r3
 ; CHECK-NEXT:    extsw r28, r4
+; CHECK-NEXT:    std r2, 24(r1)
 ; CHECK-NEXT:    cmpwi r29, 1
 ; CHECK-NEXT:    cror 4*cr5+lt, lt, 4*cr1+lt
-; CHECK-NEXT:    bc 12, 4*cr5+lt, .LBB0_3
-; CHECK-NEXT:  # %bb.1: # %for.body.us.preheader
-; CHECK-NEXT:    std r2, 24(r1)
+; CHECK-NEXT:    bc 12, 4*cr5+lt, .LBB0_2
 ; CHECK-NEXT:    .p2align 5
-; CHECK-NEXT:  .LBB0_2: # %for.body.us
+; CHECK-NEXT:  .LBB0_1: # %for.body.us
 ; CHECK-NEXT:  
 ; CHECK-NEXT:    mtctr r30
 ; CHECK-NEXT:    mr r3, r28
 ; CHECK-NEXT:    mr r12, r30
 ; CHECK-NEXT:    bctrl
 ; CHECK-NEXT:    ld 2, 24(r1)
 ; CHECK-NEXT:    addi r29, r29, -1
 ; CHECK-NEXT:    cmplwi r29, 0
-; CHECK-NEXT:    bne cr0, .LBB0_2
-; CHECK-NEXT:  .LBB0_3: # %for.cond.cleanup
+; CHECK-NEXT:    bne cr0, .LBB0_1
+; CHECK-NEXT:  .LBB0_2: # %for.cond.cleanup
 ; CHECK-NEXT:    mtctr r30
 ; CHECK-NEXT:    mr r3, r28
 ; CHECK-NEXT:    mr r12, r30
-; CHECK-NEXT:    std r2, 24(r1)
 ; CHECK-NEXT:    bctrl
 ; CHECK-NEXT:    ld 2, 24(r1)
 ; CHECK-NEXT:    addi r1, r1, 64
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
 ; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload