[ppc64] Create instruction reorder chances in prologue and epilogue for post-RA-scheduler

[cycheng]

Summary

This patch changes SPUpdate (stack pointer update) instruction position in prologue and epilogue, in order to create more reorder chances for scheduler. Concept as below example:

  Current          This Patch

prologue:
  mflr 0           stdu 1, -240(1)
  mfcr 12          ..
  std 0, 16(1)     (scheduler gets more reorder chances)
  stw 12, 8(1)
  stdu 1, -240(1)

epilogue:
  addi 1, 1, 240   (scheduler gets more reorder chances)
  ld 0, 16(1)
  lwz 12, 8(1)
  mtocrf 32, 12
  mtlr 0           addi 1, 1, 240
  blr              blr

It moves SPUpdate to the beginning of prologue, and to the end (before return) of epilogue, so it can create more chances for scheduler to reorder mflr/mfcr/lr store/cr store, as well as mtlr/mtocrf/lr load/cr load.

Original issue: https://llvm.org/bugs/show_bug.cgi?id=25685

Sorry, after re-run SPEC2006 test, I found the performance was not improved.

[kbarton]

Do we still want to pursue this given it did not fix the reported problem?
In general, I think this is a good thing to do, but I also agree it can have limited benefits. I'm not sure the added complexity is worth it, especially since it has no impact on the original problem.

[cycheng]

No, please stop reviewing this patch.

We need discussion for the next step. I'm afraid our effort in prologue/epilogue instruction scheduling is not helpful to power8 and such kind of advanced out-of-order cpu :(

[cycheng]

Abandon this patch since prologue/epilogue instruction order is not a major performance issue.

lib/Target/PowerPC/PPCFrameLowering.cpp

 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/IR/Function.h"
 #include "llvm/Target/TargetOptions.h"
 
 using namespace llvm;
 
+// It changes SPUpdate (stack pointer update) instruction position, in order to
+// create more reorder chances for scheduler.
+//   Before           After
+//
+// prologue:
+//   mflr 0           stdu 1, -240(1)
+//   mfcr 12          ..
+//   std 0, 16(1)     (scheduler gets more reorder chances)
+//   stw 12, 8(1)
+//   stdu 1, -240(1)
+//
+// epilogue:
+//   addi 1, 1, 240   (scheduler gets more reorder chances)
+//   ld 0, 16(1)
+//   lwz 12, 8(1)
+//   mtocrf 32, 12
+//   mtlr 0           addi 1, 1, 240
+//   blr              blr
+static cl::opt<bool> RelaxPrologEpilog(
+  "ppc-relax-prologepilog-inst-order", cl::Hidden, cl::init(true),
+  cl::desc("Create instruction reorder chances for post-RA-scheduler"));
+
 /// VRRegNo - Map from a numbered VR register to its enum value.
 ///
 static const MCPhysReg VRRegNo[] = {
  PPC::V0 , PPC::V1 , PPC::V2 , PPC::V3 , PPC::V4 , PPC::V5 , PPC::V6 , PPC::V7 ,
  PPC::V8 , PPC::V9 , PPC::V10, PPC::V11, PPC::V12, PPC::V13, PPC::V14, PPC::V15,
  PPC::V16, PPC::V17, PPC::V18, PPC::V19, PPC::V20, PPC::V21, PPC::V22, PPC::V23,
  PPC::V24, PPC::V25, PPC::V26, PPC::V27, PPC::V28, PPC::V29, PPC::V30, PPC::V31
 };

 }
 
 bool PPCFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
 
   return findScratchRegister(TmpMBB, true);
 }
 
+// Indicate the memory operand of MI (load/store) is a stack object.
+// E.g. Assume MI = std r0, 16(r1)
+//      Then the function add "mem:ST8[FixedStack-1]" to 16(r1), for indicating
+//      16(r1) is a stack object memory operand.
+//
+// So scheduler can reorder MI with other load/stores.
+//
+// Without this, MI is treated as an global memory object load/store when
+// building schedule dependency graph (ScheduleDAGInstrs::buildSchedGraph),
+// cause MI can not be reordered by scheduler.
+static void addMemOperandAsStackObj(MachineInstr* MI, int Offset,
+                              MachineFunction &MF, bool IsPPC64, bool IsStore) {
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+
+  unsigned PtrByteSize = IsPPC64 ? 8 : 4;
+
+  // The Offset is an index to stack object list, check if it has been added
+  // to the list before
+  int FixedObjIdx = MFI->getObjectIndexBegin();
+  const int EndFixedObjIdx = 0;
+  for (; FixedObjIdx != EndFixedObjIdx; ++FixedObjIdx)
+    if (Offset == MFI->getObjectOffset(FixedObjIdx))
+      break;
+
+  int FrameIdx = FixedObjIdx;
+
+  // Create new stack object
+  if (FixedObjIdx == EndFixedObjIdx)
+    FrameIdx = MFI->CreateFixedObject(PtrByteSize, Offset, true, false);
+
+  // Indicate memory operand as a stack object
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      MachinePointerInfo::getFixedStack(MF, FrameIdx),
+      // Is MI a load or store instruction?
+      IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad,
+      MFI->getObjectSize(FrameIdx),
+      MFI->getObjectAlignment(FrameIdx));
+  MI->addMemOperand(MF, MMO);
+}
+
+static bool isEligibleForRelaxingPrologEpilog(bool IsPPC64, bool IsSVR4ABI,
+                 bool IsLargeFrame, bool HasBP, bool UsePICBase,
+                 unsigned FrameSize, int FPOffset, int LROffset, int CROffset) {
+  // Supporting: PPC64 with ELFv1 or ELFv2.
+  // And not supporting: Has Base Pointer, Use PIC Base, Large Frame
+  return RelaxPrologEpilog &&
+         IsPPC64 && IsSVR4ABI &&
+         !IsLargeFrame && !HasBP && !UsePICBase &&
+         // We don't support if add frame size cause one of them exceed 16-bit
+         // immediate value
+         isInt<16>(FPOffset + FrameSize) &&
+         isInt<16>(LROffset + FrameSize) &&
+         isInt<16>(CROffset + FrameSize);
+}
+
 void PPCFrameLowering::emitPrologue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const PPCInstrInfo &TII =
       *static_cast<const PPCInstrInfo *>(Subtarget.getInstrInfo());
   const PPCRegisterInfo *RegInfo =
       static_cast<const PPCRegisterInfo *>(Subtarget.getRegisterInfo());
if (MustSaveCR &&
     // If only one or two CR fields are clobbered, it could be more
     // efficient to use mfocrf to selectively save just those fields.
     MachineInstrBuilder MIB =
       BuildMI(MBB, MBBI, dl, TII.get(PPC::MFCR8), TempReg);
     for (unsigned i = 0, e = MustSaveCRs.size(); i != e; ++i)
       MIB.addReg(MustSaveCRs[i], RegState::ImplicitKill);
   }
 
+  int AdjustOffset = 0;
+  bool PlaceSPUpdateToTheBegin = false;
+  if (isEligibleForRelaxingPrologEpilog(isPPC64, isSVR4ABI,
+                                        isLargeFrame, HasBP, FI->usesPICBase(),
+                                        FrameSize, FPOffset, LROffset, 8)) {
+    AdjustOffset = FrameSize;
+    PlaceSPUpdateToTheBegin = true;
+  }
+
+  auto SaveRegToStack = [&](unsigned Reg, int Offset) {
+    MachineInstr *MI = BuildMI(MBB, MBBI, dl, StoreInst)
+                         .addReg(Reg)
+                         .addImm(Offset + AdjustOffset)
+                         .addReg(SPReg);
+    addMemOperandAsStackObj(MI, Offset, MF, isPPC64, true /*IsStore*/);
+  };
+
   if (HasFP)
     // FIXME: On PPC32 SVR4, we must not spill before claiming the stackframe.
-    BuildMI(MBB, MBBI, dl, StoreInst)
-      .addReg(FPReg)
-      .addImm(FPOffset)
-      .addReg(SPReg);
+    SaveRegToStack(FPReg, FPOffset);
 
   if (FI->usesPICBase())
     // FIXME: On PPC32 SVR4, we must not spill before claiming the stackframe.
-    BuildMI(MBB, MBBI, dl, StoreInst)
-      .addReg(PPC::R30)
-      .addImm(PBPOffset)
-      .addReg(SPReg);
+    SaveRegToStack(PPC::R30, PBPOffset);
 
   if (HasBP)
     // FIXME: On PPC32 SVR4, we must not spill before claiming the stackframe.
-    BuildMI(MBB, MBBI, dl, StoreInst)
-      .addReg(BPReg)
-      .addImm(BPOffset)
-      .addReg(SPReg);
+    SaveRegToStack(BPReg, BPOffset);
 
   if (MustSaveLR)
     // FIXME: On PPC32 SVR4, we must not spill before claiming the stackframe.
-    BuildMI(MBB, MBBI, dl, StoreInst)
-      .addReg(ScratchReg)
-      .addImm(LROffset)
-      .addReg(SPReg);
+    SaveRegToStack(ScratchReg, LROffset);
 
   if (MustSaveCR &&
       !(SingleScratchReg && MustSaveLR)) // will only occur for PPC64
+  {
+    MachineInstr* MI =
       BuildMI(MBB, MBBI, dl, TII.get(PPC::STW8))
         .addReg(TempReg, getKillRegState(true))
-      .addImm(8)
+        .addImm(8 + AdjustOffset)
         .addReg(SPReg);
+    addMemOperandAsStackObj(MI, 8, MF, isPPC64, true /*IsStore*/);
+  }
 
   // Skip the rest if this is a leaf function & all spills fit in the Red Zone.
   if (!FrameSize) return;
 
   // Adjust stack pointer: r1 += NegFrameSize.
   // If there is a preferred stack alignment, align R1 now
 
   if (HasBP) {
if (!isLargeFrame) {
         .addReg(TempReg, RegState::Kill);
     }
     BuildMI(MBB, MBBI, dl, StoreUpdtIdxInst, SPReg)
       .addReg(SPReg, RegState::Kill)
       .addReg(SPReg)
       .addReg(ScratchReg);
 
   } else if (!isLargeFrame) {
-    BuildMI(MBB, MBBI, dl, StoreUpdtInst, SPReg)
+    MachineBasicBlock::iterator InsrtPos = MBBI;
+    if (PlaceSPUpdateToTheBegin)
+      InsrtPos = MBB.begin();
+    BuildMI(MBB, InsrtPos, dl, StoreUpdtInst, SPReg)
       .addReg(SPReg)
       .addImm(NegFrameSize)
       .addReg(SPReg);
 
   } else {
     BuildMI(MBB, MBBI, dl, LoadImmShiftedInst, ScratchReg)
       .addImm(NegFrameSize >> 16);
     BuildMI(MBB, MBBI, dl, OrImmInst, ScratchReg)
if (UsesTCRet) {
         FrameSize += StackAdj;
     }
   }
 
   // Frames of 32KB & larger require special handling because they cannot be
   // indexed into with a simple LD/LWZ immediate offset operand.
   bool isLargeFrame = !isInt<16>(FrameSize);
 
+  int AdjustOffset = 0;
+  bool PlaceSPUpdateToTheEnd = false;
+  if (isEligibleForRelaxingPrologEpilog(isPPC64, isSVR4ABI,
+                                        isLargeFrame, HasBP, FI->usesPICBase(),
+                                        FrameSize, FPOffset, LROffset, 8)) {
+    AdjustOffset = FrameSize;
+    PlaceSPUpdateToTheEnd = true;
+  }
+
   if (FrameSize) {
     // In the prologue, the loaded (or persistent) stack pointer value is offset
     // by the STDU/STDUX/STWU/STWUX instruction.  Add this offset back now.
 
     // If this function contained a fastcc call and GuaranteedTailCallOpt is
     // enabled (=> hasFastCall()==true) the fastcc call might contain a tail
     // call which invalidates the stack pointer value in SP(0). So we use the
     // value of R31 in this case.
if (FI->hasFastCall()) {
       BuildMI(MBB, MBBI, dl, AddImmInst, SPReg)
         .addReg(SPReg)
         .addImm(FrameSize);
     } else {
       BuildMI(MBB, MBBI, dl, LoadInst, SPReg)
         .addImm(0)
         .addReg(SPReg);
     }
-  }
+
+    // Change insert point to Stack Pointer Update Instruction. So later stack
+    // object restore instructions are inserted before SPUpdate instr.
+    if (PlaceSPUpdateToTheEnd)
+      MBBI = --MBBI;
+  }
+
+  // We need a stack pointer that keeps original sp value, so we can restore
+  // LR, CR, FP with correct offset.
+  // e.g. SPEC2006/403.gcc/real.c: asctoeg()
+  //   Prologue:
+  //     stdu    r1,-432(r1)
+  //     std     r0,448(r1)
+  //     mr      r31,r1         <= FPReg contains original sp value
+  //   Body:
+  //     stdux   r4,r1,r3       <= Because of Varied Sized Object, the SPReg
+  //                               is changed
+  //   Epilogue:
+  //     ld      r0,448(r31)    <= FPReg contains correct sp value, use it to
+  //                               restore LR, CR, FP.
+  //     ld      r1,0(r1)
+  //
+  // So when we have fp, just use it.
+  unsigned CorrectSPReg = SPReg;
+  if (PlaceSPUpdateToTheEnd && HasFP)
+    CorrectSPReg = FPReg;
+
+  if (PlaceSPUpdateToTheEnd && MFI->hasVarSizedObjects())
+    assert(HasFP && "Expecting a valid frame pointer.");
 
   assert((isPPC64 || !MustSaveCR) &&
          "Epilogue CR restoring supported only in 64-bit mode");
 
   // If we need to save both the LR and the CR and we only have one available
   // scratch register, we must do them one at a time.
   if (MustSaveCR && SingleScratchReg && MustSaveLR) {
     BuildMI(MBB, MBBI, dl, TII.get(PPC::LWZ8), TempReg)
       .addImm(8)
-      .addReg(SPReg);
+      .addReg(CorrectSPReg);
     for (unsigned i = 0, e = MustSaveCRs.size(); i != e; ++i)
       BuildMI(MBB, MBBI, dl, TII.get(PPC::MTOCRF8), MustSaveCRs[i])
         .addReg(TempReg, getKillRegState(i == e-1));
   }
 
+  auto RestoreRegFromStack = [&](const MCInstrDesc& LoadInst,
+                                 unsigned Reg, int Offset) {
+    MachineInstr *MI = BuildMI(MBB, MBBI, dl, LoadInst, Reg)
+                         .addImm(Offset + AdjustOffset)
+                         .addReg(CorrectSPReg);
+    addMemOperandAsStackObj(MI, Offset, MF, isPPC64, false /*IsStore*/);
+  };
+
   if (MustSaveLR)
-    BuildMI(MBB, MBBI, dl, LoadInst, ScratchReg)
-      .addImm(LROffset)
-      .addReg(SPReg);
+    RestoreRegFromStack(LoadInst, ScratchReg, LROffset);
 
   if (MustSaveCR &&
       !(SingleScratchReg && MustSaveLR)) // will only occur for PPC64
-    BuildMI(MBB, MBBI, dl, TII.get(PPC::LWZ8), TempReg)
-      .addImm(8)
-      .addReg(SPReg);
+    RestoreRegFromStack(TII.get(PPC::LWZ8), TempReg, 8);
 
   if (HasFP)
-    BuildMI(MBB, MBBI, dl, LoadInst, FPReg)
-      .addImm(FPOffset)
-      .addReg(SPReg);
+    RestoreRegFromStack(LoadInst, FPReg, FPOffset);
 
   if (FI->usesPICBase())
     // FIXME: On PPC32 SVR4, we must not spill before claiming the stackframe.
     BuildMI(MBB, MBBI, dl, LoadInst)
       .addReg(PPC::R30)
       .addImm(PBPOffset)
-      .addReg(SPReg);
+      .addReg(CorrectSPReg);
 
   if (HasBP)
-    BuildMI(MBB, MBBI, dl, LoadInst, BPReg)
-      .addImm(BPOffset)
-      .addReg(SPReg);
+    RestoreRegFromStack(LoadInst, BPReg, BPOffset);
 
   if (MustSaveCR &&
       !(SingleScratchReg && MustSaveLR)) // will only occur for PPC64
     for (unsigned i = 0, e = MustSaveCRs.size(); i != e; ++i)
       BuildMI(MBB, MBBI, dl, TII.get(PPC::MTOCRF8), MustSaveCRs[i])
         .addReg(TempReg, getKillRegState(i == e-1));
 
   if (MustSaveLR)
     BuildMI(MBB, MBBI, dl, MTLRInst).addReg(ScratchReg);
 
   // Callee pop calling convention. Pop parameter/linkage area. Used for tail
   // call optimization
   if (IsReturnBlock) {
+    // The current MBBI is SPUpdate instr, change it back to first terminator of
+    // MBB.
+    if (PlaceSPUpdateToTheEnd)
+      MBBI = ++MBBI;
+
     unsigned RetOpcode = MBBI->getOpcode();
     if (MF.getTarget().Options.GuaranteedTailCallOpt &&
         (RetOpcode == PPC::BLR || RetOpcode == PPC::BLR8) &&
         MF.getFunction()->getCallingConv() == CallingConv::Fast) {
       PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
       unsigned CallerAllocatedAmt = FI->getMinReservedArea();
 
       if (CallerAllocatedAmt && isInt<16>(CallerAllocatedAmt)) {

lib/Target/PowerPC/PPCInstr64Bit.td

 
 let Defs = [X1], Uses = [X1] in
 def DYNALLOC8 : Pseudo<(outs g8rc:$result), (ins g8rc:$negsize, memri:$fpsi),"#DYNALLOC8",
                        [(set i64:$result,
                              (PPCdynalloc i64:$negsize, iaddr:$fpsi))]>;
 def DYNAREAOFFSET8 : Pseudo<(outs i64imm:$result), (ins memri:$fpsi), "#DYNAREAOFFSET8",
                        [(set i64:$result, (PPCdynareaoffset iaddr:$fpsi))]>;
 
-let Defs = [LR8] in {
+// Add no side effects attribute so post-RA-scheduler can reorder it.
+let Defs = [LR8], hasSideEffects = 0 in {
 def MTLR8  : XFXForm_7_ext<31, 467, 8, (outs), (ins g8rc:$rS),
                            "mtlr $rS", IIC_SprMTSPR>,
              PPC970_DGroup_First, PPC970_Unit_FXU;
 }
-let Uses = [LR8] in {
+let Uses = [LR8], hasSideEffects = 0 in {
 def MFLR8  : XFXForm_1_ext<31, 339, 8, (outs g8rc:$rT), (ins),
                            "mflr $rT", IIC_SprMFSPR>,
              PPC970_DGroup_First, PPC970_Unit_FXU;
 }
 } // Interpretation64Bit
 
 //===----------------------------------------------------------------------===//
 // Fixed point instructions.

lib/Target/PowerPC/PPCScheduleP8.td

InstrItinData<IIC_LdStSTWCX   , [InstrStage<1, [P8_DU1], 0>,
                                    InstrStage<1, [P8_DU2], 0>,
                                    InstrStage<1, [P8_DU3], 0>,
                                    InstrStage<1, [P8_DU4], 0>,
                                    InstrStage<1, [P8_DU5], 0>,
                                    InstrStage<1, [P8_DU6], 0>,
                                    InstrStage<1, [P8_LSU1, P8_LSU2], 0>,
                                    InstrStage<1, [P8_LU1, P8_LU2]>],
                                   [1, 1, 1]>,
+  // Use 2 cycles for [P8_CRU] stage, as IIC_SprMTSPR, IIC_SprMFSPR does.
+  // See comments in IIC_SprMTSPR.
   InstrItinData<IIC_SprMFCR     , [InstrStage<1, [P8_DU1], 0>,
-                                   InstrStage<1, [P8_CRU]>],
+                                   InstrStage<2, [P8_CRU]>],
                                   [6, 1]>,
   InstrItinData<IIC_SprMFCRF    , [InstrStage<1, [P8_DU1], 0>,
                                    InstrStage<1, [P8_CRU]>],
                                   [3, 1]>,
+  // Change execution pipe from [P8_FXU1, P8_FXU2] to [P8_CRU], we tell
+  // post-RA-scheduler that 'mflr' and 'mfcr' use the same resource, and both
+  // occupy execution pipe 2 cycles, so post-RA-sched will try to separate them.
+  //
+  // The reason comes from: P8_um_external_v12_17JUN2015_pub.pdf p.221:
+  // "If two mfsprs or mtsprs are issued at the same time on any FXU or CRU"
+  // "issue port, one must be cancelled"
   InstrItinData<IIC_SprMTSPR    , [InstrStage<1, [P8_DU1], 0>,
-                                   InstrStage<1, [P8_FXU1, P8_FXU2]>],
+                                   InstrStage<2, [P8_CRU]>],
                                   [4, 1]>, // mtctr
+  // For modeling mflr/mfctr
+  InstrItinData<IIC_SprMFSPR    , [InstrStage<1, [P8_DU1]>,
+                                   InstrStage<2, [P8_CRU]>],
+                                  [6, 1]>,
   InstrItinData<IIC_FPGeneral   , [InstrStage<1, [P8_DU1, P8_DU2, P8_DU3,
                                                   P8_DU4, P8_DU5, P8_DU6], 0>,
                                    InstrStage<1, [P8_FPU1, P8_FPU2]>],
                                   [5, 1, 1]>,
   InstrItinData<IIC_FPAddSub    , [InstrStage<1, [P8_DU1, P8_DU2, P8_DU3,
                                                   P8_DU4, P8_DU5, P8_DU6], 0>,
                                    InstrStage<1, [P8_FPU1, P8_FPU2]>],
                                   [5, 1, 1]>,

test/CodeGen/PowerPC/crsave.ll

 ; RUN: llc -O0 -disable-fp-elim -mtriple=powerpc-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC32
-; RUN: llc -O0 -mtriple=powerpc64-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC64
+; RUN: llc -O0 -mtriple=powerpc64-unknown-linux-gnu -mcpu=g5 -ppc-relax-prologepilog-inst-order=false < %s | FileCheck %s -check-prefix=PPC64
+; RUN: llc -O0 -mtriple=powerpc64-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC64-OPT-PE
 
 declare void @foo()
 
 define i32 @test_cr2() nounwind uwtable {
 entry:
   %ret = alloca i32, align 4
   %0 = call i32 asm sideeffect "\0A\09mtcr $4\0A\09cmpw 2,$2,$1\0A\09mfcr $0", "=r,r,r,r,r,~{cr2}"(i32 1, i32 2, i32 3, i32 0) nounwind
   store i32 %0, i32* %ret, align 4

 ; PPC64: .cfi_def_cfa_offset 128
 ; PPC64: .cfi_offset lr, 16
 ; PPC64: .cfi_offset cr2, 8
 ; PPC64: addi 1, 1, [[AMT]]
 ; PPC64: lwz 12, 8(1)
 ; PPC64: mtocrf 32, 12
 ; PPC64: .cfi_endproc
 
+; PPC64-OPT-PE: .cfi_startproc
+; PPC64-OPT-PE: stdu 1, -128(1)
+; PPC64-OPT-PE: mfcr 12
+; PPC64-OPT-PE: stw 12, 136(1)
+; PPC64-OPT-PE: .cfi_def_cfa_offset 128
+; PPC64-OPT-PE: .cfi_offset lr, 16
+; PPC64-OPT-PE: .cfi_offset cr2, 8
+
 define i32 @test_cr234() nounwind {
 entry:
   %ret = alloca i32, align 4
   %0 = call i32 asm sideeffect "\0A\09mtcr $4\0A\09cmpw 2,$2,$1\0A\09cmpw 3,$2,$2\0A\09cmpw 4,$2,$3\0A\09mfcr $0", "=r,r,r,r,r,~{cr2},~{cr3},~{cr4}"(i32 1, i32 2, i32 3, i32 0) nounwind
   store i32 %0, i32* %ret, align 4
   call void @foo()
   %1 = load i32, i32* %ret, align 4
   ret i32 %1

 ; PPC64: stw 12, 8(1)
 ; PPC64: stdu 1, -[[AMT:[0-9]+]](1)
 ; PPC64: addi 1, 1, [[AMT]]
 ; PPC64: lwz 12, 8(1)
 ; PPC64: mtocrf 32, 12
 ; PPC64: mtocrf 16, 12
 ; PPC64: mtocrf 8, 12
 
+; PPC64-OPT-PE: stdu 1, -128(1)
+; PPC64-OPT-PE: mfcr 12
+; PPC64-OPT-PE: stw 12, 136(1)

test/CodeGen/PowerPC/ppc-shrink-wrapping.ll

 ; Set the first argument to zero.
 ; CHECK-NEXT: li 3, 0
 ; CHECK-NEXT: bl doSomething
 ;
 ; With shrink-wrapping, epilogue is just after the call.
 ; Restore the link register and return.
 ; Note that there could be other epilog code before the link register is 
 ; restored but we will not check for it here.
-; ENABLE: mtlr
+; ENABLE-DAG: mtlr
+; ENABLE-DAG: addi 1, 1, {{[0-9]+}}
 ; ENABLE-NEXT: blr
 ;
 ; DISABLE: [[EXIT_LABEL]]:
 ;
 ; Without shrink-wrapping, epilogue is in the exit block.
 ; Epilogue code. (What we pop does not matter.)
-; DISABLE: mtlr {{[0-9]+}}
+; DISABLE-DAG: mtlr {{[0-9]+}}
+; DISABLE-DAG: addi 1, 1, {{[0-9]+}}
 ; DISABLE-NEXT: blr
 ;
 
 define i32 @foo(i32 %a, i32 %b) {
   %tmp = alloca i32, align 4
   %tmp2 = icmp slt i32 %a, %b
   br i1 %tmp2, label %true, label %false
 

 ; DISABLE: b .[[EPILOG_BB:LBB[0-9_]+]]
 ;
 ; DISABLE: .[[ELSE_LABEL]]: # %if.else
 ; Shift second argument by one and store into returned register.
 ; DISABLE: slwi 3, 4, 1
 ; DISABLE: .[[EPILOG_BB]]: # %if.end
 ;
 ; Epilogue code.
-; CHECK: mtlr {{[0-9]+}}
+; CHECK-DAG: mtlr {{[0-9]+}}
+; CHECK-DAG: addi 1, 1, {{[0-9]+}}
 ; CHECK-NEXT: blr
 ;
 ; ENABLE: .[[ELSE_LABEL]]: # %if.else
 ; Shift second argument by one and store into returned register.
 ; ENABLE: slwi 3, 4, 1
 ; ENABLE-NEXT: blr
 define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) {
 entry:

 ; CHECK: bl something
 ; CHECK-DAG: addi [[IV]], [[IV]], -1
 ; CHECK-DAG: add [[SUM]], 3, [[SUM]] 
 ; CHECK-NEXT: cmplwi [[IV]], 0
 ; CHECK-NEXT: bne 0, .[[LOOP]]
 ;
 ; Next BB
 ; CHECK: %for.exit
-; CHECK: mtlr {{[0-9]+}}
+; CHECK-DAG: mtlr {{[0-9]+}}
+; CHECK-DAG: addi 1, 1, {{[0-9]+}}
 ; CHECK-NEXT: blr
 define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {
 entry:
   br label %for.preheader
 
 for.preheader:
   tail call void asm "nop", ""()
   br label %for.body

 ; DISABLE: b .[[EPILOG_BB:LBB[0-9_]+]]
 ;
 ; DISABLE: .[[ELSE_LABEL]]: # %if.else
 ; Shift second argument by one and store into returned register.
 ; DISABLE: slwi 3, 4, 1
 ;
 ; DISABLE: .[[EPILOG_BB]]: # %if.end
 ; Epilog code
-; CHECK: mtlr {{[0-9]+}}
+; CHECK-DAG: mtlr {{[0-9]+}}
+; CHECK-DAG: addi 1, 1, {{[0-9]+}}
 ; CHECK-NEXT: blr
 ; 
 ; ENABLE: .[[ELSE_LABEL]]: # %if.else
 ; Shift second argument by one and store into returned register.
 ; ENABLE: slwi 3, 4, 1
 ; ENABLE-NEXT: blr
 define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) {
 entry:

 ;
 ; ENABLE: cmplwi 0, 3, 0
 ; ENABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; Make sure we save the link register
 ; CHECK: mflr {{[0-9]+}}
 ;
-; DISABLE: cmplwi 0, 3, 0
+; DISABLE-DAG: cmplwi 0, 3, 0
+; DISABLE-DAG: std 0, {{[0-9]+([0-9]+)}}
 ; DISABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; CHECK: bl somethingElse
 ;
 ; Loop preheader
 ; CHECK-DAG: li [[SUM:[0-9]+]], 0
 ; CHECK-DAG: li [[IV:[0-9]+]], 10
 ; 

 ; DISABLE: b .[[EPILOG_BB:LBB[0-9_]+]]
 ;
 ; DISABLE: .[[ELSE_LABEL]]: # %if.else
 ; Shift second argument by one and store into returned register.
 ; DISABLE: slwi 3, 4, 1
 ; DISABLE: .[[EPILOG_BB]]: # %if.end
 ;
 ; Epilogue code.
-; CHECK: mtlr {{[0-9]+}}
+; CHECK-DAG: mtlr {{[0-9]+}}
+; CHECK-DAG: addi 1, 1, {{[0-9]+}}
 ; CHECK-NEXT: blr
 ;
 ; ENABLE: .[[ELSE_LABEL]]: # %if.else
 ; Shift second argument by one and store into returned register.
 ; ENABLE: slwi 3, 4, 1
 ; ENABLE-NEXT: blr
 define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) #0 {
 entry:

 ; CHECK-NEXT: mr 6, 3
 ; CHECK-NEXT: mr 7, 3
 ; CHECK-NEXT: mr 8, 3
 ; CHECK-NEXT: mr 9, 3
 ; CHECK-NEXT: bl someVariadicFunc
 ; CHECK: slwi 3, 3, 3
 ; DISABLE: b .[[EPILOGUE_BB:LBB[0-9_]+]]
 ;
-; ENABLE: mtlr {{[0-9]+}}
+; ENABLE-DAG: mtlr {{[0-9]+}}
+; ENABLE-DAG: addi 1, 1, {{[0-9]+}}
 ; ENABLE-NEXT: blr
 ;
 ; CHECK: .[[ELSE_LABEL]]: # %if.else
 ; CHECK-NEXT: slwi 3, 4, 1
 ; 
 ; DISABLE: .[[EPILOGUE_BB]]: # %if.end
 ; DISABLE: mtlr
 ; CHECK: blr

 ; DISABLE: mflr {{[0-9]+}}
 ;
 ; CHECK: cmplwi 3, 0
 ; CHECK-NEXT: bne{{[-]?}} 0, .[[ABORT:LBB[0-9_]+]]
 ;
 ; CHECK: li 3, 42
 ;
 ; DISABLE: mtlr {{[0-9]+}}
+; DISABLE: addi 1, 1, {{[0-9]+}}
 ;
 ; CHECK-NEXT: blr
 ;
 ; CHECK: .[[ABORT]]: # %if.abort
 ;
 ; ENABLE: mflr {{[0-9]+}}
 ;
 ; CHECK: bl abort

test/CodeGen/PowerPC/ppc64-calls.ll

 
 ; Indirect calls requires a full stub creation
 define void @test_indirect(void ()* nocapture %fp) nounwind {
 ; CHECK-LABEL: test_indirect:
   tail call void %fp() nounwind
 ; CHECK: ld [[FP:[0-9]+]], 0(3)
 ; CHECK: ld 11, 16(3)
 ; CHECK: ld 2, 8(3)
-; CHECK-NEXT: mtctr [[FP]]
+; CHECK: mtctr [[FP]]
 ; CHECK-NEXT: bctrl
 ; CHECK-NEXT: ld 2, 40(1)
   ret void
 }
 
 ; Absolute values must use the regular indirect call sequence
 ; The main purpose of this test is to ensure that BLA is not
 ; used on 64-bit SVR4 (as e.g. on Darwin).
 define void @test_abs() nounwind {
 ; CHECK-LABEL: test_abs:
   tail call void inttoptr (i64 1024 to void ()*)() nounwind
 ; CHECK: ld [[FP:[0-9]+]], 1024(0)
 ; CHECK: ld 11, 1040(0)
 ; CHECK: ld 2, 1032(0)
-; CHECK-NEXT: mtctr [[FP]]
+; CHECK: mtctr [[FP]]
 ; CHECK-NEXT: bctrl
 ; CHECK-NEXT: ld 2, 40(1)
   ret void
 }
 
 declare double @sin(double) nounwind
 
 ; External functions call should also have a 'nop'

test/CodeGen/PowerPC/ppc64-nest.ll

 ; CHECK-NEXT: blr
 
   ret i8* %arg
 }
 
 define i8* @nest_caller(i8* %arg) nounwind {
 ; CHECK-LABEL: nest_caller:
 ; CHECK: mr 11, 3
-; CHECK-NEXT: bl nest_receiver
+; CHECK: bl nest_receiver
 ; CHECK: blr
 
   %result = call i8* @nest_receiver(i8* nest %arg)
   ret i8* %result
 }
 
 define void @test_indirect(i32 ()* nocapture %f, i8* %p) {
 entry:

test/CodeGen/PowerPC/ppc64-stackmap-nops.ll

-; RUN: llc < %s -mtriple=powerpc64-unknown-gnu-linux | FileCheck %s
+; RUN: llc < %s -ppc-relax-prologepilog-inst-order=false -mtriple=powerpc64-unknown-gnu-linux | FileCheck %s
 
 define void @test_shadow_optimization() {
 entry:
 ; Expect 12 bytes worth of nops here rather than 32: With the shadow optimization
 ; in place, 20 bytes will be consumed by the frame teardown and return instr.
 ; CHECK-LABEL: test_shadow_optimization:
 
 ; CHECK:      nop

test/CodeGen/PowerPC/retaddr2.ll

-; RUN: llc -mcpu=pwr7 < %s | FileCheck %s
+; RUN: llc -mcpu=pwr7 -ppc-relax-prologepilog-inst-order=false < %s | FileCheck %s
+; RUN: llc -mcpu=pwr7 < %s | FileCheck %s -check-prefix=CHECK-OPT-PE
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind readnone
 define i8* @test1() #0 {
 entry:
   %0 = tail call i8* @llvm.returnaddress(i32 0)
   ret i8* %0
 }
 
 ; CHECK-LABEL: @test1
 ; CHECK: mflr {{[0-9]+}}
 ; CHECK: std 0, 16(1)
 ; CHECK-DAG: ld 3, 64(1)
 ; CHECK-DAG: ld [[SR:[0-9]+]], 16(1)
 ; CHECK: mtlr [[SR]]
 ; CHECK: blr
 
+; CHECK-OPT-PE-LABEL: @test1
+; CHECK-OPT-PE: mflr {{[0-9]+}}
+; CHECK-OPT-PE: std 0, 64(1)
+; CHECK-OPT-PE-NOT: addi 1, 1
+; CHECK-OPT-PE-DAG: ld 3, 64(1)
+; CHECK-OPT-PE-DAG: ld [[SR:[0-9]+]], 64(1)
+; CHECK-OPT-PE: mtlr [[SR]]
+; CHECK-OPT-PE: blr
+
 ; Function Attrs: nounwind readnone
 declare i8* @llvm.returnaddress(i32) #0
 
 attributes #0 = { nounwind readnone }
 

test/CodeGen/PowerPC/sjlj.ll

 ; CHECK: .LBB1_4:
 
 ; CHECK: lfd
 ; CHECK: lvx
 ; CHECK: ld
 ; CHECK: blr
 
 ; CHECK: .LBB1_5:
-; CHECK: mflr [[REGL:[0-9]+]]
-; CHECK: ld [[REG2:[0-9]+]], [[OFF]](31)                   # 8-byte Folded Reload
+; CHECK-DAG: mflr [[REGL:[0-9]+]]
+; CHECK-DAG: ld [[REG2:[0-9]+]], [[OFF]](31)                   # 8-byte Folded Reload
 ; CHECK: std [[REGL]], 8([[REG2]])
 ; CHECK: li 3, 0
 
 ; CHECK-NOAV: @main
 ; CHECK-NOAV-NOT: stvx
 ; CHECK-NOAV: bcl
 ; CHECK-NOAV: mflr
 ; CHECK-NOAV: bl foo