[PowerPC] duplicate the tryCandidate from base to PowerPC scheduler

[steven.zhang]

Summary

We didn't handle the scheduling heuristic for PowerPC well which might hit assertion when scheduler verifying is enabled. That because the tryCandidate have sideeffect(change the reason in Cand) and it is unable to undo the sideeffect inside powerpc scheduler.

So, we have to duplicate the tryCandidate from base and then, add the target specific heuristic. This is not a good idea and easy to be out of sync. And there is a follow up rework patch to fix this. D86684

[steven.zhang]

Confirm that no bmk deg found. Gentle ping ...

[steven.zhang]

Abandon this PR as I want to merge this patch into the refactor patch.

llvm/lib/Target/PowerPC/PPCMachineScheduler.cpp

 bool PPCPreRASchedStrategy::biasAddiLoadCandidate(SchedCandidate &Cand,
                                                   SchedCandidate &TryCand,
                                                   SchedBoundary &Zone) const {
   if (DisableAddiLoadHeuristic)
     return false;
 
   SchedCandidate &FirstCand = Zone.isTop() ? TryCand : Cand;
   SchedCandidate &SecondCand = Zone.isTop() ? Cand : TryCand;
-  if (isADDIInstr(FirstCand) && SecondCand.SU->getInstr()->mayLoad()) {
-    TryCand.Reason = Stall;
-    return true;
-  }
-  if (FirstCand.SU->getInstr()->mayLoad() && isADDIInstr(SecondCand)) {
-    TryCand.Reason = NoCand;
-    return true;
-  }
-
-  return false;
+  return tryGreater(
+      isADDIInstr(FirstCand) && SecondCand.SU->getInstr()->mayLoad(),
+      FirstCand.SU->getInstr()->mayLoad() && isADDIInstr(SecondCand), TryCand,
+      Cand, Stall);
 }
 
+// Most of the code is duplicated with GenericScheduler::tryCandidate.
+// TODO - Add some infrastructure to reuse the logic to avoid the out of sync.
 void PPCPreRASchedStrategy::tryCandidate(SchedCandidate &Cand,
                                          SchedCandidate &TryCand,
                                          SchedBoundary *Zone) const {
-  GenericScheduler::tryCandidate(Cand, TryCand, Zone);
+  // Initialize the candidate if needed.
+  if (!Cand.isValid()) {
+    TryCand.Reason = NodeOrder;
+    return;
+  }
+
+  // Bias PhysReg Defs and copies to their uses and defined respectively.
+  if (tryGreater(biasPhysReg(TryCand.SU, TryCand.AtTop),
+                 biasPhysReg(Cand.SU, Cand.AtTop), TryCand, Cand, PhysReg))
+    return;
+
+  // Avoid exceeding the target's limit.
+  if (DAG->isTrackingPressure() &&
+      tryPressure(TryCand.RPDelta.Excess, Cand.RPDelta.Excess, TryCand, Cand,
+                  RegExcess, TRI, DAG->MF))
+    return;
+
+  // Avoid increasing the max critical pressure in the scheduled region.
+  if (DAG->isTrackingPressure() &&
+      tryPressure(TryCand.RPDelta.CriticalMax, Cand.RPDelta.CriticalMax,
+                  TryCand, Cand, RegCritical, TRI, DAG->MF))
+    return;
+
+  // We only compare a subset of features when comparing nodes between
+  // Top and Bottom boundary. Some properties are simply incomparable, in many
+  // other instances we should only override the other boundary if something
+  // is a clear good pick on one boundary. Skip heuristics that are more
+  // "tie-breaking" in nature.
+  bool SameBoundary = Zone != nullptr;
+  if (SameBoundary) {
+    // For loops that are acyclic path limited, aggressively schedule for
+    // latency. Within an single cycle, whenever CurrMOps > 0, allow normal
+    // heuristics to take precedence.
+    if (Rem.IsAcyclicLatencyLimited && !Zone->getCurrMOps() &&
+        tryLatency(TryCand, Cand, *Zone))
+      return;
+
+    // Prioritize instructions that read unbuffered resources by stall cycles.
+    if (tryLess(Zone->getLatencyStallCycles(TryCand.SU),
+                Zone->getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
+      return;
+  }
 
-  if (!Cand.isValid() || !Zone)
+  // Keep clustered nodes together to encourage downstream peephole
+  // optimizations which may reduce resource requirements.
+  //
+  // This is a best effort to set things up for a post-RA pass. Optimizations
+  // like generating loads of multiple registers should ideally be done within
+  // the scheduler pass by combining the loads during DAG postprocessing.
+  const SUnit *CandNextClusterSU =
+      Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
+  const SUnit *TryCandNextClusterSU =
+      TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred();
+  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
+                 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
     return;
 
-  // Add powerpc specific heuristic only when TryCand isn't selected or
-  // selected as node order.
-  if (TryCand.Reason != NodeOrder && TryCand.Reason != NoCand)
+  if (SameBoundary) {
+    // Weak edges are for clustering and other constraints.
+    if (tryLess(getWeakLeft(TryCand.SU, TryCand.AtTop),
+                getWeakLeft(Cand.SU, Cand.AtTop), TryCand, Cand, Weak))
+      return;
+  }
+
+  // Avoid increasing the max pressure of the entire region.
+  if (DAG->isTrackingPressure() &&
+      tryPressure(TryCand.RPDelta.CurrentMax, Cand.RPDelta.CurrentMax, TryCand,
+                  Cand, RegMax, TRI, DAG->MF))
+    return;
+
+  if (SameBoundary) {
+    // Avoid critical resource consumption and balance the schedule.
+    TryCand.initResourceDelta(DAG, SchedModel);
+    if (tryLess(TryCand.ResDelta.CritResources, Cand.ResDelta.CritResources,
+                TryCand, Cand, ResourceReduce))
+      return;
+    if (tryGreater(TryCand.ResDelta.DemandedResources,
+                   Cand.ResDelta.DemandedResources, TryCand, Cand,
+                   ResourceDemand))
+      return;
+
+    // Avoid serializing long latency dependence chains.
+    // For acyclic path limited loops, latency was already checked above.
+    if (!RegionPolicy.DisableLatencyHeuristic && TryCand.Policy.ReduceLatency &&
+        !Rem.IsAcyclicLatencyLimited && tryLatency(TryCand, Cand, *Zone))
       return;
 
     // There are some benefits to schedule the ADDI before the load to hide the
     // latency, as RA may create a true dependency between the load and addi.
     if (biasAddiLoadCandidate(Cand, TryCand, *Zone))
       return;
+
+    // Fall through to original instruction order.
+    if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) ||
+        (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
+      TryCand.Reason = NodeOrder;
+    }
+  }
 }
 
 bool PPCPostRASchedStrategy::biasAddiCandidate(SchedCandidate &Cand,
                                                SchedCandidate &TryCand) const {
   if (!EnableAddiHeuristic)
     return false;
 
-  if (isADDIInstr(TryCand) && !isADDIInstr(Cand)) {
-    TryCand.Reason = Stall;
-    return true;
-  }
-  return false;
+  return tryGreater(isADDIInstr(TryCand) && !isADDIInstr(Cand),
+                    isADDIInstr(Cand) && !isADDIInstr(TryCand), TryCand, Cand,
+                    Stall);
 }
 
+// Most of the code is duplicated with PostGenericScheduler::tryCandidate.
+// TODO - Add some infrastructure to reuse the logic to avoid the out of sync.
 void PPCPostRASchedStrategy::tryCandidate(SchedCandidate &Cand,
                                           SchedCandidate &TryCand) {
-  PostGenericScheduler::tryCandidate(Cand, TryCand);
+  // Initialize the candidate if needed.
+  if (!Cand.isValid()) {
+    TryCand.Reason = NodeOrder;
+    return;
+  }
 
-  if (!Cand.isValid())
+  // Prioritize instructions that read unbuffered resources by stall cycles.
+  if (tryLess(Top.getLatencyStallCycles(TryCand.SU),
+              Top.getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
     return;
 
-  // Add powerpc post ra specific heuristic only when TryCand isn't selected or
-  // selected as node order.
-  if (TryCand.Reason != NodeOrder && TryCand.Reason != NoCand)
+  // Keep clustered nodes together.
+  if (tryGreater(TryCand.SU == DAG->getNextClusterSucc(),
+                 Cand.SU == DAG->getNextClusterSucc(), TryCand, Cand, Cluster))
+    return;
+
+  // Avoid critical resource consumption and balance the schedule.
+  if (tryLess(TryCand.ResDelta.CritResources, Cand.ResDelta.CritResources,
+              TryCand, Cand, ResourceReduce))
+    return;
+  if (tryGreater(TryCand.ResDelta.DemandedResources,
+                 Cand.ResDelta.DemandedResources, TryCand, Cand,
+                 ResourceDemand))
     return;
 
+  // Avoid serializing long latency dependence chains.
+  if (Cand.Policy.ReduceLatency && tryLatency(TryCand, Cand, Top)) {
+    return;
+  }
+
   // There are some benefits to schedule the ADDI as early as possible post ra
   // to avoid stalled by vector instructions which take up all the hw units.
   // And ADDI is usually used to post inc the loop indvar, which matters the
   // performance.
   if (biasAddiCandidate(Cand, TryCand))
     return;
+
+  // Fall through to original instruction order.
+  if (TryCand.SU->NodeNum < Cand.SU->NodeNum)
+    TryCand.Reason = NodeOrder;
 }
 
 void PPCPostRASchedStrategy::enterMBB(MachineBasicBlock *MBB) {
   // Custom PPC PostRA specific behavior here.
   PostGenericScheduler::enterMBB(MBB);
 }
 
 void PPCPostRASchedStrategy::leaveMBB() {

llvm/test/CodeGen/PowerPC/botheightreduce.mir

body: |
   ; CHECK:   [[LI8_1:%[0-9]+]]:g8rc = LI8 0
   ; CHECK:   [[LI8_2:%[0-9]+]]:g8rc = LI8 2
   ; CHECK:   [[LI8_3:%[0-9]+]]:g8rc = LI8 3
   ; CHECK:   [[LI8_4:%[0-9]+]]:g8rc = LI8 5
   ; CHECK:   [[LI8_5:%[0-9]+]]:g8rc = LI8 6
   ; CHECK:   [[LI8_6:%[0-9]+]]:g8rc = LI8 7
   ; CHECK: bb.1:
   ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK:   [[ADDI8_1:%[0-9]+]]:g8rc = ADDI8 [[ADDI8_]], 1
   ; CHECK:   [[LD:%[0-9]+]]:g8rc = LD 0, [[ADDI8_]] :: (load 8)
   ; CHECK:   [[LDX:%[0-9]+]]:g8rc = LDX [[ADDI8_]], [[LI8_]] :: (load 8)
   ; CHECK:   [[LDX1:%[0-9]+]]:g8rc = LDX [[ADDI8_]], [[LI8_3]] :: (load 8)
   ; CHECK:   [[LD1:%[0-9]+]]:g8rc = LD 4, [[ADDI8_]] :: (load 8)
   ; CHECK:   [[LDX2:%[0-9]+]]:g8rc = LDX [[ADDI8_]], [[LI8_4]] :: (load 8)
   ; CHECK:   [[LDX3:%[0-9]+]]:g8rc = LDX [[ADDI8_]], [[LI8_5]] :: (load 8)
   ; CHECK:   [[LDX4:%[0-9]+]]:g8rc = LDX [[ADDI8_]], [[LI8_6]] :: (load 8)
-  ; CHECK:   [[LDX5:%[0-9]+]]:g8rc = LDX [[ADDI8_]], [[LI8_2]] :: (load 8)
-  ; CHECK:   [[MULLD:%[0-9]+]]:g8rc = MULLD [[LDX]], [[LD]]
   ; CHECK:   [[LD2:%[0-9]+]]:g8rc = LD 8, [[ADDI8_]] :: (load 8)
+  ; CHECK:   [[MULLD:%[0-9]+]]:g8rc = MULLD [[LDX]], [[LD]]
+  ; CHECK:   [[LDX5:%[0-9]+]]:g8rc = LDX [[ADDI8_]], [[LI8_2]] :: (load 8)
+  ; CHECK:   [[ADDI8_1:%[0-9]+]]:g8rc = ADDI8 [[ADDI8_]], 1
   ; CHECK:   [[MULLD1:%[0-9]+]]:g8rc = MULLD [[MULLD]], [[LDX5]]
   ; CHECK:   [[MULLD2:%[0-9]+]]:g8rc = MULLD [[MULLD1]], [[LDX1]]
   ; CHECK:   [[MULLD3:%[0-9]+]]:g8rc = MULLD [[MULLD2]], [[LD1]]
   ; CHECK:   [[MULLD4:%[0-9]+]]:g8rc = MULLD [[MULLD3]], [[LDX2]]
   ; CHECK:   [[MULLD5:%[0-9]+]]:g8rc = MULLD [[MULLD4]], [[LDX3]]
   ; CHECK:   [[MULLD6:%[0-9]+]]:g8rc = MULLD [[MULLD5]], [[LDX4]]
   ; CHECK:   [[MADDLD8_:%[0-9]+]]:g8rc = MADDLD8 [[MULLD6]], [[LD2]], [[MADDLD8_]]
   ; CHECK:   [[COPY2:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY [[ADDI8_1]]

llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll

 ; CHECK-NEXT:    li r5, 1
 ; CHECK-NEXT:    addi r6, r3, 4002
 ; CHECK-NEXT:    li r7, -1
 ; CHECK-NEXT:    iselgt r3, r4, r5
 ; CHECK-NEXT:    mtctr r3
 ; CHECK-NEXT:    li r3, 0
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_2:
-; CHECK-NEXT:    ldx r9, r6, r7
-; CHECK-NEXT:    ld r10, 0(r6)
-; CHECK-NEXT:    ldx r11, r6, r5
-; CHECK-NEXT:    addi r8, r6, 1
-; CHECK-NEXT:    ld r6, 4(r6)
-; CHECK-NEXT:    mulld r9, r10, r9
-; CHECK-NEXT:    mulld r9, r9, r11
-; CHECK-NEXT:    maddld r3, r9, r6, r3
-; CHECK-NEXT:    mr r6, r8
+; CHECK-NEXT:    ldx r8, r6, r7
+; CHECK-NEXT:    ld r9, 0(r6)
+; CHECK-NEXT:    ldx r10, r6, r5
+; CHECK-NEXT:    ld r11, 4(r6)
+; CHECK-NEXT:    addi r6, r6, 1
+; CHECK-NEXT:    mulld r8, r9, r8
+; CHECK-NEXT:    mulld r8, r8, r10
+; CHECK-NEXT:    maddld r3, r8, r11, r3
 ; CHECK-NEXT:    bdnz .LBB1_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    add r3, r3, r4
 ; CHECK-NEXT:    blr
 ; CHECK-NEXT:  .LBB1_4:
 ; CHECK-NEXT:    addi r3, r4, 0
 ; CHECK-NEXT:    blr
   %3 = sext i32 %1 to i64

 ; CHECK-NEXT:    li r7, 3
 ; CHECK-NEXT:    li r8, 5
 ; CHECK-NEXT:    li r10, 9
 ; CHECK-NEXT:    iselgt r3, r4, r5
 ; CHECK-NEXT:    mtctr r3
 ; CHECK-NEXT:    li r3, 0
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    ldx r12, r9, r6
-; CHECK-NEXT:    ld r0, 0(r9)
-; CHECK-NEXT:    ldx r30, r9, r5
-; CHECK-NEXT:    ldx r29, r9, r7
-; CHECK-NEXT:    addi r11, r9, 1
-; CHECK-NEXT:    mulld r12, r0, r12
-; CHECK-NEXT:    ld r28, 4(r9)
-; CHECK-NEXT:    ldx r27, r9, r8
-; CHECK-NEXT:    ld r26, 12(r9)
-; CHECK-NEXT:    ld r25, 8(r9)
-; CHECK-NEXT:    ldx r9, r9, r10
-; CHECK-NEXT:    mulld r12, r12, r30
-; CHECK-NEXT:    mulld r12, r12, r29
-; CHECK-NEXT:    mulld r12, r12, r28
-; CHECK-NEXT:    mulld r12, r12, r27
-; CHECK-NEXT:    mulld r12, r12, r26
-; CHECK-NEXT:    mulld r12, r12, r25
-; CHECK-NEXT:    maddld r3, r12, r9, r3
-; CHECK-NEXT:    mr r9, r11
+; CHECK-NEXT:    ldx r11, r9, r6
+; CHECK-NEXT:    ld r12, 0(r9)
+; CHECK-NEXT:    ldx r0, r9, r5
+; CHECK-NEXT:    ldx r30, r9, r7
+; CHECK-NEXT:    mulld r11, r12, r11
+; CHECK-NEXT:    ld r29, 4(r9)
+; CHECK-NEXT:    ldx r28, r9, r8
+; CHECK-NEXT:    ld r27, 12(r9)
+; CHECK-NEXT:    ld r26, 8(r9)
+; CHECK-NEXT:    ldx r25, r9, r10
+; CHECK-NEXT:    addi r9, r9, 1
+; CHECK-NEXT:    mulld r11, r11, r0
+; CHECK-NEXT:    mulld r11, r11, r30
+; CHECK-NEXT:    mulld r11, r11, r29
+; CHECK-NEXT:    mulld r11, r11, r28
+; CHECK-NEXT:    mulld r11, r11, r27
+; CHECK-NEXT:    mulld r11, r11, r26
+; CHECK-NEXT:    maddld r3, r11, r25, r3
 ; CHECK-NEXT:    bdnz .LBB2_2
 ; CHECK-NEXT:    b .LBB2_4
 ; CHECK-NEXT:  .LBB2_3:
 ; CHECK-NEXT:    li r3, 0
 ; CHECK-NEXT:  .LBB2_4:
 ; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload

 ; CHECK-NEXT:    cmplwi r4, 0
 ; CHECK-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    beq cr0, .LBB6_8
 ; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
 ; CHECK-NEXT:    cmpldi r4, 1
 ; CHECK-NEXT:    li r7, 1
 ; CHECK-NEXT:    addi r6, r3, 4009
-; CHECK-NEXT:    addis r5, r2, .LC0@toc@ha
 ; CHECK-NEXT:    ld r5, .LC0@toc@l(r5)
 ; CHECK-NEXT:    iselgt r8, r4, r7
 ; CHECK-NEXT:    lis r4, -21846
 ; CHECK-NEXT:    li r3, 0
 ; CHECK-NEXT:    li r9, -7
 ; CHECK-NEXT:    li r10, -6
 ; CHECK-NEXT:    li r11, 1
 ; CHECK-NEXT:    li r12, 1
 ; CHECK-NEXT:    li r30, 1
 ; CHECK-NEXT:    ld r5, 0(r5)
 ; CHECK-NEXT:    mtctr r8
-; CHECK-NEXT:    li r8, -9
-; CHECK-NEXT:    addi r5, r5, -1
 ; CHECK-NEXT:    ori r4, r4, 43691
+; CHECK-NEXT:    li r8, -9
 ; CHECK-NEXT:    li r29, 1
 ; CHECK-NEXT:    li r28, 1
+; CHECK-NEXT:    addi r5, r5, -1
 ; CHECK-NEXT:    b .LBB6_4
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_2:
 ; CHECK-NEXT:    ldx r0, r6, r8
 ; CHECK-NEXT:    add r28, r0, r28
 ; CHECK-NEXT:    ld r0, -8(r6)
 ; CHECK-NEXT:    add r29, r0, r29
 ; CHECK-NEXT:  .LBB6_3:
-; CHECK-NEXT:    addi r6, r6, 1
 ; CHECK-NEXT:    mulld r0, r29, r28
+; CHECK-NEXT:    addi r6, r6, 1
 ; CHECK-NEXT:    mulld r0, r0, r30
 ; CHECK-NEXT:    mulld r0, r0, r12
 ; CHECK-NEXT:    mulld r0, r0, r11
 ; CHECK-NEXT:    maddld r3, r0, r7, r3
 ; CHECK-NEXT:    bdz .LBB6_9
 ; CHECK-NEXT:  .LBB6_4:
 ; CHECK-NEXT:    lbzu r0, 1(r5)
 ; CHECK-NEXT:    mulhwu r27, r0, r4

 ;}
 
 define float @test_ds_float(i8* %0, i32 signext %1) {
 ; CHECK-LABEL: test_ds_float:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    cmpwi r4, 1
 ; CHECK-NEXT:    blt cr0, .LBB7_4
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    addi r3, r3, 4002
 ; CHECK-NEXT:    clrldi r4, r4, 32
+; CHECK-NEXT:    addi r3, r3, 4002
 ; CHECK-NEXT:    xxlxor f1, f1, f1
 ; CHECK-NEXT:    mtctr r4
 ; CHECK-NEXT:    li r4, -1
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_2:
 ; CHECK-NEXT:    lfsx f0, r3, r4
 ; CHECK-NEXT:    lfs f2, 0(r3)
 ; CHECK-NEXT:    xsmulsp f0, f0, f2

 ;}
 
 define float @test_ds_combine_float_int(i8* %0, i32 signext %1) {
 ; CHECK-LABEL: test_ds_combine_float_int:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    cmpwi r4, 1
 ; CHECK-NEXT:    blt cr0, .LBB8_4
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    addi r3, r3, 4002
 ; CHECK-NEXT:    clrldi r4, r4, 32
+; CHECK-NEXT:    addi r3, r3, 4002
 ; CHECK-NEXT:    xxlxor f1, f1, f1
 ; CHECK-NEXT:    mtctr r4
 ; CHECK-NEXT:    li r4, -1
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_2:
 ; CHECK-NEXT:    lfd f4, 0(r3)
 ; CHECK-NEXT:    lfsx f0, r3, r4
 ; CHECK-NEXT:    xscvuxdsp f4, f4

llvm/test/CodeGen/PowerPC/rematerializable-instruction-machine-licm.ll

 ; CHECK-NEXT:    ld 2, 408(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stdx 3, 12, 2
 ; CHECK-NEXT:    ld 2, 400(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stdx 3, 12, 2
 ; CHECK-NEXT:    bdnz .LBB0_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    ld 12, 384(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lwz 4, 396(1) # 4-byte Folded Reload
-; CHECK-NEXT:    addi 4, 4, 1
 ; CHECK-NEXT:    std 3, 0(12)
 ; CHECK-NEXT:    ld 12, 376(1) # 8-byte Folded Reload
+; CHECK-NEXT:    addi 4, 4, 1
 ; CHECK-NEXT:    std 3, 0(12)
 ; CHECK-NEXT:    ld 12, 368(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    std 3, 0(12)
 ; CHECK-NEXT:    ld 12, 360(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    std 3, 0(12)
 ; CHECK-NEXT:    ld 12, 352(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    std 3, 0(12)
 ; CHECK-NEXT:    ld 12, 344(1) # 8-byte Folded Reload

llvm/test/CodeGen/PowerPC/sched-addi.ll

 @scalars = common local_unnamed_addr global %_type_of_scalars zeroinitializer, align 16
 
 define void @test([0 x %_elem_type_of_x]* noalias %.x, [0 x %_elem_type_of_a]* %.a, i64* noalias %.n) {
 ; CHECK-P9-LABEL: test:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    ld 5, 0(5)
 ; CHECK-P9-NEXT:    addis 6, 2, scalars@toc@ha
 ; CHECK-P9-NEXT:    addi 6, 6, scalars@toc@l
-; CHECK-P9-NEXT:    addi 6, 6, 16
 ; CHECK-P9-NEXT:    rldicr 5, 5, 0, 58
+; CHECK-P9-NEXT:    addi 6, 6, 16
 ; CHECK-P9-NEXT:    addi 5, 5, -32
 ; CHECK-P9-NEXT:    lxvdsx 0, 0, 6
 ; CHECK-P9-NEXT:    rldicl 5, 5, 59, 5
 ; CHECK-P9-NEXT:    addi 5, 5, 1
 ; CHECK-P9-NEXT:    mtctr 5
 ; CHECK-P9-NEXT:    .p2align 4
 ; CHECK-P9-NEXT:  .LBB0_1: # %vector.body
 ; CHECK-P9-NEXT:    #
 ; CHECK-P9-NEXT:    lxv 1, 16(4)
 ; CHECK-P9-NEXT:    lxv 2, 0(4)
 ; CHECK-P9-NEXT:    lxv 3, 48(4)
 ; CHECK-P9-NEXT:    lxv 4, 32(4)
 ; CHECK-P9-NEXT:    xvmuldp 2, 2, 0
 ; CHECK-P9-NEXT:    lxv 5, 240(4)
 ; CHECK-P9-NEXT:    lxv 6, 224(4)
 ; CHECK-P9-NEXT:    xvmuldp 1, 1, 0
 ; CHECK-P9-NEXT:    xvmuldp 4, 4, 0
 ; CHECK-P9-NEXT:    xvmuldp 3, 3, 0
+; CHECK-P9-NEXT:    xvmuldp 6, 6, 0
 ; CHECK-P9-NEXT:    xvmuldp 5, 5, 0
 ; CHECK-P9-NEXT:    addi 4, 4, 256
-; CHECK-P9-NEXT:    xvmuldp 6, 6, 0
 ; CHECK-P9-NEXT:    stxv 1, 16(3)
 ; CHECK-P9-NEXT:    stxv 2, 0(3)
 ; CHECK-P9-NEXT:    stxv 3, 48(3)
 ; CHECK-P9-NEXT:    stxv 4, 32(3)
 ; CHECK-P9-NEXT:    stxv 5, 240(3)
 ; CHECK-P9-NEXT:    stxv 6, 224(3)
 ; CHECK-P9-NEXT:    addi 3, 3, 256
 ; CHECK-P9-NEXT:    bdnz .LBB0_1

llvm/test/CodeGen/PowerPC/sms-cpy-1.ll

 ; CHECK-NEXT:    li 3, 1
 ; CHECK-NEXT:    iselgt 3, 4, 3
 ; CHECK-NEXT:    li 4, 2
 ; CHECK-NEXT:    addi 3, 3, -1
 ; CHECK-NEXT:    cmpldi 3, 2
 ; CHECK-NEXT:    isellt 3, 3, 4
 ; CHECK-NEXT:    li 4, 0
 ; CHECK-NEXT:    addi 3, 3, 1
-; CHECK-NEXT:    li 7, -1
 ; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    li 7, -1
 ; CHECK-NEXT:    mtctr 3
-; CHECK-NEXT:    li 3, 1
 ; CHECK-NEXT:    lbz 5, 0(5)
+; CHECK-NEXT:    li 3, 1
 ; CHECK-NEXT:    bdz .LBB0_6
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    addi 3, 3, 1
-; CHECK-NEXT:    addi 8, 7, -1
 ; CHECK-NEXT:    xori 6, 5, 84
 ; CHECK-NEXT:    clrldi 5, 7, 32
+; CHECK-NEXT:    addi 3, 3, 1
+; CHECK-NEXT:    addi 8, 7, -1
 ; CHECK-NEXT:    lbz 5, 0(5)
 ; CHECK-NEXT:    bdz .LBB0_5
 ; CHECK-NEXT:  # %bb.2:
-; CHECK-NEXT:    addi 3, 3, 1
 ; CHECK-NEXT:    cntlzw 6, 6
+; CHECK-NEXT:    addi 3, 3, 1
 ; CHECK-NEXT:    srwi 7, 6, 5
 ; CHECK-NEXT:    xori 6, 5, 84
 ; CHECK-NEXT:    clrldi 5, 8, 32
 ; CHECK-NEXT:    addi 8, 8, -1
 ; CHECK-NEXT:    lbz 5, 0(5)
 ; CHECK-NEXT:    bdz .LBB0_4
 ; CHECK-NEXT:  .LBB0_3:
-; CHECK-NEXT:    addi 3, 3, 1
 ; CHECK-NEXT:    clrldi 10, 8, 32
-; CHECK-NEXT:    addi 8, 8, -1
 ; CHECK-NEXT:    cntlzw 9, 6
 ; CHECK-NEXT:    xori 6, 5, 84
+; CHECK-NEXT:    addi 8, 8, -1
 ; CHECK-NEXT:    lbz 5, 0(10)
+; CHECK-NEXT:    addi 3, 3, 1
 ; CHECK-NEXT:    add 4, 4, 7
 ; CHECK-NEXT:    srwi 7, 9, 5
 ; CHECK-NEXT:    bdnz .LBB0_3
 ; CHECK-NEXT:  .LBB0_4:
 ; CHECK-NEXT:    add 4, 4, 7
 ; CHECK-NEXT:  .LBB0_5:
 ; CHECK-NEXT:    cntlzw 6, 6
 ; CHECK-NEXT:    srwi 6, 6, 5

llvm/test/CodeGen/PowerPC/sms-phi-1.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs\
 ; RUN:       -mcpu=pwr9 --ppc-enable-pipeliner 2>&1 | FileCheck %s
 
 define void @main() nounwind #0 {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    std 30, -16(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 0, 16(1)
 ; CHECK-NEXT:    stdu 1, -48(1)
 ; CHECK-NEXT:    bl strtol
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    mr 30, 3
 ; CHECK-NEXT:    bl calloc
 ; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi 4, 30, 32
 ; CHECK-NEXT:    li 5, 0
 ; CHECK-NEXT:    addi 3, 3, -4
-; CHECK-NEXT:    li 6, 1
-; CHECK-NEXT:    clrldi 4, 30, 32
 ; CHECK-NEXT:    mtctr 4
 ; CHECK-NEXT:    mullw 4, 5, 5
+; CHECK-NEXT:    li 6, 1
 ; CHECK-NEXT:    bdz .LBB0_3
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    addi 5, 6, 1
 ; CHECK-NEXT:    stwu 4, 4(3)
 ; CHECK-NEXT:    mullw 4, 6, 6
+; CHECK-NEXT:    addi 5, 6, 1
 ; CHECK-NEXT:    bdz .LBB0_3
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    stwu 4, 4(3)
 ; CHECK-NEXT:    mullw 4, 5, 5
 ; CHECK-NEXT:    addi 5, 5, 1
 ; CHECK-NEXT:    bdnz .LBB0_2
 ; CHECK-NEXT:  .LBB0_3:

llvm/test/CodeGen/PowerPC/sms-simple.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-linux-gnu \
 ; RUN:       -verify-machineinstrs -ppc-asm-full-reg-names -mcpu=pwr9 --ppc-enable-pipeliner \
 ; RUN:       | FileCheck %s
 
 @x = dso_local local_unnamed_addr global <{ i32, i32, i32, i32, [1020 x i32] }> <{ i32 1, i32 2, i32 3, i32 4, [1020 x i32] zeroinitializer }>, align 4
 @y = dso_local global [1024 x i32] zeroinitializer, align 4
 
 define dso_local i32* @foo() local_unnamed_addr {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis r5, r2, x@toc@ha
-; CHECK-NEXT:    addis r6, r2, y@toc@ha
+; CHECK-NEXT:    addis r5, r2, y@toc@ha
 ; CHECK-NEXT:    li r7, 340
-; CHECK-NEXT:    addi r5, r5, x@toc@l
-; CHECK-NEXT:    addi r5, r5, -8
-; CHECK-NEXT:    addi r3, r6, y@toc@l
-; CHECK-NEXT:    lwz r6, y@toc@l(r6)
+; CHECK-NEXT:    addi r3, r5, y@toc@l
+; CHECK-NEXT:    lwz r6, y@toc@l(r5)
+; CHECK-NEXT:    addis r5, r2, x@toc@ha
 ; CHECK-NEXT:    mtctr r7
+; CHECK-NEXT:    addi r5, r5, x@toc@l
 ; CHECK-NEXT:    addi r4, r3, -8
+; CHECK-NEXT:    addi r5, r5, -8
 ; CHECK-NEXT:    lwzu r7, 12(r5)
 ; CHECK-NEXT:    maddld r6, r7, r7, r6
 ; CHECK-NEXT:    lwz r7, 4(r5)
 ; CHECK-NEXT:    stwu r6, 12(r4)
 ; CHECK-NEXT:    maddld r6, r7, r7, r6
 ; CHECK-NEXT:    lwz r7, 8(r5)
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.body

llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll

 ; CHECK-LE-NEXT:    ld r31, -8(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-P9-LE-LABEL: foo:
 ; CHECK-P9-LE:       # %bb.0:
 ; CHECK-P9-LE-NEXT:    std r31, -8(r1)
 ; CHECK-P9-LE-NEXT:    stdu r1, -48(r1)
 ; CHECK-P9-LE-NEXT:    rldic r3, r3, 2, 30
-; CHECK-P9-LE-NEXT:    addi r3, r3, 15
 ; CHECK-P9-LE-NEXT:    li r6, -32768
 ; CHECK-P9-LE-NEXT:    mr r31, r1
+; CHECK-P9-LE-NEXT:    addi r3, r3, 15
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 4, 29
 ; CHECK-P9-LE-NEXT:    neg r5, r3
 ; CHECK-P9-LE-NEXT:    addi r3, r31, 48
 ; CHECK-P9-LE-NEXT:    divd r7, r5, r6
 ; CHECK-P9-LE-NEXT:    add r4, r1, r5
 ; CHECK-P9-LE-NEXT:    mulld r6, r7, r6
 ; CHECK-P9-LE-NEXT:    sub r5, r5, r6

 ; CHECK-LE-NEXT:    ld r31, -8(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-P9-LE-LABEL: bar:
 ; CHECK-P9-LE:       # %bb.0:
 ; CHECK-P9-LE-NEXT:    std r31, -8(r1)
 ; CHECK-P9-LE-NEXT:    stdu r1, -48(r1)
 ; CHECK-P9-LE-NEXT:    rldic r4, r3, 2, 30
-; CHECK-P9-LE-NEXT:    addi r4, r4, 15
 ; CHECK-P9-LE-NEXT:    li r7, -4096
 ; CHECK-P9-LE-NEXT:    mr r31, r1
+; CHECK-P9-LE-NEXT:    addi r4, r4, 15
 ; CHECK-P9-LE-NEXT:    rldicl r4, r4, 60, 4
 ; CHECK-P9-LE-NEXT:    rldicl r4, r4, 4, 29
 ; CHECK-P9-LE-NEXT:    neg r6, r4
 ; CHECK-P9-LE-NEXT:    addi r4, r31, 48
 ; CHECK-P9-LE-NEXT:    divd r8, r6, r7
 ; CHECK-P9-LE-NEXT:    add r5, r1, r6
 ; CHECK-P9-LE-NEXT:    mulld r7, r8, r7
 ; CHECK-P9-LE-NEXT:    sub r6, r6, r7

 ; CHECK-LE-NEXT:    ld r31, -8(r1)
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-P9-LE-LABEL: f:
 ; CHECK-P9-LE:       # %bb.0:
 ; CHECK-P9-LE-NEXT:    std r31, -8(r1)
 ; CHECK-P9-LE-NEXT:    stdu r1, -48(r1)
 ; CHECK-P9-LE-NEXT:    rldic r3, r3, 2, 30
-; CHECK-P9-LE-NEXT:    addi r3, r3, 15
 ; CHECK-P9-LE-NEXT:    lis r5, -1
-; CHECK-P9-LE-NEXT:    ori r5, r5, 0
 ; CHECK-P9-LE-NEXT:    mr r31, r1
+; CHECK-P9-LE-NEXT:    addi r3, r3, 15
+; CHECK-P9-LE-NEXT:    ori r5, r5, 0
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 4, 29
 ; CHECK-P9-LE-NEXT:    neg r6, r3
 ; CHECK-P9-LE-NEXT:    addi r3, r31, 48
 ; CHECK-P9-LE-NEXT:    divd r7, r6, r5
 ; CHECK-P9-LE-NEXT:    add r4, r1, r6
 ; CHECK-P9-LE-NEXT:    mulld r7, r7, r5
 ; CHECK-P9-LE-NEXT:    sub r6, r6, r7