[Power9] Add a specific heuristic to schedule the addi before the load

[steven.zhang]

Summary

When we are scheduling the load and addi, if all other heuristic didn't take effect, we will try to schedule the addi before the load, to hide the latency, and avoid the true dependency added by RA. And this only take effects for Power9.

[jsji]

I think we should update the title or summary to indicate that this is P9 only.

llvm/lib/Target/PowerPC/PPCMachineScheduler.cpp

12 ↗

(On Diff #199549)

Can this be a lamda or std::function?

32 ↗

(On Diff #199549)

These logic are similar to above, just swapped the Candidates.
Can we do it something like:

Cand1st= Zone.isTop() ? TryCand.SU->getInstr(): Cand.SU->getInstr();
Cand2nd= Zone.isTop() ? Cand.SU->getInstr(): *TryCand.SU->getInstr();

if(isADDIInstr(*Cand1st)) && Cand2nd->mayLoad())){
...
}
..

61 ↗

(On Diff #199549)

Can we add an option to disable this heuristic?

llvm/lib/Target/PowerPC/PPCMachineScheduler.h

29 ↗

(On Diff #199549)

I am seeing this as a bias heuristic, so maybe biasAddi?

llvm/test/CodeGen/PowerPC/schedule-addi-load.mir

1 ↗

(On Diff #199549)

Can we commit this testcase as a NFC before this patch? So that the only difference in this patch will be the scheduling change. Thanks.

2 ↗

(On Diff #199549)

We can also add a test for the newly added option to disable this heuristic

3 ↗

(On Diff #199549)

Add a line as a negative test for pwr8 , as it doesn't use FeaturePPCPreRASched.

[steven.zhang]

Address jinong's comments.

[jsji]

LGTM.

llvm/trunk/lib/Target/PowerPC/PPCMachineScheduler.h

 
 namespace llvm {
 
 /// A MachineSchedStrategy implementation for PowerPC pre RA scheduling.
 class PPCPreRASchedStrategy : public GenericScheduler {
 public:
   PPCPreRASchedStrategy(const MachineSchedContext *C) :
     GenericScheduler(C) {}
+protected:
+  void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand,
+                    SchedBoundary *Zone) const override;
+private:
+  bool biasAddiLoadCandidate(SchedCandidate &Cand,
+                             SchedCandidate &TryCand,
+                             SchedBoundary &Zone) const;
 };
 
 /// A MachineSchedStrategy implementation for PowerPC post RA scheduling.
 class PPCPostRASchedStrategy : public PostGenericScheduler {
 public:
   PPCPostRASchedStrategy(const MachineSchedContext *C) :
     PostGenericScheduler(C) {}
 

llvm/trunk/lib/Target/PowerPC/PPCMachineScheduler.cpp

 //===- PPCMachineScheduler.cpp - MI Scheduler for PowerPC -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+#include "PPC.h"
 #include "PPCMachineScheduler.h"
 using namespace llvm;
 
+static cl::opt<bool> 
+DisableAddiLoadHeuristic("disable-ppc-sched-addi-load",
+                         cl::desc("Disable scheduling addi instruction before" 
+                                  "load for ppc"), cl::Hidden);
+
+bool PPCPreRASchedStrategy::biasAddiLoadCandidate(SchedCandidate &Cand,
+                                                  SchedCandidate &TryCand,
+                                                  SchedBoundary &Zone) const {
+  if (DisableAddiLoadHeuristic)
+    return false;
+
+  auto isADDIInstr = [&] (const MachineInstr &Inst) {
+    return Inst.getOpcode() == PPC::ADDI || Inst.getOpcode() == PPC::ADDI8;
+  };
+
+  SchedCandidate &FirstCand = Zone.isTop() ? TryCand : Cand;
+  SchedCandidate &SecondCand = Zone.isTop() ? Cand : TryCand;
+  if (isADDIInstr(*FirstCand.SU->getInstr()) &&
+      SecondCand.SU->getInstr()->mayLoad()) {
+    TryCand.Reason = Stall;
+    return true;
+  }
+  if (FirstCand.SU->getInstr()->mayLoad() &&
+      isADDIInstr(*SecondCand.SU->getInstr())) {
+    TryCand.Reason = NoCand;
+    return true;
+  }
+
+  return false;
+}
+
+void PPCPreRASchedStrategy::tryCandidate(SchedCandidate &Cand,
+                                         SchedCandidate &TryCand,
+                                         SchedBoundary *Zone) const {
+  GenericScheduler::tryCandidate(Cand, TryCand, Zone);
+
+  if (!Cand.isValid() || !Zone)
+    return;
+
+  // Add powerpc specific heuristic only when TryCand isn't selected or
+  // selected as node order.
+  if (TryCand.Reason != NodeOrder && TryCand.Reason != NoCand)
+    return;
+
+  // There are some benefits to schedule the ADDI before the load to hide the
+  // latency, as RA may create a true dependency between the load and addi.
+  if (biasAddiLoadCandidate(Cand, TryCand, *Zone))
+    return;
+}
+
 void PPCPostRASchedStrategy::enterMBB(MachineBasicBlock *MBB) {
   // Custom PPC PostRA specific behavior here.
   PostGenericScheduler::enterMBB(MBB);
 }
 
 void PPCPostRASchedStrategy::leaveMBB() {
   // Custom PPC PostRA specific behavior here.
   PostGenericScheduler::leaveMBB();

llvm/trunk/test/CodeGen/PowerPC/schedule-addi-load.mir

 # RUN: llc -mcpu=pwr9 -mtriple powerpc64le-unknown-linux-gnu -start-before machine-scheduler -stop-after machine-scheduler -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mcpu=pwr9 -mtriple powerpc64le-unknown-linux-gnu -disable-ppc-sched-addi-load -start-before machine-scheduler -stop-after machine-scheduler \
+# RUN:   -verify-machineinstrs %s -o - | FileCheck --check-prefix=CHECK-DISABLE %s
+# RUN: llc -mcpu=pwr8 -mtriple powerpc64le-unknown-linux-gnu -start-before machine-scheduler -stop-after machine-scheduler -verify-machineinstrs %s -o - | FileCheck --check-prefix=CHECK-P8 %s
 
 # Test that if the scheduler moves the addi before the load.
 --- |
   target datalayout = "e-m:e-i64:64-n32:64"
   
   define i64 @foo(i8* %p, i8* %q) {
   entry:
     br label %while.cond6.i
bb.1.while.cond6.i:
     %6:gprc = LBZX %2, %5 :: (load 1 from %ir.arrayidx)
     %7:gprc = LBZX %3, %5 :: (load 1 from %ir.arrayidx4)
     %9:g8rc = ADDI8 %5, 1
     %8:crrc = CMPLW %6, %7
     BCC 76, %8, %bb.1
     B %bb.2
     ; CHECK-LABEL: foo
     ; CHECK: %5:g8rc_and_g8rc_nox0 = RLDICL %0, 0, 32
+    ; CHECK-NEXT: %9:g8rc = ADDI8 %5, 1
     ; CHECK-NEXT: %6:gprc = LBZX %2, %5 :: (load 1 from %ir.arrayidx) 
     ; CHECK-NEXT: %7:gprc = LBZX %3, %5 :: (load 1 from %ir.arrayidx4) 
-    ; CHECK-NEXT: %9:g8rc = ADDI8 %5, 1
     ; CHECK-NEXT: %8:crrc = CMPLW %6, %7 
     ; CHECK-NEXT: BCC 76, %8
+    ; CHECK-DISABLE-LABEL: foo
+    ; CHECK-DISABLE: %5:g8rc_and_g8rc_nox0 = RLDICL %0, 0, 32
+    ; CHECK-DISABLE-NEXT: %6:gprc = LBZX %2, %5 :: (load 1 from %ir.arrayidx) 
+    ; CHECK-DISABLE-NEXT: %7:gprc = LBZX %3, %5 :: (load 1 from %ir.arrayidx4) 
+    ; CHECK-DISABLE-NEXT: %9:g8rc = ADDI8 %5, 1
+    ; CHECK-DISABLE-NEXT: %8:crrc = CMPLW %6, %7 
+    ; CHECK-DISABLE-NEXT: BCC 76, %8
+    ; CHECK-P8-LABEL: foo
+    ; CHECK-P8: %5:g8rc_and_g8rc_nox0 = RLDICL %0, 0, 32
+    ; CHECK-P8-NEXT: %6:gprc = LBZX %2, %5 :: (load 1 from %ir.arrayidx)
+    ; CHECK-P8-NEXT: %7:gprc = LBZX %3, %5 :: (load 1 from %ir.arrayidx4)
+    ; CHECK-P8-NEXT: %8:crrc = CMPLW %6, %7
+    ; CHECK-P8-NEXT: %9:g8rc = ADDI8 %5, 1
+    ; CHECK-P8-NEXT: BCC 76, %8
   
   bb.2.while.end:
     $x3 = COPY %0
     BLR8 implicit $lr8, implicit $rm, implicit $x3
 
 ...