diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -264,10 +264,6 @@ LiveIntervals *LIS; std::unique_ptr SchedImpl; - /// Topo - A topological ordering for SUnits which permits fast IsReachable - /// and similar queries. - ScheduleDAGTopologicalSort Topo; - /// Ordered list of DAG postprocessing steps. std::vector> Mutations; @@ -291,7 +287,7 @@ ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr S, bool RemoveKillFlags) : ScheduleDAGInstrs(*C->MF, C->MLI, RemoveKillFlags), AA(C->AA), - LIS(C->LIS), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU) {} + LIS(C->LIS), SchedImpl(std::move(S)) {} // Provide a vtable anchor ~ScheduleDAGMI() override; @@ -319,17 +315,6 @@ Mutations.push_back(std::move(Mutation)); } - /// True if an edge can be added from PredSU to SuccSU without creating - /// a cycle. - bool canAddEdge(SUnit *SuccSU, SUnit *PredSU); - - /// Add a DAG edge to the given SU with the given predecessor - /// dependence data. - /// - /// \returns true if the edge may be added without creating a cycle OR if an - /// equivalent edge already existed (false indicates failure). - bool addEdge(SUnit *SuccSU, const SDep &PredDep); - MachineBasicBlock::iterator top() const { return CurrentTop; } MachineBasicBlock::iterator bottom() const { return CurrentBottom; } diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -234,6 +234,11 @@ /// For an unanalyzable memory access, this Value is used in maps. UndefValue *UnknownValue; + + /// Topo - A topological ordering for SUnits which permits fast IsReachable + /// and similar queries. + ScheduleDAGTopologicalSort Topo; + using DbgValueVector = std::vector>; /// Remember instruction that precedes DBG_VALUE. @@ -338,6 +343,17 @@ /// Fixes register kill flags that scheduling has made invalid. void fixupKills(MachineBasicBlock &MBB); + /// True if an edge can be added from PredSU to SuccSU without creating + /// a cycle. + bool canAddEdge(SUnit *SuccSU, SUnit *PredSU); + + /// Add a DAG edge to the given SU with the given predecessor + /// dependence data. + /// + /// \returns true if the edge may be added without creating a cycle OR if an + /// equivalent edge already existed (false indicates failure). + bool addEdge(SUnit *SuccSU, const SDep &PredDep); + protected: void initSUnits(); void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx); diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -604,23 +604,6 @@ // Provide a vtable anchor. ScheduleDAGMI::~ScheduleDAGMI() = default; -bool ScheduleDAGMI::canAddEdge(SUnit *SuccSU, SUnit *PredSU) { - return SuccSU == &ExitSU || !Topo.IsReachable(PredSU, SuccSU); -} - -bool ScheduleDAGMI::addEdge(SUnit *SuccSU, const SDep &PredDep) { - if (SuccSU != &ExitSU) { - // Do not use WillCreateCycle, it assumes SD scheduling. - // If Pred is reachable from Succ, then the edge creates a cycle. - if (Topo.IsReachable(PredDep.getSUnit(), SuccSU)) - return false; - Topo.AddPred(SuccSU, PredDep.getSUnit()); - } - SuccSU->addPred(PredDep, /*Required=*/!PredDep.isArtificial()); - // Return true regardless of whether a new edge needed to be inserted. - return true; -} - /// ReleaseSucc - Decrement the NumPredsLeft count of a successor. When /// NumPredsLeft reaches zero, release the successor node. /// @@ -761,8 +744,6 @@ // Build the DAG. buildSchedGraph(AA); - Topo.InitDAGTopologicalSorting(); - postprocessDAG(); SmallVector TopRoots, BotRoots; @@ -1211,8 +1192,6 @@ LLVM_DEBUG(SchedImpl->dumpPolicy()); buildDAGWithRegPressure(); - Topo.InitDAGTopologicalSorting(); - postprocessDAG(); SmallVector TopRoots, BotRoots; @@ -1532,7 +1511,7 @@ void apply(ScheduleDAGInstrs *DAGInstrs) override; protected: - void clusterNeighboringMemOps(ArrayRef MemOps, ScheduleDAGMI *DAG); + void clusterNeighboringMemOps(ArrayRef MemOps, ScheduleDAGInstrs *DAG); }; class StoreClusterMutation : public BaseMemOpClusterMutation { @@ -1569,7 +1548,7 @@ } // end namespace llvm void BaseMemOpClusterMutation::clusterNeighboringMemOps( - ArrayRef MemOps, ScheduleDAGMI *DAG) { + ArrayRef MemOps, ScheduleDAGInstrs *DAG) { SmallVector MemOpRecords; for (SUnit *SU : MemOps) { MachineOperand *BaseOp; @@ -1609,9 +1588,7 @@ } /// Callback from DAG postProcessing to create cluster edges for loads. -void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAGInstrs) { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - +void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) { // Map DAG NodeNum to store chain ID. DenseMap StoreChainIDs; // Map each store chain to a set of dependent MemOps. diff --git a/llvm/lib/CodeGen/MacroFusion.cpp b/llvm/lib/CodeGen/MacroFusion.cpp --- a/llvm/lib/CodeGen/MacroFusion.cpp +++ b/llvm/lib/CodeGen/MacroFusion.cpp @@ -36,7 +36,7 @@ return Dep.getKind() == SDep::Anti || Dep.getKind() == SDep::Output; } -static bool fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU, +static bool fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU, SUnit &SecondSU) { // Check that neither instr is already paired with another along the edge // between them. @@ -48,7 +48,7 @@ if (SI.isCluster()) return false; // Though the reachability checks above could be made more generic, - // perhaps as part of ScheduleDAGMI::addEdge(), since such edges are valid, + // perhaps as part of ScheduleDAGInstrs::addEdge(), since such edges are valid, // the extra computation cost makes it less interesting in general cases. // Create a single weak edge between the adjacent instrs. The only effect is @@ -117,7 +117,7 @@ class MacroFusion : public ScheduleDAGMutation { ShouldSchedulePredTy shouldScheduleAdjacent; bool FuseBlock; - bool scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU); + bool scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU); public: MacroFusion(ShouldSchedulePredTy shouldScheduleAdjacent, bool FuseBlock) @@ -128,9 +128,7 @@ } // end anonymous namespace -void MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - +void MacroFusion::apply(ScheduleDAGInstrs *DAG) { if (FuseBlock) // For each of the SUnits in the scheduling block, try to fuse the instr in // it with one in its predecessors. @@ -144,7 +142,7 @@ /// Implement the fusion of instr pairs in the scheduling DAG, /// anchored at the instr in AnchorSU.. -bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU) { +bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU) { const MachineInstr &AnchorMI = *AnchorSU.getInstr(); const TargetInstrInfo &TII = *DAG.TII; const TargetSubtargetInfo &ST = DAG.MF.getSubtarget(); diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp --- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -114,7 +114,7 @@ : ScheduleDAG(mf), MLI(mli), MFI(mf.getFrameInfo()), RemoveKillFlags(RemoveKillFlags), UnknownValue(UndefValue::get( - Type::getVoidTy(mf.getFunction().getContext()))) { + Type::getVoidTy(mf.getFunction().getContext()))), Topo(SUnits, &ExitSU) { DbgValues.clear(); const TargetSubtargetInfo &ST = mf.getSubtarget(); @@ -968,6 +968,8 @@ Uses.clear(); CurrentVRegDefs.clear(); CurrentVRegUses.clear(); + + Topo.InitDAGTopologicalSorting(); } raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV) { @@ -1146,6 +1148,23 @@ return "dag." + BB->getFullName(); } +bool ScheduleDAGInstrs::canAddEdge(SUnit *SuccSU, SUnit *PredSU) { + return SuccSU == &ExitSU || !Topo.IsReachable(PredSU, SuccSU); +} + +bool ScheduleDAGInstrs::addEdge(SUnit *SuccSU, const SDep &PredDep) { + if (SuccSU != &ExitSU) { + // Do not use WillCreateCycle, it assumes SD scheduling. + // If Pred is reachable from Succ, then the edge creates a cycle. + if (Topo.IsReachable(PredDep.getSUnit(), SuccSU)) + return false; + Topo.AddPred(SuccSU, PredDep.getSUnit()); + } + SuccSU->addPred(PredDep, /*Required=*/!PredDep.isArtificial()); + // Return true regardless of whether a new edge needed to be inserted. + return true; +} + //===----------------------------------------------------------------------===// // SchedDFSResult Implementation //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -632,9 +632,7 @@ MemOpClusterMutation(const SIInstrInfo *tii) : TII(tii) {} - void apply(ScheduleDAGInstrs *DAGInstrs) override { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - + void apply(ScheduleDAGInstrs *DAG) override { SUnit *SUa = nullptr; // Search for two consequent memory operations and link them // to prevent scheduler from moving them apart.