diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -264,10 +264,6 @@ LiveIntervals *LIS; std::unique_ptr SchedImpl; - /// Topo - A topological ordering for SUnits which permits fast IsReachable - /// and similar queries. - ScheduleDAGTopologicalSort Topo; - /// Ordered list of DAG postprocessing steps. std::vector> Mutations; @@ -291,7 +287,7 @@ ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr S, bool RemoveKillFlags) : ScheduleDAGInstrs(*C->MF, C->MLI, RemoveKillFlags), AA(C->AA), - LIS(C->LIS), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU) {} + LIS(C->LIS), SchedImpl(std::move(S)) {} // Provide a vtable anchor ~ScheduleDAGMI() override; @@ -319,17 +315,6 @@ Mutations.push_back(std::move(Mutation)); } - /// True if an edge can be added from PredSU to SuccSU without creating - /// a cycle. - bool canAddEdge(SUnit *SuccSU, SUnit *PredSU); - - /// Add a DAG edge to the given SU with the given predecessor - /// dependence data. - /// - /// \returns true if the edge may be added without creating a cycle OR if an - /// equivalent edge already existed (false indicates failure). - bool addEdge(SUnit *SuccSU, const SDep &PredDep); - MachineBasicBlock::iterator top() const { return CurrentTop; } MachineBasicBlock::iterator bottom() const { return CurrentBottom; } diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -234,6 +234,10 @@ /// For an unanalyzable memory access, this Value is used in maps. UndefValue *UnknownValue; + /// Topo - A topological ordering for SUnits which permits fast IsReachable + /// and similar queries. + ScheduleDAGTopologicalSort Topo; + using DbgValueVector = std::vector>; /// Remember instruction that precedes DBG_VALUE. @@ -338,6 +342,17 @@ /// Fixes register kill flags that scheduling has made invalid. void fixupKills(MachineBasicBlock &MBB); + /// True if an edge can be added from PredSU to SuccSU without creating + /// a cycle. + bool canAddEdge(SUnit *SuccSU, SUnit *PredSU); + + /// Add a DAG edge to the given SU with the given predecessor + /// dependence data. + /// + /// \returns true if the edge may be added without creating a cycle OR if an + /// equivalent edge already existed (false indicates failure). + bool addEdge(SUnit *SuccSU, const SDep &PredDep); + protected: void initSUnits(); void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx); diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -604,23 +604,6 @@ // Provide a vtable anchor. ScheduleDAGMI::~ScheduleDAGMI() = default; -bool ScheduleDAGMI::canAddEdge(SUnit *SuccSU, SUnit *PredSU) { - return SuccSU == &ExitSU || !Topo.IsReachable(PredSU, SuccSU); -} - -bool ScheduleDAGMI::addEdge(SUnit *SuccSU, const SDep &PredDep) { - if (SuccSU != &ExitSU) { - // Do not use WillCreateCycle, it assumes SD scheduling. - // If Pred is reachable from Succ, then the edge creates a cycle. - if (Topo.IsReachable(PredDep.getSUnit(), SuccSU)) - return false; - Topo.AddPred(SuccSU, PredDep.getSUnit()); - } - SuccSU->addPred(PredDep, /*Required=*/!PredDep.isArtificial()); - // Return true regardless of whether a new edge needed to be inserted. - return true; -} - /// ReleaseSucc - Decrement the NumPredsLeft count of a successor. When /// NumPredsLeft reaches zero, release the successor node. /// @@ -761,8 +744,6 @@ // Build the DAG. buildSchedGraph(AA); - Topo.InitDAGTopologicalSorting(); - postprocessDAG(); SmallVector TopRoots, BotRoots; @@ -1211,8 +1192,6 @@ LLVM_DEBUG(SchedImpl->dumpPolicy()); buildDAGWithRegPressure(); - Topo.InitDAGTopologicalSorting(); - postprocessDAG(); SmallVector TopRoots, BotRoots; @@ -1532,7 +1511,8 @@ void apply(ScheduleDAGInstrs *DAGInstrs) override; protected: - void clusterNeighboringMemOps(ArrayRef MemOps, ScheduleDAGMI *DAG); + void clusterNeighboringMemOps(ArrayRef MemOps, + ScheduleDAGInstrs *DAG); }; class StoreClusterMutation : public BaseMemOpClusterMutation { @@ -1569,7 +1549,7 @@ } // end namespace llvm void BaseMemOpClusterMutation::clusterNeighboringMemOps( - ArrayRef MemOps, ScheduleDAGMI *DAG) { + ArrayRef MemOps, ScheduleDAGInstrs *DAG) { SmallVector MemOpRecords; for (SUnit *SU : MemOps) { MachineOperand *BaseOp; @@ -1609,9 +1589,7 @@ } /// Callback from DAG postProcessing to create cluster edges for loads. -void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAGInstrs) { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - +void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) { // Map DAG NodeNum to store chain ID. DenseMap StoreChainIDs; // Map each store chain to a set of dependent MemOps. diff --git a/llvm/lib/CodeGen/MacroFusion.cpp b/llvm/lib/CodeGen/MacroFusion.cpp --- a/llvm/lib/CodeGen/MacroFusion.cpp +++ b/llvm/lib/CodeGen/MacroFusion.cpp @@ -36,7 +36,7 @@ return Dep.getKind() == SDep::Anti || Dep.getKind() == SDep::Output; } -static bool fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU, +static bool fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU, SUnit &SecondSU) { // Check that neither instr is already paired with another along the edge // between them. @@ -48,8 +48,9 @@ if (SI.isCluster()) return false; // Though the reachability checks above could be made more generic, - // perhaps as part of ScheduleDAGMI::addEdge(), since such edges are valid, - // the extra computation cost makes it less interesting in general cases. + // perhaps as part of ScheduleDAGInstrs::addEdge(), since such edges are + // valid, the extra computation cost makes it less interesting in general + // cases. // Create a single weak edge between the adjacent instrs. The only effect is // to cause bottom-up scheduling to heavily prioritize the clustered instrs. @@ -117,7 +118,7 @@ class MacroFusion : public ScheduleDAGMutation { ShouldSchedulePredTy shouldScheduleAdjacent; bool FuseBlock; - bool scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU); + bool scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU); public: MacroFusion(ShouldSchedulePredTy shouldScheduleAdjacent, bool FuseBlock) @@ -128,9 +129,7 @@ } // end anonymous namespace -void MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - +void MacroFusion::apply(ScheduleDAGInstrs *DAG) { if (FuseBlock) // For each of the SUnits in the scheduling block, try to fuse the instr in // it with one in its predecessors. @@ -144,7 +143,8 @@ /// Implement the fusion of instr pairs in the scheduling DAG, /// anchored at the instr in AnchorSU.. -bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU) { +bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, + SUnit &AnchorSU) { const MachineInstr &AnchorMI = *AnchorSU.getInstr(); const TargetInstrInfo &TII = *DAG.TII; const TargetSubtargetInfo &ST = DAG.MF.getSubtarget(); diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp --- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -113,8 +113,9 @@ bool RemoveKillFlags) : ScheduleDAG(mf), MLI(mli), MFI(mf.getFrameInfo()), RemoveKillFlags(RemoveKillFlags), - UnknownValue(UndefValue::get( - Type::getVoidTy(mf.getFunction().getContext()))) { + UnknownValue( + UndefValue::get(Type::getVoidTy(mf.getFunction().getContext()))), + Topo(SUnits, &ExitSU) { DbgValues.clear(); const TargetSubtargetInfo &ST = mf.getSubtarget(); @@ -968,6 +969,8 @@ Uses.clear(); CurrentVRegDefs.clear(); CurrentVRegUses.clear(); + + Topo.InitDAGTopologicalSorting(); } raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV) { @@ -1146,6 +1149,23 @@ return "dag." + BB->getFullName(); } +bool ScheduleDAGInstrs::canAddEdge(SUnit *SuccSU, SUnit *PredSU) { + return SuccSU == &ExitSU || !Topo.IsReachable(PredSU, SuccSU); +} + +bool ScheduleDAGInstrs::addEdge(SUnit *SuccSU, const SDep &PredDep) { + if (SuccSU != &ExitSU) { + // Do not use WillCreateCycle, it assumes SD scheduling. + // If Pred is reachable from Succ, then the edge creates a cycle. + if (Topo.IsReachable(PredDep.getSUnit(), SuccSU)) + return false; + Topo.AddPred(SuccSU, PredDep.getSUnit()); + } + SuccSU->addPred(PredDep, /*Required=*/!PredDep.isArtificial()); + // Return true regardless of whether a new edge needed to be inserted. + return true; +} + //===----------------------------------------------------------------------===// // SchedDFSResult Implementation //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -632,9 +632,7 @@ MemOpClusterMutation(const SIInstrInfo *tii) : TII(tii) {} - void apply(ScheduleDAGInstrs *DAGInstrs) override { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - + void apply(ScheduleDAGInstrs *DAG) override { SUnit *SUa = nullptr; // Search for two consequent memory operations and link them // to prevent scheduler from moving them apart. diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h --- a/llvm/lib/Target/X86/X86Subtarget.h +++ b/llvm/lib/Target/X86/X86Subtarget.h @@ -835,6 +835,9 @@ bool enableEarlyIfConversion() const override; + void getPostRAMutations( + std::vector> &Mutations) const; + AntiDepBreakMode getAntiDepBreakMode() const override { return TargetSubtargetInfo::ANTIDEP_CRITICAL; } diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp --- a/llvm/lib/Target/X86/X86Subtarget.cpp +++ b/llvm/lib/Target/X86/X86Subtarget.cpp @@ -14,6 +14,7 @@ #include "X86CallLowering.h" #include "X86LegalizerInfo.h" +#include "X86MacroFusion.h" #include "X86RegisterBankInfo.h" #include "X86Subtarget.h" #include "MCTargetDesc/X86BaseInfo.h" @@ -366,3 +367,8 @@ bool X86Subtarget::enableEarlyIfConversion() const { return hasCMov() && X86EarlyIfConv; } + +void X86Subtarget::getPostRAMutations( + std::vector> &Mutations) const { + Mutations.push_back(createX86MacroFusionDAGMutation()); +}