Index: include/llvm/CodeGen/MachineScheduler.h =================================================================== --- include/llvm/CodeGen/MachineScheduler.h +++ include/llvm/CodeGen/MachineScheduler.h @@ -476,6 +476,67 @@ const std::vector &NewMaxPressure); }; +/// \brief Post-process the DAG to create cluster edges between neighboring +/// loads or between neighboring stores. +class BaseMemOpClusterMutation : public ScheduleDAGMutation { + struct MemOpInfo { + SUnit *SU; + unsigned BaseReg; + int64_t Offset; + MemOpInfo(SUnit *su, unsigned reg, int64_t ofs) + : SU(su), BaseReg(reg), Offset(ofs) {} + + bool operator<(const MemOpInfo&RHS) const { + return std::tie(BaseReg, Offset) < std::tie(RHS.BaseReg, RHS.Offset); + } + }; + + const TargetInstrInfo *TII; + const TargetRegisterInfo *TRI; + bool IsLoad; + +public: + BaseMemOpClusterMutation(const TargetInstrInfo *tii, + const TargetRegisterInfo *tri, bool IsLoad) + : TII(tii), TRI(tri), IsLoad(IsLoad) {} + + void apply(ScheduleDAGInstrs *DAGInstrs) override; + +protected: + void clusterNeighboringMemOps(ArrayRef MemOps, ScheduleDAGMI *DAG); +}; + +class StoreClusterMutation : public BaseMemOpClusterMutation { +public: + StoreClusterMutation(const TargetInstrInfo *tii, + const TargetRegisterInfo *tri) + : BaseMemOpClusterMutation(tii, tri, false) {} +}; + +class LoadClusterMutation : public BaseMemOpClusterMutation { +public: + LoadClusterMutation(const TargetInstrInfo *tii, const TargetRegisterInfo *tri) + : BaseMemOpClusterMutation(tii, tri, true) {} +}; + +/// \brief Post-process the DAG to create weak edges from all uses of a copy to +/// the one use that defines the copy's source vreg, most likely an induction +/// variable increment. +class CopyConstrain : public ScheduleDAGMutation { + // Transient state. + SlotIndex RegionBeginIdx; + // RegionEndIdx is the slot index of the last non-debug instruction in the + // scheduling region. So we may have RegionBeginIdx == RegionEndIdx. + SlotIndex RegionEndIdx; +public: + CopyConstrain(const TargetInstrInfo *, const TargetRegisterInfo *) {} + + void apply(ScheduleDAGInstrs *DAGInstrs) override; + +protected: + void constrainLocalCopy(SUnit *CopySU, ScheduleDAGMILive *DAG); +}; + //===----------------------------------------------------------------------===// /// /// Helpers for implementing custom MachineSchedStrategy classes. These take Index: lib/CodeGen/MachineScheduler.cpp =================================================================== --- lib/CodeGen/MachineScheduler.cpp +++ lib/CodeGen/MachineScheduler.cpp @@ -1351,51 +1351,6 @@ // BaseMemOpClusterMutation - DAG post-processing to cluster loads or stores. //===----------------------------------------------------------------------===// -namespace { -/// \brief Post-process the DAG to create cluster edges between neighboring -/// loads or between neighboring stores. -class BaseMemOpClusterMutation : public ScheduleDAGMutation { - struct MemOpInfo { - SUnit *SU; - unsigned BaseReg; - int64_t Offset; - MemOpInfo(SUnit *su, unsigned reg, int64_t ofs) - : SU(su), BaseReg(reg), Offset(ofs) {} - - bool operator<(const MemOpInfo&RHS) const { - return std::tie(BaseReg, Offset) < std::tie(RHS.BaseReg, RHS.Offset); - } - }; - - const TargetInstrInfo *TII; - const TargetRegisterInfo *TRI; - bool IsLoad; - -public: - BaseMemOpClusterMutation(const TargetInstrInfo *tii, - const TargetRegisterInfo *tri, bool IsLoad) - : TII(tii), TRI(tri), IsLoad(IsLoad) {} - - void apply(ScheduleDAGInstrs *DAGInstrs) override; - -protected: - void clusterNeighboringMemOps(ArrayRef MemOps, ScheduleDAGMI *DAG); -}; - -class StoreClusterMutation : public BaseMemOpClusterMutation { -public: - StoreClusterMutation(const TargetInstrInfo *tii, - const TargetRegisterInfo *tri) - : BaseMemOpClusterMutation(tii, tri, false) {} -}; - -class LoadClusterMutation : public BaseMemOpClusterMutation { -public: - LoadClusterMutation(const TargetInstrInfo *tii, const TargetRegisterInfo *tri) - : BaseMemOpClusterMutation(tii, tri, true) {} -}; -} // anonymous - void BaseMemOpClusterMutation::clusterNeighboringMemOps( ArrayRef MemOps, ScheduleDAGMI *DAG) { SmallVector MemOpRecords; @@ -1553,26 +1508,6 @@ // CopyConstrain - DAG post-processing to encourage copy elimination. //===----------------------------------------------------------------------===// -namespace { -/// \brief Post-process the DAG to create weak edges from all uses of a copy to -/// the one use that defines the copy's source vreg, most likely an induction -/// variable increment. -class CopyConstrain : public ScheduleDAGMutation { - // Transient state. - SlotIndex RegionBeginIdx; - // RegionEndIdx is the slot index of the last non-debug instruction in the - // scheduling region. So we may have RegionBeginIdx == RegionEndIdx. - SlotIndex RegionEndIdx; -public: - CopyConstrain(const TargetInstrInfo *, const TargetRegisterInfo *) {} - - void apply(ScheduleDAGInstrs *DAGInstrs) override; - -protected: - void constrainLocalCopy(SUnit *CopySU, ScheduleDAGMILive *DAG); -}; -} // anonymous - /// constrainLocalCopy handles two possibilities: /// 1) Local src: /// I0: = dst