Index: llvm/lib/CodeGen/MacroFusion.cpp =================================================================== --- llvm/lib/CodeGen/MacroFusion.cpp +++ llvm/lib/CodeGen/MacroFusion.cpp @@ -33,42 +33,71 @@ static cl::opt EnableMacroFusion("misched-fusion", cl::Hidden, cl::desc("Enable scheduling for macro fusion."), cl::init(true)); -static void fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU, +static bool fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU, SUnit &SecondSU) { + // Check that neither instr is already paired with another along the edge + // between them. + for (SDep &SI : FirstSU.Succs) + if (SI.isCluster()) + return false; + + for (SDep &SI : SecondSU.Preds) + if (SI.isCluster()) + return false; + // Though the reachability checks above could be made more generic, + // perhaps as part of ScheduleDAGMI::addEdge(), since such edges are valid, + // the extra computation cost makes it less interesting in general cases. + // Create a single weak edge between the adjacent instrs. The only effect is // to cause bottom-up scheduling to heavily prioritize the clustered instrs. - DAG.addEdge(&SecondSU, SDep(&FirstSU, SDep::Cluster)); + if (!DAG.addEdge(&SecondSU, SDep(&FirstSU, SDep::Cluster))) + return false; - // Adjust the latency between the anchor instr and its - // predecessors. - for (SDep &IDep : SecondSU.Preds) - if (IDep.getSUnit() == &FirstSU) - IDep.setLatency(0); + // Adjust the latency between both instrs. + for (SDep &SI : FirstSU.Succs) + if (SI.getSUnit() == &SecondSU) + SI.setLatency(0); - // Adjust the latency between the dependent instr and its - // predecessors. - for (SDep &IDep : FirstSU.Succs) - if (IDep.getSUnit() == &SecondSU) - IDep.setLatency(0); + for (SDep &SI : SecondSU.Preds) + if (SI.getSUnit() == &FirstSU) + SI.setLatency(0); - DEBUG(dbgs() << DAG.MF.getName() << "(): Macro fuse "; + DEBUG(dbgs() << "Macro fuse: "; FirstSU.print(dbgs(), &DAG); dbgs() << " - "; SecondSU.print(dbgs(), &DAG); dbgs() << " / "; dbgs() << DAG.TII->getName(FirstSU.getInstr()->getOpcode()) << " - " << DAG.TII->getName(SecondSU.getInstr()->getOpcode()) << '\n'; ); + // Make data dependencies from the FirstSU also dependent on the SecondSU to + // prevent them from being scheduled between the FirstSU and the SecondSU. if (&SecondSU != &DAG.ExitSU) - // Make instructions dependent on FirstSU also dependent on SecondSU to - // prevent them from being scheduled between FirstSU and and SecondSU. for (const SDep &SI : FirstSU.Succs) { - if (SI.getSUnit() == &SecondSU) + SUnit *SU = SI.getSUnit(); + if (SI.isWeak() || (SI.getKind() != SDep::Order && SI.isCtrl()) || + SU == &DAG.ExitSU || SU == &SecondSU || SU->isPred(&SecondSU)) + continue; + DEBUG(dbgs() << " Bind "; + SecondSU.print(dbgs(), &DAG); dbgs() << " - "; + SU->print(dbgs(), &DAG); dbgs() << '\n';); + DAG.addEdge(SU, SDep(&SecondSU, SDep::Artificial)); + } + + // Make the FirstSU also dependent on the dependencies of the SecondSU to + // prevent them from being scheduled between the FirstSU and the SecondSU. + if (&FirstSU != &DAG.EntrySU) + for (const SDep &SI : SecondSU.Preds) { + SUnit *SU = SI.getSUnit(); + if (SI.isWeak() || (SI.getKind() != SDep::Order && SI.isCtrl()) || + &FirstSU == SU || FirstSU.isSucc(SU)) continue; - DEBUG(dbgs() << " Copy Succ "; - SI.getSUnit()->print(dbgs(), &DAG); dbgs() << '\n';); - DAG.addEdge(SI.getSUnit(), SDep(&SecondSU, SDep::Artificial)); + DEBUG(dbgs() << " Bind "; + SU->print(dbgs(), &DAG); dbgs() << " - "; + FirstSU.print(dbgs(), &DAG); dbgs() << '\n';); + DAG.addEdge(&FirstSU, SDep(SU, SDep::Artificial)); } ++NumFused; + return true; } namespace { @@ -116,7 +145,7 @@ // Explorer for fusion candidates among the dependencies of the anchor instr. for (SDep &Dep : AnchorSU.Preds) { - // Ignore dependencies that don't enforce ordering. + // Ignore dependencies other than data or strong ordering. if (Dep.getKind() == SDep::Anti || Dep.getKind() == SDep::Output || Dep.isWeak()) continue; @@ -129,8 +158,8 @@ if (!shouldScheduleAdjacent(TII, ST, DepMI, AnchorMI)) continue; - fuseInstructionPair(DAG, DepSU, AnchorSU); - return true; + if (fuseInstructionPair(DAG, DepSU, AnchorSU)) + return true; } return false;