Index: include/llvm/CodeGen/MachinePipeliner.h =================================================================== --- include/llvm/CodeGen/MachinePipeliner.h +++ include/llvm/CodeGen/MachinePipeliner.h @@ -278,6 +278,12 @@ static bool classof(const ScheduleDAGInstrs *DAG) { return true; } + void removeDeadInstructions(MachineBasicBlock *KernelBB, + MBBVectorTy &EpilogBBs); + void splitLifetimes(MachineBasicBlock *KernelBB, + ArrayRef EpilogBBs, + SMSchedule &Schedule); + private: void addLoopCarriedDependences(AliasAnalysis *AA); void updatePhiDependences(); @@ -315,11 +321,6 @@ SMSchedule &Schedule, ValueMapTy *VRMap, InstrMapTy &InstrMap, unsigned LastStageNum, unsigned CurStageNum, bool IsLast); - void removeDeadInstructions(MachineBasicBlock *KernelBB, - MBBVectorTy &EpilogBBs); - void splitLifetimes(MachineBasicBlock *KernelBB, - ArrayRef EpilogBBs, - SMSchedule &Schedule); void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, SMSchedule &Schedule, ValueMapTy *VRMap); @@ -529,7 +530,7 @@ /// The SMS algorithm allows negative values for cycles, so the first cycle /// in the schedule is the smallest cycle value. class SMSchedule { -private: +protected: /// Map from execution cycle to instructions. DenseMap> ScheduledInstrs; @@ -665,6 +666,177 @@ void dump() const; }; +/// Once SwingSchedulerDAG has generated a correct schedule, it calls an +/// implementation of this class to generate the code for that schedule. +/// +/// The PipelineEmitter class itself is abstract, but defines a suite of helper +/// objects that can be used to generate the schedule. +class PipelineEmitter { +protected: + /// A simple class that takes identical copies of a single basic block and + /// allows + /// querying of equivalent instructions at a later point when the blocks are + /// no longer identical. + class EquivalenceRegistry { + public: + EquivalenceRegistry() {} + void insert(MachineBasicBlock *MBB) { + unsigned Idx = 0; + for (auto &MI : *MBB) { + Insts[{MBB, Idx}] = &MI; + Indexes[&MI] = Idx++; + } + } + + MachineInstr *lookup(MachineInstr *MI, MachineBasicBlock *MBB) { + unsigned Idx = Indexes[MI]; + return Insts[{MBB, Idx}]; + } + + MachineOperand *lookup(MachineOperand *MO, MachineBasicBlock *MBB) { + unsigned Idx = MO->getParent()->findRegisterDefOperandIdx(MO->getReg()); + return &lookup(MO->getParent(), MBB)->getOperand(Idx); + } + + bool contains(MachineInstr *MI) { return Indexes.count(MI); } + + private: + DenseMap, MachineInstr *> Insts; + DenseMap Indexes; + }; + + /// A CGBlock wraps a MachineBasicBlock with a copy of the scheduled kernel. + /// One prototypical CGBlock is created from the original unpipelined loop and + /// the SMSchedule. This can then be duplicated to form the prolog and epilog + /// code. + class CGBlock { + public: + /// Create a ScheduledBlock from the original loop in BB. BB is rewritten + /// in-place to adhere to Schedule. Where schedule finalization has updated + /// instructions to use pre-inc values (NewMIs) the rewritten block uses + /// these updates. + CGBlock(MachineBasicBlock *BB, SMSchedule &Schedule, PipelineEmitter &PE, + LiveIntervals &LIS, EquivalenceRegistry &ER); + + /// Peel this loop. If PeelFront is true, peel the first iteration otherwise + /// peel the last iteration. + CGBlock peel(bool PeelFront); + + /// Mark all instructions that are not in the set LiveStages as Pruned. + /// Pruning does not happen in this function. + void setLiveStages(ArrayRef LiveStages); + /// Rewrite all uses of pruned-out instructions. + void rewritePrunedInstUses(MachineBasicBlock *LoopExit); + /// Actually prune all instructions marked as Pruned by setLiveStages. + void prune(); + void addPredecessor(CGBlock &PredCGB); + void setSinglePredecessor(CGBlock &PredCGB); + + MachineBasicBlock *getBB() { return BB; } + + private: + struct InstrInfo { + InstrInfo() = default; + InstrInfo(int Stage, int Cycle, unsigned Index, bool Prune) + : Stage(Stage), Cycle(Cycle), Index(Index), Prune(Prune) {} + /// The stage this instruction is scheduled in. + int Stage; + /// The cycle this instruction is scheduled in. + int Cycle; + /// The linear order of this instruction. Cycle only provides a partial + /// order - this provides a total order. + unsigned Index; + /// True if this instruction should be pruned out of this block. + bool Prune = false; + }; + + /// Private constructor for use by peel(). + CGBlock(MachineBasicBlock *BB, const CGBlock &CGB, + DenseMap InstrInfos); + + /// Reg is used by MI. Return the new register MI should use to adhere to + /// the schedule. Insert phis as necessary. + Register remapUse(Register Reg, MachineInstr &MI); + /// Insert a phi that carries LoopReg from the loop body and InitReg + /// otherwise. If InitReg is not given it is chosen arbitrarily. It will + /// either be undef or will be chosen so as to share another phi. + Register phi(Register LoopReg, Optional InitReg = {}, + const TargetRegisterClass *RC = nullptr); + /// Create an undef register of the given register class. + Register undef(const TargetRegisterClass *RC); + + MachineBasicBlock *BB; + SMSchedule &Schedule; + MachineRegisterInfo &MRI; + const TargetInstrInfo *TII; + LiveIntervals &LIS; + EquivalenceRegistry &ER; + + MachineBasicBlock *PreheaderBB; + DenseMap InstrInfos; + // Map from register class to canonical undef register for that class. + DenseMap Undefs; + // Map from to phi register for all created phis. Note + // that this map is only used when InitReg is non-undef. + DenseMap, Register> Phis; + // Map from LoopReg to phi register where the InitReg is undef. + DenseMap UndefPhis; + }; + + SMSchedule &Schedule; + MachineBasicBlock *BB; + MachineLoop &L; + LiveIntervals &LIS; + SwingSchedulerDAG *DAG; + const TargetInstrInfo *TII; + MachineRegisterInfo &MRI; + +public: + PipelineEmitter(SMSchedule &Schedule, MachineBasicBlock *BB, + LiveIntervals &LIS, MachineLoop &L, + SwingSchedulerDAG *DAG = nullptr) + : Schedule(Schedule), BB(BB), L(L), LIS(LIS), DAG(DAG), + TII(BB->getParent()->getSubtarget().getInstrInfo()), + MRI(BB->getParent()->getRegInfo()) {} + virtual ~PipelineEmitter(); + + // Rewrite BB to adhere to Schedule and emit any prolog or epilog code. + virtual void emit() = 0; +}; + +/// PipelinerEmitter implementation that produces prolog blocks linked to a +/// single shared set of epilog blocks. This reduces the number of epilog blocks +/// required, but assumes that it is correct to emit stages in non-FIFO order: +/// +/// Prolog0 -- executes A[0] +/// | \ +/// Prolog1 \ | executes A[1], B[0] +/// | \ | +/// Prolog2 \ || executes A[2], B[1], C[0] +/// | ||| +/// Kernel ||| executes X[3], Y[2], Z[1], W[0] +/// | / || +/// Epilog0 || executes (A or Y)[3] +/// | / | +/// Epilog1 / executes (A or Z)[2, 3] +/// | / +/// Epilog2 executes (A or W)[1, 2, 3] +/// +/// Note that assuming the kernel is taken the epilog order is: +/// Y[3], Z[2], Z[3], W[1], W[2], W[3] +/// Whereas a FIFO order for the epilog would be: +/// Y[3], Z[2], W[1], Z[3], W[2], W[3] +/// +/// For many targets this ordering does not matter. However, if a hardware +/// functional unit is being used that depends on FIFO order (for example a +/// streaming hardware unit), wrong code may be generated. +class SingleEpilogPipelineEmitter : public PipelineEmitter { +public: + using PipelineEmitter::PipelineEmitter; + + void emit() override; +}; + } // end namespace llvm #endif // LLVM_LIB_CODEGEN_MACHINEPIPELINER_H Index: lib/CodeGen/MachinePipeliner.cpp =================================================================== --- lib/CodeGen/MachinePipeliner.cpp +++ lib/CodeGen/MachinePipeliner.cpp @@ -167,7 +167,7 @@ static cl::opt EnableExperimentalCodeGen( "swp-experimental-cg", cl::desc("Enable experimental CG code in MachinePipeliner."), cl::Hidden, - cl::init(false)); + cl::init(true)); unsigned SwingSchedulerDAG::Circuits::MaxPaths = 5; char MachinePipeliner::ID = 0; @@ -522,10 +522,14 @@ } bool CanUseExperimentalCodeGen = InstrChanges.empty(); - if (EnableExperimentalCodeGen && CanUseExperimentalCodeGen) - experimentalGeneratePipeline(Schedule); - else + if (EnableExperimentalCodeGen && CanUseExperimentalCodeGen) { + SingleEpilogPipelineEmitter Emitter(Schedule, BB, LIS, Loop, this); + Emitter.emit(); + // NewMIs have been added to BB, so don't delete them. + NewMIs.clear(); + } else { generatePipelinedLoop(Schedule); + } ++NumPipelined; } @@ -4128,124 +4132,22 @@ } } -// A simple class that takes identical copies of a single basic block and allows -// querying of equivalent instructions at a later point when the blocks are no -// longer identical. -class EquivalenceRegistry { -public: - EquivalenceRegistry() {} - void insert(MachineBasicBlock *MBB) { - unsigned Idx = 0; - for (auto &MI : *MBB) { - Insts[{MBB, Idx}] = &MI; - Indexes[&MI] = Idx++; - } - } - - MachineInstr *lookup(MachineInstr *MI, MachineBasicBlock *MBB) { - unsigned Idx = Indexes[MI]; - return Insts[{MBB, Idx}]; - } - - MachineOperand *lookup(MachineOperand *MO, MachineBasicBlock *MBB) { - unsigned Idx = MO->getParent()->findRegisterDefOperandIdx(MO->getReg()); - return &lookup(MO->getParent(), MBB)->getOperand(Idx); - } - - bool contains(MachineInstr *MI) { return Indexes.count(MI); } - -private: - DenseMap, MachineInstr *> Insts; - DenseMap Indexes; -}; - -// A CGBlock wraps a MachineBasicBlock with a copy of the scheduled kernel. -// One prototypical CGBlock is created from the original unpipelined loop and -// the SMSchedule. This can then be duplicated to form the prolog and epilog -// code. -class CGBlock { -public: - // Create a ScheduledBlock from the original loop in BB. BB is rewritten - // in-place to adhere to Schedule. Where schedule finalization has updated - // instructions to use pre-inc values (NewMIs) the rewritten block uses these - // updates. - CGBlock(MachineBasicBlock *BB, SMSchedule &Schedule, SwingSchedulerDAG &DAG, - LiveIntervals &LIS, EquivalenceRegistry &ER); - - // Peel this loop in the direction given by LPD. - CGBlock peel(LoopPeelDirection LPD); - - // Mark all instructions that are not in the set LiveStages as Pruned. Pruning - // does not happen in this function. - void setLiveStages(ArrayRef LiveStages); - // Rewrite all uses of pruned-out instructions. - void rewritePrunedInstUses(MachineBasicBlock *LoopExit); - // Actually prune all instructions marked as Pruned by setLiveStages. - void prune(); - void addPredecessor(CGBlock &PredCGB); - void setSinglePredecessor(CGBlock &PredCGB); - void insertBranchToSuccessors(ArrayRef Cond); - - MachineBasicBlock *getBB() { return BB; } - -private: - struct InstrInfo { - InstrInfo() = default; - InstrInfo(int Stage, int Cycle, unsigned Index, bool Prune) : - Stage(Stage), Cycle(Cycle), Index(Index), Prune(Prune) {} - // The stage this instruction is scheduled in. - int Stage; - // The cycle this instruction is scheduled in. - int Cycle; - // The linear order of this instruction. Cycle only provides a partial - // order - this provides a total order. - unsigned Index; - // True if this instruction should be pruned out of this block. - bool Prune = false; - }; - - // Private constructor for use by peel(). - CGBlock(MachineBasicBlock *BB, const CGBlock &CGB, - DenseMap InstrInfos); - - // Reg is used by MI. Return the new register MI should use to adhere to the - // schedule. Insert phis as necessary. - Register remapUse(Register Reg, MachineInstr &MI); - // Insert a phi that carries LoopReg from the loop body and InitReg otherwise. - // If InitReg is not given it is chosen arbitrarily. It will either be undef - // or will be chosen so as to share another phi. - Register phi(Register LoopReg, Optional InitReg = {}, - const TargetRegisterClass *RC = nullptr); - // Create an undef register of the given register class. - Register undef(const TargetRegisterClass *RC); - - MachineBasicBlock *BB; - SMSchedule &Schedule; - SwingSchedulerDAG &DAG; - MachineRegisterInfo &MRI; - const TargetInstrInfo *TII; - LiveIntervals &LIS; - EquivalenceRegistry &ER; - - MachineBasicBlock *PreheaderBB; - DenseMap InstrInfos; - // Map from register class to canonical undef register for that class. - DenseMap Undefs; - // Map from to phi register for all created phis. Note that - // this map is only used when InitReg is non-undef. - DenseMap, Register> Phis; - // Map from LoopReg to phi register where the InitReg is undef. - DenseMap UndefPhis; -}; - +/// Encapsulates the state required for interacting with TII->reduceLoopCount. +/// FIXME: The reduceLoopCount API is overfit for hardware loop instructions. +/// Architectures that use "normal" conditional branches with induction +/// variables don't get a way to indicate instructions that are necessary for +/// calculating the loop count (which must be in stage0!) and don't get enough +/// information to reconstruct conditional branches in the kernel and prologs. struct ReduceLoopCountState { - ReduceLoopCountState(MachineBasicBlock *Preheader, - MachinePipeliner::LoopInfo &LI, unsigned NumPeeled) - : Preheader(Preheader), LI(LI), NumPeeled(NumPeeled) {} + ReduceLoopCountState(MachineBasicBlock *Preheader, MachineInstr *LoopCompare, + MachineInstr *LoopInductionVar, unsigned NumPeeled) + : Preheader(Preheader), LoopCompare(LoopCompare), + LoopInductionVar(LoopInductionVar), NumPeeled(NumPeeled) {} SmallVector Cond; MachineBasicBlock *Preheader; - MachinePipeliner::LoopInfo &LI; + MachineInstr *LoopCompare; + MachineInstr *LoopInductionVar; unsigned PrevLC = UINT_MAX; unsigned FirstLC = UINT_MAX; SmallVector PrevInsts; @@ -4259,8 +4161,8 @@ if (S.PrevLC == 0) return S.FirstLC; S.Cond.clear(); - unsigned LC = TII->reduceLoopCount(*BB, *S.Preheader, S.LI.LoopInductionVar, - *S.LI.LoopCompare, S.Cond, S.PrevInsts, + unsigned LC = TII->reduceLoopCount(*BB, *S.Preheader, S.LoopInductionVar, + *S.LoopCompare, S.Cond, S.PrevInsts, S.NumPeeled - 1 - S.I++, S.NumPeeled - 1); if (S.FirstLC == UINT_MAX) S.FirstLC = LC; @@ -4269,12 +4171,16 @@ } } // namespace -void SwingSchedulerDAG::experimentalGeneratePipeline(SMSchedule &Schedule) { +PipelineEmitter::~PipelineEmitter() {} + +void SingleEpilogPipelineEmitter::emit() { EquivalenceRegistry ER; MachineBasicBlock *PreheaderBB = *BB->pred_begin(); if (PreheaderBB == BB) PreheaderBB = *std::next(BB->pred_begin()); - MachineBasicBlock *LoopExitBB = Loop.getExitBlock(); + MachineBasicBlock *LoopExitBB = *BB->succ_begin(); + if (LoopExitBB == BB) + LoopExitBB = *std::next(BB->succ_begin()); std::list CGBlocks; SmallVector Prologs, Epilogs; @@ -4282,6 +4188,10 @@ unsigned NumStages = Schedule.getMaxStageCount() + 1; unsigned NumPrologs = NumStages - 1; + MachineInstr *LoopInductionVar = nullptr; + MachineInstr *LoopCompare = nullptr; + assert(!TII->analyzeLoop(L, LoopInductionVar, LoopCompare)); + // Create the first CGBlock, which is a rescheduled loop BB. CGBlocks.emplace_back(BB, Schedule, *this, LIS, ER); Kernel = CGBlocks.begin(); @@ -4293,7 +4203,7 @@ SmallVector LiveStages; for (unsigned I = 0; I < NumPrologs; ++I) { LiveStages.push_back(I); - auto It = CGBlocks.insert(Kernel, Kernel->peel(LPD_Front)); + auto It = CGBlocks.insert(Kernel, Kernel->peel(/*PeelFront=*/true)); It->setLiveStages(LiveStages); Prologs.push_back(&*It); } @@ -4302,9 +4212,9 @@ // stage NumStages-1, Epilog 1 performs stage NumStages-2 THEN NumStages-1. for (unsigned I = 0; I < NumPrologs; ++I) { unsigned StartStage = NumStages - 1 - I; - CGBlocks.insert(std::next(Kernel), Kernel->peel(LPD_Back)); + CGBlocks.insert(std::next(Kernel), Kernel->peel(/*PeelFront=*/false)); for (unsigned Stage = StartStage + 1; Stage < NumStages; ++Stage) { - CGBlocks.insert(std::next(Kernel), Kernel->peel(LPD_Back)); + CGBlocks.insert(std::next(Kernel), Kernel->peel(/*PeelFront=*/false)); } } auto EBI = std::next(Kernel); @@ -4321,7 +4231,8 @@ auto EI = Epilogs.begin(); auto PI = Prologs.rbegin(); unsigned N = 0; - ReduceLoopCountState RLCS(PreheaderBB, Pass.LI, NumPrologs); + ReduceLoopCountState RLCS(PreheaderBB, LoopCompare, LoopInductionVar, + NumPrologs); for (; EI != Epilogs.end(); ++EI, ++PI, ++N) { CGBlock &PB = **PI; CGBlock &EB = **EI; @@ -4371,37 +4282,48 @@ }); // Do a final optimization pass on the kernel. - SmallVector EpilogBBs; - for (auto I = std::next(Kernel); I != CGBlocks.end(); ++I) - EpilogBBs.push_back(I->getBB()); - splitLifetimes(Kernel->getBB(), EpilogBBs, Schedule); - removeDeadInstructions(Kernel->getBB(), EpilogBBs); - - for (CGBlock &CGB : CGBlocks) { - if (CGB.getBB()->pred_empty() || CGB.getBB()->succ_empty()) - CGB.getBB()->eraseFromParent(); + if (DAG) { + SmallVector EpilogBBs; + for (auto I = std::next(Kernel); I != CGBlocks.end(); ++I) + EpilogBBs.push_back(I->getBB()); + DAG->splitLifetimes(Kernel->getBB(), EpilogBBs, Schedule); + DAG->removeDeadInstructions(Kernel->getBB(), EpilogBBs); + } + + SmallPtrSet CreatedBlocks; + for (CGBlock &CGB : CGBlocks) + CreatedBlocks.insert(CGB.getBB()); + bool Changed = true; + while (Changed) { + Changed = false; + for (MachineBasicBlock *BB : CreatedBlocks) { + if (BB->pred_empty()) { + while (!BB->succ_empty()) + BB->removeSuccessor(BB->succ_begin()); + BB->eraseFromParent(); + CreatedBlocks.erase(BB); + Changed = true; + } + } } if (Kernel->getBB()->succ_size() == 1) Kernel->getBB()->eraseFromParent(); - - // NewMIs have been added to BB, so don't delete them. - NewMIs.clear(); } -CGBlock::CGBlock(MachineBasicBlock *BB, const CGBlock &CGB, - DenseMap NewInstrInfos) - : BB(BB), Schedule(CGB.Schedule), DAG(CGB.DAG), MRI(DAG.MRI), TII(DAG.TII), - LIS(CGB.LIS), ER(CGB.ER), InstrInfos(std::move(NewInstrInfos)) { +PipelineEmitter::CGBlock::CGBlock( + MachineBasicBlock *BB, const CGBlock &CGB, + DenseMap NewInstrInfos) + : BB(BB), Schedule(CGB.Schedule), MRI(CGB.MRI), TII(CGB.TII), LIS(CGB.LIS), + ER(CGB.ER), InstrInfos(std::move(NewInstrInfos)) { ER.insert(BB); for (auto &II : InstrInfos) II.second.Prune = false; } -CGBlock::CGBlock(MachineBasicBlock *BB, SMSchedule &Schedule, - SwingSchedulerDAG &DAG, LiveIntervals &LIS, - EquivalenceRegistry &ER) - : BB(BB), Schedule(Schedule), DAG(DAG), MRI(DAG.MRI), TII(DAG.TII), - LIS(LIS), ER(ER) { +PipelineEmitter::CGBlock::CGBlock(MachineBasicBlock *BB, SMSchedule &Schedule, + PipelineEmitter &PE, LiveIntervals &LIS, + EquivalenceRegistry &ER) + : BB(BB), Schedule(Schedule), MRI(PE.MRI), TII(PE.TII), LIS(LIS), ER(ER) { // Discover this block's preheader. PreheaderBB = *BB->pred_begin(); if (PreheaderBB == BB) @@ -4414,14 +4336,13 @@ auto InsertPt = BB->getFirstTerminator(); unsigned Index = 0; MachineInstr *FirstMI = nullptr; - for (int Cycle = Schedule.getFirstCycle(); Cycle <= Schedule.getFinalCycle(); ++Cycle) for (SUnit *SU : Schedule.getInstructions(Cycle)) { auto *MI = SU->getInstr(); - InstrInfos[MI] = {Schedule.stageScheduled(SU), Cycle, Index++, false}; if (MI->isPHI()) continue; + InstrInfos[MI] = {Schedule.stageScheduled(SU), Cycle, Index++, false}; if (MI->getParent()) MI->removeFromParent(); BB->insert(InsertPt, MI); @@ -4433,6 +4354,7 @@ // and the end of the block. Kill from the first non-phi to FirstMI. for (auto I = BB->getFirstNonPHI(); I != FirstMI->getIterator();) { LIS.RemoveMachineInstrFromMaps(*I); + InstrInfos.erase(&*I); (I++)->eraseFromParent(); } @@ -4473,7 +4395,7 @@ ER.insert(BB); } -Register CGBlock::remapUse(Register Reg, MachineInstr &MI) { +Register PipelineEmitter::CGBlock::remapUse(Register Reg, MachineInstr &MI) { MachineInstr *Producer = MRI.getUniqueVRegDef(Reg); if (!Producer) return Reg; @@ -4559,8 +4481,9 @@ return LoopReg; } -Register CGBlock::phi(Register LoopReg, Optional InitReg, - const TargetRegisterClass *RC) { +Register PipelineEmitter::CGBlock::phi(Register LoopReg, + Optional InitReg, + const TargetRegisterClass *RC) { // If the init register is not undef, try and find an existing phi. if (InitReg.hasValue()) { auto I = Phis.find({LoopReg, InitReg.getValue()}); @@ -4609,7 +4532,7 @@ return R; } -Register CGBlock::undef(const TargetRegisterClass *RC) { +Register PipelineEmitter::CGBlock::undef(const TargetRegisterClass *RC) { Register &R = Undefs[RC]; if (R == 0) { // Create an IMPLICIT_DEF that defines this register if we need it. @@ -4623,7 +4546,8 @@ return R; } -CGBlock CGBlock::peel(LoopPeelDirection LPD) { +PipelineEmitter::CGBlock PipelineEmitter::CGBlock::peel(bool PeelFront) { + auto LPD = PeelFront ? LPD_Front : LPD_Back; MachineBasicBlock *NewBB = PeelSingleBlockLoop(LPD, BB, MRI, TII); auto OI = BB->begin(); auto NI = NewBB->begin(); @@ -4635,7 +4559,7 @@ return CGBlock(NewBB, *this, std::move(NewInstrInfos)); } -void CGBlock::setLiveStages(ArrayRef LiveStages) { +void PipelineEmitter::CGBlock::setLiveStages(ArrayRef LiveStages) { for (auto &KV : InstrInfos) { if (KV.second.Stage != -1 && find(LiveStages, KV.second.Stage) == LiveStages.end()) @@ -4652,7 +4576,8 @@ llvm_unreachable("findPhiProviding failed to find an existing phi!"); } -void CGBlock::rewritePrunedInstUses(MachineBasicBlock *LoopExit) { +void PipelineEmitter::CGBlock::rewritePrunedInstUses( + MachineBasicBlock *LoopExit) { // Rewrite all uses of predicated-out instructions. for (MachineInstr &MI : *BB) { if (MI.isPHI() || InstrInfos.count(&MI) == 0) @@ -4720,7 +4645,7 @@ } } -void CGBlock::prune() { +void PipelineEmitter::CGBlock::prune() { // As a final pass, fix up any temporary illegal phis we created. These phis // were created in the middle of the block to switch between either a // loop-carried value computed in this block if available, or a default value @@ -4744,15 +4669,14 @@ } for (auto &KV : InstrInfos) { - if (!KV.first->isPHI() && KV.second.Prune) { + if (KV.second.Prune) { LIS.RemoveMachineInstrFromMaps(*KV.first); KV.first->eraseFromParent(); } } - EliminateDeadPhis(BB, MRI, LIS); } -void CGBlock::addPredecessor(CGBlock &PredCGB) { +void PipelineEmitter::CGBlock::addPredecessor(CGBlock &PredCGB) { // PredBB is our current unique predecessor. MachineBasicBlock *PredBB = *BB->pred_begin(); // OtherPredBB is the BB that we are adding as a predecessor. @@ -4778,7 +4702,7 @@ } } -void CGBlock::setSinglePredecessor(CGBlock &PredCGB) { +void PipelineEmitter::CGBlock::setSinglePredecessor(CGBlock &PredCGB) { // PredBB is our current unique predecessor. MachineBasicBlock *PredBB = *BB->pred_begin(); // OtherPredBB is the BB that we are adding as a predecessor.