Index: lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp =================================================================== --- lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -105,6 +105,80 @@ cl::desc("Average inst/cycle whan no target itinerary exists.")); namespace { + +class LiveRegManager { + /// LiveRegDefs - A set of physical registers and their definition + /// that are "live". These nodes must be scheduled before any other nodes that + /// modifies the registers can be scheduled. + unsigned NumLiveRegs = 0; + std::vector Defs; + std::vector Gens; + +public: + LiveRegManager() = default; + LiveRegManager(const LiveRegManager &) = delete; + LiveRegManager& operator=(const LiveRegManager &) = delete; + + void init(unsigned NumRegs) { + // Allocate slots for each physical register, plus one for a special register + // to track the virtual resource of a calling sequence. + Defs.resize(NumRegs + 1); + Gens.resize(NumRegs + 1); + } + + SUnit* getDef(unsigned Reg) const { + assert((!Defs[Reg] || NumLiveRegs) && "Invalid number of live regs"); + return Defs[Reg]; + } + + void setDef(unsigned Reg, SUnit* SU) { + assert(SU && "Invalid Def"); + Defs[Reg] = SU; + } + + SUnit* getGen(unsigned Reg) const { + assert((!Gens[Reg] || NumLiveRegs) && "Invalid number of live regs"); + assert((!Gens[Reg] || Defs[Reg]) && "Def missing"); + return Gens[Reg]; + } + + void setGen(unsigned Reg, SUnit* SU) { + assert(SU && "Invalid Gen"); + assert(Defs[Reg] && "Def missing"); + Gens[Reg] = SU; + } + + void clear(unsigned Reg) { + assert(Reg < Defs.size() - 1 && "Invalid live reg number"); + assert(NumLiveRegs > 0 && Defs[Reg] && Gens[Reg] && "Live reg not set"); + --NumLiveRegs; + Defs[Reg] = nullptr; + Gens[Reg] = nullptr; + } + + SUnit* getCallResourceDef() const { return Defs.back(); } + SUnit* getCallResourceGen() const { return Gens.back(); } + void setCallResource(SUnit* Def, SUnit* Gen) { + ++NumLiveRegs; + Defs.back() = Def; + Gens.back() = Gen; + } + + void clearCallResource() { + assert(NumLiveRegs > 0 && Defs.back() && Gens.back() && "Call resource not set"); + --NumLiveRegs; + Defs.back() = nullptr; + Gens.back() = nullptr; + } + + bool empty() const { return NumLiveRegs == 0; } + + void incCounter() { ++NumLiveRegs; } + + const std::vector& getDefs() const { return Defs; } +}; + + //===----------------------------------------------------------------------===// /// ScheduleDAGRRList - The actual register reduction list scheduler /// implementation. This supports both top-down and bottom-up scheduling. @@ -133,17 +207,12 @@ /// MinAvailableCycle - Cycle of the soonest available instruction. unsigned MinAvailableCycle; + LiveRegManager LiveRegs; + /// IssueCount - Count instructions issued in this cycle /// Currently valid only for bottom-up scheduling. unsigned IssueCount; - /// LiveRegDefs - A set of physical registers and their definition - /// that are "live". These nodes must be scheduled before any other nodes that - /// modifies the registers can be scheduled. - unsigned NumLiveRegs; - std::vector LiveRegDefs; - std::vector LiveRegGens; - // Collect interferences between physical register use/defs. // Each interference is an SUnit and set of physical registers. SmallVector Interferences; @@ -325,11 +394,7 @@ CurCycle = 0; IssueCount = 0; MinAvailableCycle = DisableSchedCycles ? 0 : UINT_MAX; - NumLiveRegs = 0; - // Allocate slots for each physical register, plus one for a special register - // to track the virtual resource of a calling sequence. - LiveRegDefs.resize(TRI->getNumRegs() + 1, nullptr); - LiveRegGens.resize(TRI->getNumRegs() + 1, nullptr); + LiveRegs.init(TRI->getNumRegs()); CallSeqEndForStart.clear(); assert(Interferences.empty() && LRegsMap.empty() && "stale Interferences"); @@ -532,13 +597,13 @@ // expensive to copy the register. Make sure nothing that can // clobber the register is scheduled between the predecessor and // this node. - SUnit *RegDef = LiveRegDefs[I->getReg()]; (void)RegDef; + SUnit *RegDef = LiveRegs.getDef(I->getReg()); (void)RegDef; assert((!RegDef || RegDef == SU || RegDef == I->getSUnit()) && "interference on register dependence"); - LiveRegDefs[I->getReg()] = I->getSUnit(); - if (!LiveRegGens[I->getReg()]) { - ++NumLiveRegs; - LiveRegGens[I->getReg()] = SU; + LiveRegs.setDef(I->getReg(), I->getSUnit()); + if (!LiveRegs.getGen(I->getReg())) { + LiveRegs.incCounter(); + LiveRegs.setGen(I->getReg(), SU); } } } @@ -546,8 +611,7 @@ // If we're scheduling a lowered CALLSEQ_END, find the corresponding // CALLSEQ_BEGIN. Inject an artificial physical register dependence between // these nodes, to prevent other calls from being interscheduled with them. - unsigned CallResource = TRI->getNumRegs(); - if (!LiveRegDefs[CallResource]) + if (!LiveRegs.getCallResourceDef()) for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) if (Node->isMachineOpcode() && Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { @@ -558,9 +622,7 @@ SUnit *Def = &SUnits[N->getNodeId()]; CallSeqEndForStart[Def] = SU; - ++NumLiveRegs; - LiveRegDefs[CallResource] = Def; - LiveRegGens[CallResource] = SU; + LiveRegs.setCallResource(Def, SU); break; } } @@ -740,26 +802,20 @@ for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); I != E; ++I) { // LiveRegDegs[I->getReg()] != SU when SU is a two-address node. - if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] == SU) { - assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); - --NumLiveRegs; - LiveRegDefs[I->getReg()] = nullptr; - LiveRegGens[I->getReg()] = nullptr; + if (I->isAssignedRegDep() && LiveRegs.getDef(I->getReg()) == SU) { + LiveRegs.clear(I->getReg()); releaseInterferences(I->getReg()); } } // Release the special call resource dependence, if this is the beginning // of a call. - unsigned CallResource = TRI->getNumRegs(); - if (LiveRegDefs[CallResource] == SU) + if (LiveRegs.getCallResourceDef() == SU) for (const SDNode *SUNode = SU->getNode(); SUNode; SUNode = SUNode->getGluedNode()) { if (SUNode->isMachineOpcode() && SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) { - assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); - --NumLiveRegs; - LiveRegDefs[CallResource] = nullptr; - LiveRegGens[CallResource] = nullptr; + LiveRegs.clearCallResource(); + unsigned CallResource = TRI->getNumRegs(); releaseInterferences(CallResource); } } @@ -809,13 +865,10 @@ for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); I != E; ++I) { CapturePred(&*I); - if (I->isAssignedRegDep() && SU == LiveRegGens[I->getReg()]){ - assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); - assert(LiveRegDefs[I->getReg()] == I->getSUnit() && + if (I->isAssignedRegDep() && LiveRegs.getGen(I->getReg()) == SU) { + assert(LiveRegs.getDef(I->getReg()) == I->getSUnit() && "Physical register dependency violated?"); - --NumLiveRegs; - LiveRegDefs[I->getReg()] = nullptr; - LiveRegGens[I->getReg()] = nullptr; + LiveRegs.clear(I->getReg()); releaseInterferences(I->getReg()); } } @@ -827,23 +880,18 @@ SUNode = SUNode->getGluedNode()) { if (SUNode->isMachineOpcode() && SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) { - ++NumLiveRegs; - LiveRegDefs[CallResource] = SU; - LiveRegGens[CallResource] = CallSeqEndForStart[SU]; + LiveRegs.setCallResource(SU, CallSeqEndForStart[SU]); } } // Release the special call resource dependence, if this is the end // of a call. - if (LiveRegGens[CallResource] == SU) + if (LiveRegs.getCallResourceGen() == SU) for (const SDNode *SUNode = SU->getNode(); SUNode; SUNode = SUNode->getGluedNode()) { if (SUNode->isMachineOpcode() && SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { - assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!"); - --NumLiveRegs; - LiveRegDefs[CallResource] = nullptr; - LiveRegGens[CallResource] = nullptr; + LiveRegs.clearCallResource(); releaseInterferences(CallResource); } } @@ -851,22 +899,23 @@ for (auto &Succ : SU->Succs) { if (Succ.isAssignedRegDep()) { auto Reg = Succ.getReg(); - if (!LiveRegDefs[Reg]) - ++NumLiveRegs; + if (!LiveRegs.getDef(Reg)) + LiveRegs.incCounter(); // This becomes the nearest def. Note that an earlier def may still be // pending if this is a two-address node. - LiveRegDefs[Reg] = SU; + LiveRegs.setDef(Reg, SU); // Update LiveRegGen only if was empty before this unscheduling. // This is to avoid incorrect updating LiveRegGen set in previous run. - if (!LiveRegGens[Reg]) { + if (!LiveRegs.getGen(Reg)) { // Find the successor with the lowest height. - LiveRegGens[Reg] = Succ.getSUnit(); + auto Gen = Succ.getSUnit(); for (auto &Succ2 : SU->Succs) { if (Succ2.isAssignedRegDep() && Succ2.getReg() == Reg && - Succ2.getSUnit()->getHeight() < LiveRegGens[Reg]->getHeight()) - LiveRegGens[Reg] = Succ2.getSUnit(); + Succ2.getSUnit()->getHeight() < Gen->getHeight()) + Gen = Succ2.getSUnit(); } + LiveRegs.setGen(Reg, Gen); } } } @@ -1218,7 +1267,7 @@ /// CheckForLiveRegDef - Return true and update live register vector if the /// specified register def of the specified SUnit clobbers any "live" registers. static void CheckForLiveRegDef(SUnit *SU, unsigned Reg, - std::vector &LiveRegDefs, + const std::vector &LiveRegDefs, SmallSet &RegAdded, SmallVectorImpl &LRegs, const TargetRegisterInfo *TRI) { @@ -1240,7 +1289,7 @@ /// CheckForLiveRegDefMasked - Check for any live physregs that are clobbered /// by RegMask, and add them to LRegs. static void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask, - std::vector &LiveRegDefs, + const std::vector &LiveRegDefs, SmallSet &RegAdded, SmallVectorImpl &LRegs) { // Look at all live registers. Skip Reg0 and the special CallResource. @@ -1267,7 +1316,7 @@ /// whatever is necessary (i.e. backtracking or cloning) to make it possible. bool ScheduleDAGRRList:: DelayForLiveRegsBottomUp(SUnit *SU, SmallVectorImpl &LRegs) { - if (NumLiveRegs == 0) + if (LiveRegs.empty()) return false; SmallSet RegAdded; @@ -1277,8 +1326,8 @@ // then we are free to schedule it. for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); I != E; ++I) { - if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] != SU) - CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs, + if (I->isAssignedRegDep() && LiveRegs.getDef(I->getReg()) != SU) + CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegs.getDefs(), RegAdded, LRegs, TRI); } @@ -1302,7 +1351,7 @@ for (; NumVals; --NumVals, ++i) { unsigned Reg = cast(Node->getOperand(i))->getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) - CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI); + CheckForLiveRegDef(SU, Reg, LiveRegs.getDefs(), RegAdded, LRegs, TRI); } } else i += NumVals; @@ -1318,8 +1367,8 @@ if (Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) { // Check the special calling-sequence resource. unsigned CallResource = TRI->getNumRegs(); - if (LiveRegDefs[CallResource]) { - SDNode *Gen = LiveRegGens[CallResource]->getNode(); + if (LiveRegs.getCallResourceDef()) { + SDNode *Gen = LiveRegs.getCallResourceGen()->getNode(); while (SDNode *Glued = Gen->getGluedNode()) Gen = Glued; if (!IsChainDependent(Gen, Node, 0, TII) && @@ -1328,13 +1377,13 @@ } } if (const uint32_t *RegMask = getNodeRegMask(Node)) - CheckForLiveRegDefMasked(SU, RegMask, LiveRegDefs, RegAdded, LRegs); + CheckForLiveRegDefMasked(SU, RegMask, LiveRegs.getDefs(), RegAdded, LRegs); const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode()); if (!MCID.ImplicitDefs) continue; for (const uint16_t *Reg = MCID.getImplicitDefs(); *Reg; ++Reg) - CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI); + CheckForLiveRegDef(SU, *Reg, LiveRegs.getDefs(), RegAdded, LRegs, TRI); } return !LRegs.empty(); @@ -1408,8 +1457,8 @@ unsigned LiveCycle = UINT_MAX; for (unsigned j = 0, ee = LRegs.size(); j != ee; ++j) { unsigned Reg = LRegs[j]; - if (LiveRegGens[Reg]->getHeight() < LiveCycle) { - BtSU = LiveRegGens[Reg]; + if (LiveRegs.getGen(Reg)->getHeight() < LiveCycle) { + BtSU = LiveRegs.getGen(Reg); LiveCycle = BtSU->getHeight(); } } @@ -1452,7 +1501,8 @@ SmallVectorImpl &LRegs = LRegsMap[TrySU]; assert(LRegs.size() == 1 && "Can't handle this yet!"); unsigned Reg = LRegs[0]; - SUnit *LRDef = LiveRegDefs[Reg]; + SUnit *LRDef = LiveRegs.getDef(Reg); + assert(LRDef && "LiveRegDef not set"); MVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII); const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); @@ -1483,7 +1533,7 @@ DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum << " to SU #" << TrySU->NodeNum << "\n"); - LiveRegDefs[Reg] = NewDef; + LiveRegs.setDef(Reg, NewDef); AddPred(NewDef, SDep(TrySU, SDep::Artificial)); TrySU->isAvailable = false; CurSU = NewDef;