Index: include/llvm/CodeGen/VirtRegMap.h =================================================================== --- include/llvm/CodeGen/VirtRegMap.h +++ include/llvm/CodeGen/VirtRegMap.h @@ -17,6 +17,7 @@ #ifndef LLVM_CODEGEN_VIRTREGMAP_H #define LLVM_CODEGEN_VIRTREGMAP_H +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -60,6 +61,10 @@ /// mapping. IndexedMap Virt2SplitMap; + /// Virt2SibingsMap - This is original register to a set containing all + /// its siblings mapping. + DenseMap> Virt2SiblingsMap; + /// createSpillSlot - Allocate a spill slot for RC from MFI. unsigned createSpillSlot(const TargetRegisterClass *RC); @@ -144,6 +149,15 @@ return Virt2SplitMap[virtReg]; } + /// @brief records Sibling is a sibling register generated from OrigReg. + void setIsSiblingsOfReg(unsigned OrigReg, unsigned Sibling) { + Virt2SiblingsMap[OrigReg].insert(Sibling); + } + + DenseSet &getSiblings(unsigned OrigReg) { + return Virt2SiblingsMap[OrigReg]; + } + /// getOriginal - Return the original virtual register that VirtReg descends /// from through splitting. /// A register that was not created by splitting is its own original. Index: lib/CodeGen/InlineSpiller.cpp =================================================================== --- lib/CodeGen/InlineSpiller.cpp +++ lib/CodeGen/InlineSpiller.cpp @@ -132,6 +132,13 @@ typedef DenseMap SibValueMap; SibValueMap SibValues; + // Mapping from pair of (StackSlot and Original VNI) to a set of spills which + // have the same stackslot and have equal values defined by Original VNI. + // These spills are mergable and are hoist candiates. + typedef DenseMap, SmallPtrSet> + MergableSpillsMap; + MergableSpillsMap MergableSpills; + // Dead defs generated during spilling. SmallVector DeadDefs; @@ -151,6 +158,16 @@ void spill(LiveRangeEdit &) override; + void addToMergableSpills(MachineInstr *Spill); + void rmFromMergableSpills(MachineInstr *Spill); + bool isSpillCandBB(unsigned OrigReg, VNInfo *OrigVNI, MachineBasicBlock *BB, + unsigned &LiveReg); + void runHoistSpills(unsigned OrigReg, VNInfo *OrigVNI, + SmallPtrSet &Spills, + SmallPtrSet &SpillsToRm, + DenseMap &SpillsToIns); + void hoistAllSpills() override; + private: bool isSnippet(const LiveInterval &SnipLI); void collectRegsToSpill(); @@ -809,6 +826,7 @@ DeadDefs.push_back(MI); ++NumSpillsRemoved; --NumSpills; + rmFromMergableSpills(MI); } } } while (!WorkList.empty()); @@ -1023,6 +1041,9 @@ if (InstrReg != Reg || FI != StackSlot) return false; + if (!IsLoad) + rmFromMergableSpills(MI); + DEBUG(dbgs() << "Coalescing stack access: " << *MI); LIS.RemoveMachineInstrFromMaps(MI); MI->eraseFromParent(); @@ -1173,9 +1194,10 @@ if (!WasCopy) ++NumFolded; - else if (Ops.front().second == 0) + else if (Ops.front().second == 0) { ++NumSpills; - else + addToMergableSpills(FoldMI); + } else ++NumReloads; return true; } @@ -1210,6 +1232,7 @@ DEBUG(dumpMachineInstrRangeWithSlotIndex(std::next(MI), MIS.end(), LIS, "spill")); ++NumSpills; + addToMergableSpills(std::next(MI)); } /// spillAroundUses - insert spill code around each use of Reg. @@ -1387,7 +1410,6 @@ assert(DeadDefs.empty() && "Previous spill didn't remove dead defs"); collectRegsToSpill(); - analyzeSiblingValues(); reMaterializeAll(); // Remat may handle everything. @@ -1396,3 +1418,299 @@ Edit->calculateRegClassAndHint(MF, Loops, MBFI); } + +// When a spill is inserted, add the spill to MergableSpills map. +void InlineSpiller::addToMergableSpills(MachineInstr *Spill) { + SlotIndex Idx = LIS.getInstructionIndex(Spill); + VNInfo *OrigVNI = LIS.getInterval(Original).getVNInfoAt(Idx.getRegSlot()); + bool Inserted; + std::pair MIdx = std::make_pair(StackSlot, OrigVNI); + MergableSpillsMap::iterator VI = MergableSpills.find(MIdx); + if (VI == MergableSpills.end()) + std::tie(VI, Inserted) = MergableSpills.insert( + std::make_pair(MIdx, SmallPtrSet())); + VI->second.insert(Spill); + int size = VI->second.size(); + assert(size > 0); +} + +// When a spill is removed, remove the spill from MergableSpills map. +void InlineSpiller::rmFromMergableSpills(MachineInstr *Spill) { + SlotIndex Idx = LIS.getInstructionIndex(Spill); + VNInfo *OrigVNI = LIS.getInterval(Original).getVNInfoAt(Idx.getRegSlot()); + std::pair MIdx = std::make_pair(StackSlot, OrigVNI); + SmallPtrSet &SpillsSet = MergableSpills[MIdx]; + SpillsSet.erase(Spill); +} + +// Check BB to see if it is a possible target BB to place a hoisted spill, +// .i.e, there should be a living sibling of OrigReg at the insert point. +bool InlineSpiller::isSpillCandBB(unsigned OrigReg, VNInfo *OrigVNI, + MachineBasicBlock *BB, unsigned &LiveReg) { + SlotIndex Idx; + MachineBasicBlock::iterator MI = BB->getFirstTerminator(); + if (MI != BB->end()) + Idx = LIS.getInstructionIndex(MI); + else + Idx = LIS.getMBBEndIdx(BB).getPrevSlot(); + DenseSet &Siblings = VRM.getSiblings(OrigReg); + LiveInterval &OrigLI = LIS.getInterval(OrigReg); + VNInfo *BBVNI = OrigLI.getVNInfoAt(Idx); + assert(BBVNI == OrigVNI && "Unexpected VNI"); + + for (auto const ent : Siblings) { + LiveInterval &LI = LIS.getInterval(ent); + VNInfo *VNI = LI.getVNInfoAt(Idx); + if (VNI) { + LiveReg = ent; + return true; + } + } + return false; +} + +// Try to hoist spills according to BB hotness. The spills to removed will +// be saved in SpillsToRm. The spills to be inserted will be saved in +// SpillsToIns. +void InlineSpiller::runHoistSpills( + unsigned OrigReg, VNInfo *OrigVNI, SmallPtrSet &Spills, + SmallPtrSet &SpillsToRm, + DenseMap &SpillsToIns) { + // Clear non-live siblings. A BB is a proper hoist target only if there + // is a sibling reg living there, so the sibling reg can be used as the + // RHS of the hoisted spill. + DenseSet &Siblings = VRM.getSiblings(OrigReg); + DenseSet::Iterator SibIt = Siblings.begin(); + DenseSet::Iterator SibNextIt = SibIt; + for (; SibIt != Siblings.end(); SibIt = SibNextIt) { + ++SibNextIt; + if (MRI.def_empty(*SibIt)) + Siblings.erase(SibIt); + } + + // SpillsToKept contains all the nodes where spills are to be inserted + // during hoisting. If the spill to be inserted is an original spill + // (not a hoisted one), the value of the map entry is 0. If the spill + // is a hoisted spill, the value of the map entry is the VReg to be used + // in the RHS of the spill. + DenseMap SpillsToKept; + // Map from BB to the spill inside of it. + DenseMap SpillBBToSpill; + for (const auto Spill : Spills) { + MachineBasicBlock *Block = Spill->getParent(); + SpillsToKept[MDT.DT->getNode(Block)] = 0; + SpillBBToSpill[MDT.DT->getNode(Block)] = Spill; + } + + // Get the preorder walk order of the dominator tree. + SmallVector Orders; + SmallVector WorkList; + MachineBasicBlock *Root = LIS.getMBBFromIndex(OrigVNI->def); + WorkList.push_back(MDT.DT->getNode(Root)); + do { + MachineDomTreeNode *Node = WorkList.pop_back_val(); + Orders.push_back(Node); + // Don't include children of BB if the BB contains original spill. + if (SpillsToKept.find(Node) != SpillsToKept.end()) + continue; + const std::vector &Children = Node->getChildren(); + unsigned NumChildren = Children.size(); + for (unsigned i = 0; i != NumChildren; ++i) { + MachineDomTreeNode *Child = Children[i]; + WorkList.push_back(Child); + } + } while (!WorkList.empty()); + + // SpillsInSubTree keeps the map from a dom tree node to a nodes set. + // It indicates the places where spills are to be inserted in the subtree + // of the dom node for the time being. + DenseMap> + SpillsInSubTree; + + // Perform a reverse preorder walk of the dominator tree. Once we visit + // a dom tree node, we know its children has already been visited. When + // we are visiting a dom tree node, collect spills scattered inside the + // subtree of current node and see if it is proper to hoist those spills + // to current node. + SmallVector::reverse_iterator RIt = Orders.rbegin(); + for (; RIt != Orders.rend(); RIt++) { + MachineBasicBlock *Block = (*RIt)->getBlock(); + + // If Block contains an original spill, simply continue. + if (SpillsToKept.find(*RIt) != SpillsToKept.end() && !SpillsToKept[*RIt]) { + SpillsInSubTree[*RIt].insert(*RIt); + continue; + } + + // Collect spills in subtree of current node (*RIt) to + // SpillsInSubTree[*RIt]. + const std::vector &Children = (*RIt)->getChildren(); + unsigned NumChildren = Children.size(); + for (unsigned i = 0; i != NumChildren; ++i) { + MachineDomTreeNode *Child = Children[i]; + SpillsInSubTree[*RIt].insert(SpillsInSubTree[Child].begin(), + SpillsInSubTree[Child].end()); + SpillsInSubTree.erase(Child); + } + + // No spills in subtree, simply continue. + if (SpillsInSubTree[*RIt].empty()) + continue; + + // Check whether Block is a possible candidate to insert spill. + unsigned LiveReg = 0; + if (!isSpillCandBB(OrigReg, OrigVNI, Block, LiveReg)) + continue; + + // Now Block is a proper target BB for hoisting spills. Decide whether to + // hoist the spills to current node. Get existing cost of all the spills + // in SpillsInSubTree[Block]. + BlockFrequency SpillCost = 0; + for (const auto SpillBB : SpillsInSubTree[*RIt]) + SpillCost += MBFI.getBlockFreq(SpillBB->getBlock()); + + // If there are multiple spills that could be merged, bias a little + // to hoist the spill. + BranchProbability MarginProb = (SpillsInSubTree[*RIt].size() > 1) + ? BranchProbability(9, 10) + : BranchProbability(1, 1); + if (SpillCost > MBFI.getBlockFreq(Block) * MarginProb) { + // Hoist: Move spills to current Block. + for (const auto SpillBB : SpillsInSubTree[*RIt]) { + // When SpillBB is a BB contains original spill, insert the spill + // to SpillsToRm. + if (SpillsToKept.find(SpillBB) != SpillsToKept.end() && + !SpillsToKept[SpillBB]) { + MachineInstr *SpillToRm = SpillBBToSpill[SpillBB]; + SpillsToRm.insert(SpillToRm); + } + // SpillBB will not contain spill anymore, remove it from SpillsToKept. + SpillsToKept.erase(SpillBB); + } + // Current Block is the BB containing the new hoisted spill. Add it to + // SpillsToKept. LiveReg is the RHS of the spill. + SpillsToKept[*RIt] = LiveReg; + DEBUG({ + dbgs() << "spills in BB: "; + for (const auto Rspill : SpillsInSubTree[*RIt]) + dbgs() << Rspill->getBlock()->getNumber() << " "; + dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() + << "\n"; + }); + SpillsInSubTree[*RIt].clear(); + SpillsInSubTree[*RIt].insert(*RIt); + } + } + // For spills in SpillsToKept with LiveReg set (.i.e, not original spill), + // save them to SpillsToIns. + for (const auto ent : SpillsToKept) { + if (ent.second) + SpillsToIns[ent.first->getBlock()] = ent.second; + } +} + +// For spills with equal values, remove redundent spills and hoist spills +// to a less hot spot. +void InlineSpiller::hoistAllSpills() { + // Save the mapping between stackslot and its original reg. + DenseMap SlotToOrigReg; + for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) { + unsigned Reg = TargetRegisterInfo::index2VirtReg(i); + int Slot = VRM.getStackSlot(Reg); + if (Slot != VirtRegMap::NO_STACK_SLOT) { + for (const auto &ent : MergableSpills) { + if (ent.first.first == Slot && + SlotToOrigReg.find(Slot) == SlotToOrigReg.end()) + SlotToOrigReg[Slot] = VRM.getOriginal(Reg); + } + } + } + + // Each entry in MergableSpills contains a spill set with equal values. + for (auto &ent : MergableSpills) { + int Slot = ent.first.first; + unsigned OrigReg = SlotToOrigReg[Slot]; + VNInfo *OrigVNI = ent.first.second; + SmallPtrSet &EqValSpills = ent.second; + + DEBUG({ + dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" + << "Equal spills in BB: "; + for (const auto spill : EqValSpills) + dbgs() << spill->getParent()->getNumber() << " "; + dbgs() << "\n"; + }); + + // SpillsToRm is the spill set to be removed from EqValSpills. + SmallPtrSet SpillsToRm; + // SpillsToIns is the spill set to be newly inserted after hoisting. + DenseMap SpillsToIns; + + // First, clean up trivially redundent spills, .i.e, one spill is dominated + // by another in EqValSpills. + SmallPtrSetIterator SI = EqValSpills.begin(); + SmallPtrSetIterator SJ = SI; + for (; SI != EqValSpills.end(); SI++) { + for (SJ = SI, SJ++; SJ != EqValSpills.end(); SJ++) { + MachineBasicBlock *SIBB = (*SI)->getParent(); + MachineBasicBlock *SJBB = (*SJ)->getParent(); + if (MDT.properlyDominates(SIBB, SJBB)) + SpillsToRm.insert(*SJ); + else if (MDT.properlyDominates(SJBB, SIBB)) + SpillsToRm.insert(*SI); + else if (SIBB == SJBB) { + SlotIndex IdxI = LIS.getInstructionIndex(*SI); + SlotIndex IdxJ = LIS.getInstructionIndex(*SJ); + SpillsToRm.insert((IdxI > IdxJ) ? *SI : *SJ); + } + } + } + for (const auto Rspill : SpillsToRm) + EqValSpills.erase(Rspill); + + DEBUG({ + dbgs() << "Trivially redundent spills in BB: "; + for (const auto Rspill : SpillsToRm) + dbgs() << Rspill->getParent()->getNumber() << " "; + dbgs() << "\n"; + }); + + runHoistSpills(OrigReg, OrigVNI, EqValSpills, SpillsToRm, SpillsToIns); + + DEBUG({ + dbgs() << "Finally inserted spills in BB: "; + for (const auto Ispill : SpillsToIns) + dbgs() << Ispill.first->getNumber() << " "; + dbgs() << "\nFinally removed spills in BB: "; + for (const auto Rspill : SpillsToRm) + dbgs() << Rspill->getParent()->getNumber() << " "; + dbgs() << "\n"; + }); + + // Stack live range update. + LiveInterval &StackIntvl = LSS.getInterval(Slot); + if (!SpillsToIns.empty() || !SpillsToRm.empty()) { + LiveInterval &OrigLI = LIS.getInterval(OrigReg); + StackIntvl.MergeValueInAsValue(OrigLI, OrigVNI, + StackIntvl.getValNumInfo(0)); + } + + // Insert hoisted spills. + for (auto const ent : SpillsToIns) { + MachineBasicBlock *BB = ent.first; + unsigned LiveReg = ent.second; + MachineBasicBlock::iterator MI = BB->getFirstTerminator(); + TII.storeRegToStackSlot(*BB, MI, LiveReg, false, Slot, + MRI.getRegClass(LiveReg), &TRI); + LIS.InsertMachineInstrRangeInMaps(std::prev(MI), MI); + ++NumSpills; + } + + // Remove redundent spills. + for (auto const ent : SpillsToRm) { + LIS.RemoveMachineInstrFromMaps(ent); + ent->eraseFromParent(); + --NumSpills; + } + } +} Index: lib/CodeGen/LiveRangeEdit.cpp =================================================================== --- lib/CodeGen/LiveRangeEdit.cpp +++ lib/CodeGen/LiveRangeEdit.cpp @@ -35,6 +35,7 @@ unsigned VReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg)); if (VRM) { VRM->setIsSplitFromReg(VReg, VRM->getOriginal(OldReg)); + VRM->setIsSiblingsOfReg(VRM->getOriginal(OldReg), VReg); } LiveInterval &LI = LIS.createEmptyInterval(VReg); return LI; @@ -44,6 +45,7 @@ unsigned VReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg)); if (VRM) { VRM->setIsSplitFromReg(VReg, VRM->getOriginal(OldReg)); + VRM->setIsSiblingsOfReg(VRM->getOriginal(OldReg), VReg); } return VReg; } Index: lib/CodeGen/RegAllocGreedy.cpp =================================================================== --- lib/CodeGen/RegAllocGreedy.cpp +++ lib/CodeGen/RegAllocGreedy.cpp @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/Passes.h" #include "AllocationOrder.h" #include "InterferenceCache.h" #include "LiveDebugVariables.h" @@ -33,6 +32,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/VirtRegMap.h" @@ -44,6 +44,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" #include @@ -2601,6 +2602,8 @@ allocatePhysRegs(); tryHintsRecoloring(); + spiller().hoistAllSpills(); + releaseMemory(); return true; } Index: lib/CodeGen/Spiller.h =================================================================== --- lib/CodeGen/Spiller.h +++ lib/CodeGen/Spiller.h @@ -29,6 +29,9 @@ /// spill - Spill the LRE.getParent() live interval. virtual void spill(LiveRangeEdit &LRE) = 0; + /// hoistAllSpills - Hoist Spills to place them in less frequent executed BB + /// and remove redundent spills. + virtual void hoistAllSpills() = 0; }; /// Create and return a spiller that will insert spill code directly instead Index: lib/CodeGen/VirtRegMap.cpp =================================================================== --- lib/CodeGen/VirtRegMap.cpp +++ lib/CodeGen/VirtRegMap.cpp @@ -62,6 +62,7 @@ Virt2PhysMap.clear(); Virt2StackSlotMap.clear(); Virt2SplitMap.clear(); + Virt2SiblingsMap.clear(); grow(); return false; @@ -72,6 +73,7 @@ Virt2PhysMap.resize(NumRegs); Virt2StackSlotMap.resize(NumRegs); Virt2SplitMap.resize(NumRegs); + Virt2SiblingsMap.resize(NumRegs); } unsigned VirtRegMap::createSpillSlot(const TargetRegisterClass *RC) {