Index: include/llvm/CodeGen/LiveRangeEdit.h =================================================================== --- include/llvm/CodeGen/LiveRangeEdit.h +++ include/llvm/CodeGen/LiveRangeEdit.h @@ -60,6 +60,7 @@ private: LiveInterval *Parent; SmallVectorImpl &NewRegs; + SmallPtrSet *DeadRemats; MachineRegisterInfo &MRI; LiveIntervals &LIS; VirtRegMap *VRM; @@ -111,18 +112,21 @@ /// @param parent The register being spilled or split. /// @param newRegs List to receive any new registers created. This needn't be /// empty initially, any existing registers are ignored. + /// @param deadRemats The collection of all the instructions defining an + /// original reg and are dead after remat. /// @param MF The MachineFunction the live range edit is taking place in. /// @param lis The collection of all live intervals in this function. /// @param vrm Map of virtual registers to physical registers for this /// function. If NULL, no virtual register map updates will /// be done. This could be the case if called before Regalloc. LiveRangeEdit(LiveInterval *parent, SmallVectorImpl &newRegs, + SmallPtrSet *deadRemats, MachineFunction &MF, LiveIntervals &lis, VirtRegMap *vrm, Delegate *delegate = nullptr) - : Parent(parent), NewRegs(newRegs), MRI(MF.getRegInfo()), LIS(lis), - VRM(vrm), TII(*MF.getSubtarget().getInstrInfo()), - TheDelegate(delegate), FirstNew(newRegs.size()), - ScannedRemattable(false) { + : Parent(parent), NewRegs(newRegs), DeadRemats(deadRemats), + MRI(MF.getRegInfo()), LIS(lis), VRM(vrm), + TII(*MF.getSubtarget().getInstrInfo()), TheDelegate(delegate), + FirstNew(newRegs.size()), ScannedRemattable(false) { MRI.setDelegate(this); } @@ -141,6 +145,7 @@ unsigned size() const { return NewRegs.size()-FirstNew; } bool empty() const { return size() == 0; } unsigned get(unsigned idx) const { return NewRegs[idx+FirstNew]; } + void pop_back() { NewRegs.pop_back(); } ArrayRef regs() const { return makeArrayRef(NewRegs).slice(FirstNew); @@ -208,6 +213,8 @@ return Rematted.count(ParentVNI); } + void markDeadRemat(MachineInstr *inst) { DeadRemats->insert(inst); } + /// eraseVirtReg - Notify the delegate that Reg is no longer in use, and try /// to erase it from LIS. void eraseVirtReg(unsigned Reg); Index: lib/CodeGen/InlineSpiller.cpp =================================================================== --- lib/CodeGen/InlineSpiller.cpp +++ lib/CodeGen/InlineSpiller.cpp @@ -881,11 +881,11 @@ if (SnippetCopies.count(MI)) return false; - // Use an OrigVNI from traceSiblingValue when ParentVNI is a sibling copy. LiveRangeEdit::Remat RM(ParentVNI); - SibValueMap::const_iterator SibI = SibValues.find(ParentVNI); - if (SibI != SibValues.end()) - RM.OrigMI = SibI->second.DefMI; + LiveInterval &OrigLI = LIS.getInterval(Original); + VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx); + RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def); + if (!Edit->canRematerializeAt(RM, UseIdx, false)) { markValueUsed(&VirtReg, ParentVNI); DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << *MI); Index: lib/CodeGen/LiveRangeEdit.cpp =================================================================== --- lib/CodeGen/LiveRangeEdit.cpp +++ lib/CodeGen/LiveRangeEdit.cpp @@ -64,8 +64,14 @@ if (VNI->isUnused()) continue; MachineInstr *DefMI = LIS.getInstructionFromIndex(VNI->def); - if (!DefMI) - continue; + if (!DefMI) { + unsigned Original = VRM->getOriginal(getReg()); + LiveInterval &OrigLI = LIS.getInterval(Original); + VNInfo *OrigVNI = OrigLI.getVNInfoAt(VNI->def); + DefMI = LIS.getInstructionFromIndex(OrigVNI->def); + if (!DefMI) + continue; + } checkRematerializable(VNI, DefMI, aa); } ScannedRemattable = true; @@ -261,6 +267,15 @@ // Collect virtual registers to be erased after MI is gone. SmallVector RegsToErase; bool ReadsPhysRegs = false; + bool isOrigDef = false; + unsigned Dest; + if (VRM && MI->getOperand(0).isReg()) { + Dest = MI->getOperand(0).getReg(); + unsigned Original = VRM->getOriginal(Dest); + LiveInterval &OrigLI = LIS.getInterval(Original); + VNInfo *OrigVNI = OrigLI.getVNInfoAt(Idx); + isOrigDef = SlotIndex::isSameInstr(OrigVNI->def, Idx); + } // Check for live intervals that may shrink for (MachineInstr::mop_iterator MOI = MI->operands_begin(), @@ -314,11 +329,24 @@ } DEBUG(dbgs() << "Converted physregs to:\t" << *MI); } else { - if (TheDelegate) - TheDelegate->LRE_WillEraseInstruction(MI); - LIS.RemoveMachineInstrFromMaps(MI); - MI->eraseFromParent(); - ++NumDCEDeleted; + // If the dest of MI is an original reg, don't delete the inst. Replace + // the dest with a new reg, keep the inst for remat of other siblings. + // The inst is saved in LiveRangeEdit::DeadRemats and will be deleted + // after all the allocations of the func are done. + if (isOrigDef) { + unsigned NewDest = createFrom(Dest); + pop_back(); + markDeadRemat(MI); + const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + MI->substituteRegister(Dest, NewDest, 0, TRI); + MI->getOperand(0).setIsDead(false); + } else { + if (TheDelegate) + TheDelegate->LRE_WillEraseInstruction(MI); + LIS.RemoveMachineInstrFromMaps(MI); + MI->eraseFromParent(); + ++NumDCEDeleted; + } } // Erase any virtregs that are now empty and unused. There may be Index: lib/CodeGen/RegAllocBasic.cpp =================================================================== --- lib/CodeGen/RegAllocBasic.cpp +++ lib/CodeGen/RegAllocBasic.cpp @@ -199,7 +199,7 @@ Matrix->unassign(Spill); // Spill the extracted interval. - LiveRangeEdit LRE(&Spill, SplitVRegs, *MF, *LIS, VRM); + LiveRangeEdit LRE(&Spill, SplitVRegs, nullptr, *MF, *LIS, VRM); spiller().spill(LRE); } return true; @@ -258,7 +258,7 @@ DEBUG(dbgs() << "spilling: " << VirtReg << '\n'); if (!VirtReg.isSpillable()) return ~0u; - LiveRangeEdit LRE(&VirtReg, SplitVRegs, *MF, *LIS, VRM); + LiveRangeEdit LRE(&VirtReg, SplitVRegs, nullptr, *MF, *LIS, VRM); spiller().spill(LRE); // The live virtual register requesting allocation was spilled, so tell Index: lib/CodeGen/RegAllocGreedy.cpp =================================================================== --- lib/CodeGen/RegAllocGreedy.cpp +++ lib/CodeGen/RegAllocGreedy.cpp @@ -312,6 +312,12 @@ /// Set of broken hints that may be reconciled later because of eviction. SmallSetVector SetOfBrokenHints; + /// Inst which is a def of an original reg and whose defs are already all + /// dead after remat is saved in DeadRemats. The deletion of such inst is + /// postponed till all the allocations are done, so its remat expr is + /// always available for the remat of all the siblings of the original reg. + SmallPtrSet DeadRemats; + public: RAGreedy(); @@ -1457,7 +1463,7 @@ SmallVectorImpl &NewVRegs) { SmallVector UsedCands; // Prepare split editor. - LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this); + LiveRangeEdit LREdit(&VirtReg, NewVRegs, &DeadRemats, *MF, *LIS, VRM, this); SE->reset(LREdit, SplitSpillMode); // Assign all edge bundles to the preferred candidate, or NoCand. @@ -1505,7 +1511,7 @@ assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed"); unsigned Reg = VirtReg.reg; bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)); - LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this); + LiveRangeEdit LREdit(&VirtReg, NewVRegs, &DeadRemats, *MF, *LIS, VRM, this); SE->reset(LREdit, SplitSpillMode); ArrayRef UseBlocks = SA->getUseBlocks(); for (unsigned i = 0; i != UseBlocks.size(); ++i) { @@ -1577,7 +1583,7 @@ // Always enable split spill mode, since we're effectively spilling to a // register. - LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this); + LiveRangeEdit LREdit(&VirtReg, NewVRegs, &DeadRemats, *MF, *LIS, VRM, this); SE->reset(LREdit, SplitEditor::SM_Size); ArrayRef Uses = SA->getUseSlots(); @@ -1900,7 +1906,7 @@ << '-' << Uses[BestAfter] << ", " << BestDiff << ", " << (BestAfter - BestBefore + 1) << " instrs\n"); - LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this); + LiveRangeEdit LREdit(&VirtReg, NewVRegs, &DeadRemats, *MF, *LIS, VRM, this); SE->reset(LREdit); SE->openIntv(); @@ -2543,7 +2549,7 @@ NewVRegs.push_back(VirtReg.reg); } else { NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled); - LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this); + LiveRangeEdit LRE(&VirtReg, NewVRegs, &DeadRemats, *MF, *LIS, VRM, this); spiller().spill(LRE); setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done); @@ -2598,9 +2604,16 @@ IntfCache.init(MF, Matrix->getLiveUnions(), Indexes, LIS, TRI); GlobalCand.resize(32); // This will grow as needed. SetOfBrokenHints.clear(); + DeadRemats.clear(); allocatePhysRegs(); tryHintsRecoloring(); + + for (auto ent : DeadRemats) { + LIS->RemoveMachineInstrFromMaps(ent); + ent->eraseFromParent(); + } + releaseMemory(); return true; } Index: lib/CodeGen/RegAllocPBQP.cpp =================================================================== --- lib/CodeGen/RegAllocPBQP.cpp +++ lib/CodeGen/RegAllocPBQP.cpp @@ -631,7 +631,8 @@ VirtRegMap &VRM, Spiller &VRegSpiller) { VRegsToAlloc.erase(VReg); - LiveRangeEdit LRE(&LIS.getInterval(VReg), NewIntervals, MF, LIS, &VRM); + LiveRangeEdit LRE(&LIS.getInterval(VReg), NewIntervals, nullptr, MF, LIS, + &VRM); VRegSpiller.spill(LRE); const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); Index: lib/CodeGen/RegisterCoalescer.cpp =================================================================== --- lib/CodeGen/RegisterCoalescer.cpp +++ lib/CodeGen/RegisterCoalescer.cpp @@ -449,8 +449,8 @@ void RegisterCoalescer::eliminateDeadDefs() { SmallVector NewRegs; - LiveRangeEdit(nullptr, NewRegs, *MF, *LIS, - nullptr, this).eliminateDeadDefs(DeadDefs); + LiveRangeEdit(nullptr, NewRegs, nullptr, *MF, *LIS, nullptr, this) + .eliminateDeadDefs(DeadDefs); } void RegisterCoalescer::LRE_WillEraseInstruction(MachineInstr *MI) { Index: lib/CodeGen/SplitKit.cpp =================================================================== --- lib/CodeGen/SplitKit.cpp +++ lib/CodeGen/SplitKit.cpp @@ -431,6 +431,11 @@ // Attempt cheap-as-a-copy rematerialization. LiveRangeEdit::Remat RM(ParentVNI); + unsigned Original = VRM.getOriginal(Edit->get(RegIdx)); + LiveInterval &OrigLI = LIS.getInterval(Original); + VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx); + RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def); + if (Edit->canRematerializeAt(RM, UseIdx, true)) { Def = Edit->rematerializeAt(MBB, I, LI->reg, RM, TRI, Late); ++NumRemats; @@ -1003,6 +1008,8 @@ // Dead defs end at the dead slot. if (S.end != S.valno->def.getDeadSlot()) continue; + if (S.valno->isPHIDef()) + continue; MachineInstr *MI = LIS.getInstructionFromIndex(S.valno->def); assert(MI && "Missing instruction for dead def"); MI->addRegisterDead(LI->reg, &TRI);