Index: llvm/lib/CodeGen/LiveDebugValues.cpp =================================================================== --- llvm/lib/CodeGen/LiveDebugValues.cpp +++ llvm/lib/CodeGen/LiveDebugValues.cpp @@ -112,6 +112,32 @@ using DefinedRegsSet = SmallSet; +/// A type-checked pair of {Register Location (or 0), Index}, used to index +/// into a \ref VarLocMap. This can be efficiently converted to a 64-bit int +/// for insertion into a \ref VarLocSet, and efficiently converted back. The +/// type-checker helps ensure that the conversions aren't lossy. +/// +/// Why encode a location /into/ the VarLocMap index? This makes it possible +/// to find the open VarLocs killed by a register def very quickly. This is a +/// performance-critical operation for LiveDebugValues. +struct LocIndex { + uint32_t Location; // Physical registers live in the range [1;2^30) (see + // \ref MCRegister), so we have plenty of range left here + // to encode non-register locations. + uint32_t Index; + + LocIndex(uint32_t Location, uint32_t Index) + : Location(Location), Index(Index) {} + + uint64_t getAsRawInteger() const { + return (static_cast(Location) << 32) | Index; + } + + static LocIndex fromRawInteger(uint64_t ID) { + return {static_cast(ID >> 32), static_cast(ID)}; + } +}; + class LiveDebugValues : public MachineFunctionPass { private: const TargetRegisterInfo *TRI; @@ -407,12 +433,23 @@ } }; - using VarLocMap = UniqueVector; + class VarLocMap { + UniqueVector VarLocs; + + public: + LocIndex insert(const VarLoc &VL) { + uint32_t Index = VarLocs.insert(VL); + return {0, Index}; + } + + const VarLoc &operator[](LocIndex ID) const { return VarLocs[ID.Index]; } + }; + using VarLocSet = SparseBitVector<>; using VarLocInMBB = SmallDenseMap; struct TransferDebugPair { MachineInstr *TransferInst; /// Instruction where this transfer occurs. - unsigned LocationID; /// Location number for the transfer dest. + LocIndex LocationID; /// Location number for the transfer dest. }; using TransferMap = SmallVector; @@ -441,9 +478,9 @@ class OpenRangesSet { VarLocSet VarLocs; // Map the DebugVariable to recent primary location ID. - SmallDenseMap Vars; + SmallDenseMap Vars; // Map the DebugVariable to recent backup location ID. - SmallDenseMap EntryValuesBackupVars; + SmallDenseMap EntryValuesBackupVars; OverlapMap &OverlappingFragments; public: @@ -459,17 +496,18 @@ void erase(const VarLocSet &KillSet, const VarLocMap &VarLocIDs); /// Insert a new range into the set. - void insert(unsigned VarLocID, const VarLoc &VL); + void insert(LocIndex VarLocID, const VarLoc &VL); /// Insert a set of ranges. void insertFromLocSet(const VarLocSet &ToLoad, const VarLocMap &Map) { - for (unsigned Id : ToLoad) { - const VarLoc &VarL = Map[Id]; - insert(Id, VarL); + for (uint64_t ID : ToLoad) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VarL = Map[Idx]; + insert(Idx, VarL); } } - llvm::Optional getEntryValueBackup(DebugVariable Var); + llvm::Optional getEntryValueBackup(DebugVariable Var); /// Empty the set. void clear() { @@ -517,7 +555,7 @@ VarLoc::SpillLoc extractSpillBaseRegAndOffset(const MachineInstr &MI); void insertTransferDebugPair(MachineInstr &MI, OpenRangesSet &OpenRanges, TransferMap &Transfers, VarLocMap &VarLocIDs, - unsigned OldVarID, TransferKind Kind, + LocIndex OldVarID, TransferKind Kind, unsigned NewReg = 0); void transferDebugValue(const MachineInstr &MI, OpenRangesSet &OpenRanges, @@ -617,8 +655,8 @@ auto *EraseFrom = VL.isEntryBackupLoc() ? &EntryValuesBackupVars : &Vars; auto It = EraseFrom->find(VarToErase); if (It != EraseFrom->end()) { - unsigned ID = It->second; - VarLocs.reset(ID); + LocIndex ID = It->second; + VarLocs.reset(ID.getAsRawInteger()); EraseFrom->erase(It); } }; @@ -648,23 +686,23 @@ void LiveDebugValues::OpenRangesSet::erase(const VarLocSet &KillSet, const VarLocMap &VarLocIDs) { VarLocs.intersectWithComplement(KillSet); - for (unsigned ID : KillSet) { - const VarLoc *VL = &VarLocIDs[ID]; + for (uint64_t ID : KillSet) { + const VarLoc *VL = &VarLocIDs[LocIndex::fromRawInteger(ID)]; auto *EraseFrom = VL->isEntryBackupLoc() ? &EntryValuesBackupVars : &Vars; EraseFrom->erase(VL->Var); } } -void LiveDebugValues::OpenRangesSet::insert(unsigned VarLocID, +void LiveDebugValues::OpenRangesSet::insert(LocIndex VarLocID, const VarLoc &VL) { auto *InsertInto = VL.isEntryBackupLoc() ? &EntryValuesBackupVars : &Vars; - VarLocs.set(VarLocID); + VarLocs.set(VarLocID.getAsRawInteger()); InsertInto->insert({VL.Var, VarLocID}); } /// Return the Loc ID of an entry value backup location, if it exists for the /// variable. -llvm::Optional +llvm::Optional LiveDebugValues::OpenRangesSet::getEntryValueBackup(DebugVariable Var) { auto It = EntryValuesBackupVars.find(Var); if (It != EntryValuesBackupVars.end()) @@ -689,8 +727,8 @@ if (L.empty()) continue; Out << "MBB: " << BB.getNumber() << ":\n"; - for (unsigned VLL : L) { - const VarLoc &VL = VarLocIDs[VLL]; + for (uint64_t VLL : L) { + const VarLoc &VL = VarLocIDs[LocIndex::fromRawInteger(VLL)]; Out << " Var: " << VL.Var.getVariable()->getName(); Out << " MI: "; VL.dump(TRI, Out); @@ -757,8 +795,8 @@ } if (TrySalvageEntryValue) { - for (unsigned ID : OpenRanges.getVarLocs()) { - const VarLoc &VL = VarLocIDs[ID]; + for (uint64_t ID : OpenRanges.getVarLocs()) { + const VarLoc &VL = VarLocIDs[LocIndex::fromRawInteger(ID)]; if (!VL.isEntryBackupLoc()) continue; @@ -801,7 +839,6 @@ } } - unsigned ID; if (isDbgValueDescribedByReg(MI) || MI.getOperand(0).isImm() || MI.getOperand(0).isFPImm() || MI.getOperand(0).isCImm()) { // Use normal VarLoc constructor for registers and immediates. @@ -809,7 +846,7 @@ // End all previous ranges of VL.Var. OpenRanges.erase(VL); - ID = VarLocIDs.insert(VL); + LocIndex ID = VarLocIDs.insert(VL); // Add the VarLoc to OpenRanges from this DBG_VALUE. OpenRanges.insert(ID, VL); } else if (MI.hasOneMemOperand()) { @@ -827,12 +864,15 @@ VarLocMap &VarLocIDs, TransferMap &Transfers, SparseBitVector<> &KillSet) { - for (unsigned ID : KillSet) { - if (!VarLocIDs[ID].Var.getVariable()->isParameter()) + for (uint64_t ID : KillSet) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; + if (!VL.Var.getVariable()->isParameter()) continue; - auto DebugVar = VarLocIDs[ID].Var; - auto EntryValBackupID = OpenRanges.getEntryValueBackup(DebugVar); + auto DebugVar = VL.Var; + Optional EntryValBackupID = + OpenRanges.getEntryValueBackup(DebugVar); // If the parameter has the entry value backup, it means we should // be able to use its entry value. @@ -842,7 +882,7 @@ const VarLoc &EntryVL = VarLocIDs[*EntryValBackupID]; VarLoc EntryLoc = VarLoc::CreateEntryLoc(EntryVL.MI, LS, EntryVL.Expr, EntryVL.Loc.RegNo); - unsigned EntryValueID = VarLocIDs.insert(EntryLoc); + LocIndex EntryValueID = VarLocIDs.insert(EntryLoc); Transfers.push_back({&MI, EntryValueID}); OpenRanges.insert(EntryValueID, EntryLoc); } @@ -855,12 +895,12 @@ /// otherwise it is variable's location on the stack. void LiveDebugValues::insertTransferDebugPair( MachineInstr &MI, OpenRangesSet &OpenRanges, TransferMap &Transfers, - VarLocMap &VarLocIDs, unsigned OldVarID, TransferKind Kind, + VarLocMap &VarLocIDs, LocIndex OldVarID, TransferKind Kind, unsigned NewReg) { const MachineInstr *DebugInstr = &VarLocIDs[OldVarID].MI; auto ProcessVarLoc = [&MI, &OpenRanges, &Transfers, &VarLocIDs](VarLoc &VL) { - unsigned LocId = VarLocIDs.insert(VL); + LocIndex LocId = VarLocIDs.insert(VL); // Close this variable's previous location range. OpenRanges.erase(VL); @@ -961,8 +1001,9 @@ } // For performance reasons, it's critical to iterate over the open var locs // at most once. - for (unsigned ID : OpenRanges.getVarLocs()) { - unsigned Reg = VarLocIDs[ID].isDescribedByReg(); + for (uint64_t ID : OpenRanges.getVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + unsigned Reg = VarLocIDs[Idx].isDescribedByReg(); if (!Reg) continue; @@ -1081,8 +1122,9 @@ VarLocSet KillSet; if (isSpillInstruction(MI, MF)) { Loc = extractSpillBaseRegAndOffset(MI); - for (unsigned ID : OpenRanges.getVarLocs()) { - const VarLoc &VL = VarLocIDs[ID]; + for (uint64_t ID : OpenRanges.getVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; if (VL.Kind == VarLoc::SpillLocKind && VL.Loc.SpillLocation == *Loc) { // This location is overwritten by the current instruction -- terminate // the open range, and insert an explicit DBG_VALUE $noreg. @@ -1096,7 +1138,7 @@ // where they are located; it's best to fix handle overwrites now. KillSet.set(ID); VarLoc UndefVL = VarLoc::CreateCopyLoc(VL.MI, LS, 0); - unsigned UndefLocID = VarLocIDs.insert(UndefVL); + LocIndex UndefLocID = VarLocIDs.insert(UndefVL); Transfers.push_back({&MI, UndefLocID}); } } @@ -1119,19 +1161,20 @@ << "\n"); } // Check if the register or spill location is the location of a debug value. - for (unsigned ID : OpenRanges.getVarLocs()) { - if (TKind == TransferKind::TransferSpill && - VarLocIDs[ID].isDescribedByReg() == Reg) { + for (uint64_t ID : OpenRanges.getVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; + if (TKind == TransferKind::TransferSpill && VL.isDescribedByReg() == Reg) { LLVM_DEBUG(dbgs() << "Spilling Register " << printReg(Reg, TRI) << '(' - << VarLocIDs[ID].Var.getVariable()->getName() << ")\n"); + << VL.Var.getVariable()->getName() << ")\n"); } else if (TKind == TransferKind::TransferRestore && - VarLocIDs[ID].Kind == VarLoc::SpillLocKind && - VarLocIDs[ID].Loc.SpillLocation == *Loc) { + VL.Kind == VarLoc::SpillLocKind && + VL.Loc.SpillLocation == *Loc) { LLVM_DEBUG(dbgs() << "Restoring Register " << printReg(Reg, TRI) << '(' - << VarLocIDs[ID].Var.getVariable()->getName() << ")\n"); + << VL.Var.getVariable()->getName() << ")\n"); } else continue; - insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID, TKind, + insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, Idx, TKind, Reg); return; } @@ -1176,17 +1219,19 @@ // a parameter describing only a moving of the value around, rather then // modifying it, we are still able to use the entry value if needed. if (isRegOtherThanSPAndFP(*DestRegOp, MI, TRI)) { - for (unsigned ID : OpenRanges.getVarLocs()) { - if (VarLocIDs[ID].getEntryValueBackupReg() == SrcReg) { + for (uint64_t ID : OpenRanges.getVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + const VarLoc &VL = VarLocIDs[Idx]; + if (VL.getEntryValueBackupReg() == SrcReg) { LLVM_DEBUG(dbgs() << "Copy of the entry value: "; MI.dump();); - VarLoc EntryValLocCopyBackup = VarLoc::CreateEntryCopyBackupLoc( - VarLocIDs[ID].MI, LS, VarLocIDs[ID].Expr, DestReg); + VarLoc EntryValLocCopyBackup = + VarLoc::CreateEntryCopyBackupLoc(VL.MI, LS, VL.Expr, DestReg); // Stop tracking the original entry value. - OpenRanges.erase(VarLocIDs[ID]); + OpenRanges.erase(VL); // Start tracking the entry value copy. - unsigned EntryValCopyLocID = VarLocIDs.insert(EntryValLocCopyBackup); + LocIndex EntryValCopyLocID = VarLocIDs.insert(EntryValLocCopyBackup); OpenRanges.insert(EntryValCopyLocID, EntryValLocCopyBackup); break; } @@ -1196,9 +1241,10 @@ if (!SrcRegOp->isKill()) return; - for (unsigned ID : OpenRanges.getVarLocs()) { - if (VarLocIDs[ID].isDescribedByReg() == SrcReg) { - insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID, + for (uint64_t ID : OpenRanges.getVarLocs()) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + if (VarLocIDs[Idx].isDescribedByReg() == SrcReg) { + insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, Idx, TransferKind::TransferCopy, DestReg); return; } @@ -1212,11 +1258,11 @@ const VarLocMap &VarLocIDs) { bool Changed = false; - LLVM_DEBUG(for (unsigned ID + LLVM_DEBUG(for (uint64_t ID : OpenRanges.getVarLocs()) { // Copy OpenRanges to OutLocs, if not already present. dbgs() << "Add to OutLocs in MBB #" << CurMBB->getNumber() << ": "; - VarLocIDs[ID].dump(TRI); + VarLocIDs[LocIndex::fromRawInteger(ID)].dump(TRI); }); VarLocSet &VLS = OutLocs[CurMBB]; Changed = VLS != OpenRanges.getVarLocs(); @@ -1340,9 +1386,12 @@ LLVM_DEBUG({ if (!InLocsT.empty()) { - for (auto ID : InLocsT) + for (uint64_t ID : InLocsT) dbgs() << " gathered candidate incoming var: " - << VarLocIDs[ID].Var.getVariable()->getName() << "\n"; + << VarLocIDs[LocIndex::fromRawInteger(ID)] + .Var.getVariable() + ->getName() + << "\n"; } }); @@ -1353,11 +1402,12 @@ VarLocSet KillSet; bool IsArtificial = ArtificialBlocks.count(&MBB); if (!IsArtificial) { - for (auto ID : InLocsT) { - if (!VarLocIDs[ID].dominates(MBB)) { + for (uint64_t ID : InLocsT) { + LocIndex Idx = LocIndex::fromRawInteger(ID); + if (!VarLocIDs[Idx].dominates(MBB)) { KillSet.set(ID); LLVM_DEBUG({ - auto Name = VarLocIDs[ID].Var.getVariable()->getName(); + auto Name = VarLocIDs[Idx].Var.getVariable()->getName(); dbgs() << " killing " << Name << ", it doesn't dominate MBB\n"; }); } @@ -1379,24 +1429,22 @@ // that we need to perform in the Pending set. VarLocSet Diff = InLocsT; Diff.intersectWithComplement(ILS); - for (auto ID : Diff) { - Pending.set(ID); - ILS.set(ID); - ++NumInserted; - Changed = true; - } + Pending |= Diff; + ILS |= Diff; + unsigned JustInserted = Diff.count(); + NumInserted += JustInserted; + Changed = JustInserted > 0; // We may have lost locations by learning about a predecessor that either // loses or moves a variable. Find any locations in ILS that are not in the // new in-locations, and delete those. VarLocSet Removed = ILS; Removed.intersectWithComplement(InLocsT); - for (auto ID : Removed) { - Pending.reset(ID); - ILS.reset(ID); - ++NumRemoved; - Changed = true; - } + Pending.intersectWithComplement(Removed); + ILS.intersectWithComplement(Removed); + unsigned JustRemoved = Removed.count(); + NumRemoved += JustRemoved; + Changed |= JustRemoved > 0; return Changed; } @@ -1410,10 +1458,10 @@ auto &MBB = const_cast(*Iter.first); VarLocSet &Pending = Iter.second; - for (unsigned ID : Pending) { + for (uint64_t ID : Pending) { // The ID location is live-in to MBB -- work out what kind of machine // location it is and create a DBG_VALUE. - const VarLoc &DiffIt = VarLocIDs[ID]; + const VarLoc &DiffIt = VarLocIDs[LocIndex::fromRawInteger(ID)]; if (DiffIt.isEntryBackupLoc()) continue; MachineInstr *MI = DiffIt.BuildDbgValue(*MBB.getParent()); @@ -1502,7 +1550,7 @@ DIExpression *NewExpr = DIExpression::prepend(MI.getDebugExpression(), DIExpression::EntryValue); VarLoc EntryValLocAsBackup = VarLoc::CreateEntryBackupLoc(MI, LS, NewExpr); - unsigned EntryValLocID = VarLocIDs.insert(EntryValLocAsBackup); + LocIndex EntryValLocID = VarLocIDs.insert(EntryValLocAsBackup); OpenRanges.insert(EntryValLocID, EntryValLocAsBackup); }