diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -16,6 +16,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" @@ -464,9 +465,10 @@ MI.getDebugLoc()->getInlinedAt()); bool SeenBefore = SeenDbgVars.contains(Var); - MachineOperand &MO = MI.getDebugOperand(0); - if (MO.isReg() && MO.getReg().isVirtual()) - SeenDbgUsers[MO.getReg()].push_back(SeenDbgUser(&MI, SeenBefore)); + for (MachineOperand &MO : MI.debug_operands()) { + if (MO.isReg() && MO.getReg().isVirtual()) + SeenDbgUsers[MO.getReg()].push_back(SeenDbgUser(&MI, SeenBefore)); + } // Record the variable for any DBG_VALUE, to avoid re-ordering any of them. SeenDbgVars.insert(Var); @@ -926,14 +928,14 @@ /// leaving an 'undef' DBG_VALUE in the original location. Don't do this if /// there's any subregister weirdness involved. Returns true if copy /// propagation occurred. -static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI) { +static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI, + Register Reg) { const MachineRegisterInfo &MRI = SinkInst.getMF()->getRegInfo(); const TargetInstrInfo &TII = *SinkInst.getMF()->getSubtarget().getInstrInfo(); // Copy DBG_VALUE operand and set the original to undef. We then check to // see whether this is something that can be copy-forwarded. If it isn't, // continue around the loop. - MachineOperand &DbgMO = DbgMI.getDebugOperand(0); const MachineOperand *SrcMO = nullptr, *DstMO = nullptr; auto CopyOperands = TII.isCopyInstr(SinkInst); @@ -946,36 +948,41 @@ bool PostRA = MRI.getNumVirtRegs() == 0; // Trying to forward between physical and virtual registers is too hard. - if (DbgMO.getReg().isVirtual() != SrcMO->getReg().isVirtual()) + if (Reg.isVirtual() != SrcMO->getReg().isVirtual()) return false; // Only try virtual register copy-forwarding before regalloc, and physical // register copy-forwarding after regalloc. - bool arePhysRegs = !DbgMO.getReg().isVirtual(); + bool arePhysRegs = !Reg.isVirtual(); if (arePhysRegs != PostRA) return false; // Pre-regalloc, only forward if all subregisters agree (or there are no // subregs at all). More analysis might recover some forwardable copies. - if (!PostRA && (DbgMO.getSubReg() != SrcMO->getSubReg() || - DbgMO.getSubReg() != DstMO->getSubReg())) - return false; + if (!PostRA) + for (auto &DbgMO : DbgMI.getDebugOperandsForReg(Reg)) + if (DbgMO.getSubReg() != SrcMO->getSubReg() || + DbgMO.getSubReg() != DstMO->getSubReg()) + return false; // Post-regalloc, we may be sinking a DBG_VALUE of a sub or super-register // of this copy. Only forward the copy if the DBG_VALUE operand exactly // matches the copy destination. - if (PostRA && DbgMO.getReg() != DstMO->getReg()) + if (PostRA && Reg != DstMO->getReg()) return false; - DbgMO.setReg(SrcMO->getReg()); - DbgMO.setSubReg(SrcMO->getSubReg()); + for (auto &DbgMO : DbgMI.getDebugOperandsForReg(Reg)) { + DbgMO.setReg(SrcMO->getReg()); + DbgMO.setSubReg(SrcMO->getSubReg()); + } return true; } +using MIRegs = std::pair>; /// Sink an instruction and its associated debug instructions. static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo, MachineBasicBlock::iterator InsertPos, - SmallVectorImpl &DbgValuesToSink) { + SmallVectorImpl &DbgValuesToSink) { // If we cannot find a location to use (merge with), then we erase the debug // location to prevent debug-info driven tools from potentially reporting @@ -995,14 +1002,21 @@ // DBG_VALUE location as 'undef', indicating that any earlier variable // location should be terminated as we've optimised away the value at this // point. - for (SmallVectorImpl::iterator DBI = DbgValuesToSink.begin(), - DBE = DbgValuesToSink.end(); - DBI != DBE; ++DBI) { - MachineInstr *DbgMI = *DBI; - MachineInstr *NewDbgMI = DbgMI->getMF()->CloneMachineInstr(*DBI); + for (auto DbgValueToSink : DbgValuesToSink) { + MachineInstr *DbgMI = DbgValueToSink.first; + MachineInstr *NewDbgMI = DbgMI->getMF()->CloneMachineInstr(DbgMI); SuccToSinkTo.insert(InsertPos, NewDbgMI); - if (!attemptDebugCopyProp(MI, *DbgMI)) + bool PropagatedAllSunkOps = true; + for (unsigned Reg : DbgValueToSink.second) { + if (DbgMI->hasDebugOperandForReg(Reg)) { + if (!attemptDebugCopyProp(MI, *DbgMI, Reg)) { + PropagatedAllSunkOps = false; + break; + } + } + } + if (!PropagatedAllSunkOps) DbgMI->setDebugValueUndef(); } } @@ -1214,7 +1228,7 @@ ++InsertPos; // Collect debug users of any vreg that this inst defines. - SmallVector DbgUsersToSink; + SmallVector DbgUsersToSink; for (auto &MO : MI.operands()) { if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual()) continue; @@ -1228,10 +1242,11 @@ if (User.getInt()) { // This DBG_VALUE would re-order assignments. If we can't copy-propagate // it, it can't be recovered. Set it undef. - if (!attemptDebugCopyProp(MI, *DbgMI)) + if (!attemptDebugCopyProp(MI, *DbgMI, MO.getReg())) DbgMI->setDebugValueUndef(); } else { - DbgUsersToSink.push_back(DbgMI); + DbgUsersToSink.push_back( + {DbgMI, SmallVector(1, MO.getReg())}); } } } @@ -1266,10 +1281,12 @@ // be sunk. For the rest, if they are not dominated by the block we will sink // MI into, propagate the copy source to them. SmallVector DbgDefUsers; + SmallVector DbgUseRegs; const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); for (auto &MO : MI.operands()) { if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual()) continue; + DbgUseRegs.push_back(MO.getReg()); for (auto &User : MRI.use_instructions(MO.getReg())) { if (!User.isDebugValue() || DT->dominates(TargetBlock, User.getParent())) continue; @@ -1278,8 +1295,8 @@ if (User.getParent() == MI.getParent()) continue; - assert(User.getDebugOperand(0).isReg() && - "DBG_VALUE user of vreg, but non reg operand?"); + assert(User.hasDebugOperandForReg(MO.getReg()) && + "DBG_VALUE user of vreg, but has no operand for it?"); DbgDefUsers.push_back(&User); } } @@ -1287,8 +1304,12 @@ // Point the users of this copy that are no longer dominated, at the source // of the copy. for (auto *User : DbgDefUsers) { - User->getDebugOperand(0).setReg(MI.getOperand(1).getReg()); - User->getDebugOperand(0).setSubReg(MI.getOperand(1).getSubReg()); + for (auto &Reg : DbgUseRegs) { + for (auto &DbgOp : User->getDebugOperandsForReg(Reg)) { + DbgOp.setReg(MI.getOperand(1).getReg()); + DbgOp.setSubReg(MI.getOperand(1).getSubReg()); + } + } } } @@ -1351,8 +1372,10 @@ LiveRegUnits ModifiedRegUnits, UsedRegUnits; /// Track DBG_VALUEs of (unmodified) register units. Each DBG_VALUE has an - /// entry in this map for each unit it touches. - DenseMap> SeenDbgInstrs; + /// entry in this map for each unit it touches. The DBG_VALUE's entry + /// consists of a pointer to the instruction itself, and a vector of registers + /// referred to by the instruction that overlap the key register unit. + DenseMap> SeenDbgInstrs; /// Sink Copy instructions unused in the same block close to their uses in /// successors. @@ -1534,18 +1557,27 @@ // We must sink this DBG_VALUE if its operand is sunk. To avoid searching // for DBG_VALUEs later, record them when they're encountered. if (MI->isDebugValue()) { - auto &MO = MI->getDebugOperand(0); - if (MO.isReg() && Register::isPhysicalRegister(MO.getReg())) { - // Bail if we can already tell the sink would be rejected, rather - // than needlessly accumulating lots of DBG_VALUEs. - if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy, - ModifiedRegUnits, UsedRegUnits)) - continue; - - // Record debug use of each reg unit. - SmallSet Units = getRegUnits(MO.getReg(), TRI); - for (MCRegister Reg : Units) - SeenDbgInstrs[Reg].push_back(MI); + SmallDenseMap, 4> MIUnits; + bool IsValid = true; + for (MachineOperand &MO : MI->debug_operands()) { + if (MO.isReg() && Register::isPhysicalRegister(MO.getReg())) { + // Bail if we can already tell the sink would be rejected, rather + // than needlessly accumulating lots of DBG_VALUEs. + if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy, + ModifiedRegUnits, UsedRegUnits)) { + IsValid = false; + break; + } + + // Record debug use of each reg unit. + SmallSet RegUnits = getRegUnits(MO.getReg(), TRI); + for (MCRegister Reg : RegUnits) + MIUnits[Reg].push_back(MO.getReg()); + } + } + if (IsValid) { + for (auto RegOps : MIUnits) + SeenDbgInstrs[RegOps.first].push_back({MI, RegOps.second}); } continue; } @@ -1587,18 +1619,22 @@ // Collect DBG_VALUEs that must sink with this copy. We've previously // recorded which reg units that DBG_VALUEs read, if this instruction // writes any of those units then the corresponding DBG_VALUEs must sink. - SetVector DbgValsToSinkSet; + MapVector DbgValsToSinkMap; for (auto &MO : MI->operands()) { if (!MO.isReg() || !MO.isDef()) continue; SmallSet Units = getRegUnits(MO.getReg(), TRI); - for (MCRegister Reg : Units) - for (auto *MI : SeenDbgInstrs.lookup(Reg)) - DbgValsToSinkSet.insert(MI); + for (MCRegister Reg : Units) { + for (auto MIRegs : SeenDbgInstrs.lookup(Reg)) { + auto &Regs = DbgValsToSinkMap[MIRegs.first]; + for (unsigned Reg : MIRegs.second) + Regs.push_back(Reg); + } + } } - SmallVector DbgValsToSink(DbgValsToSinkSet.begin(), - DbgValsToSinkSet.end()); + SmallVector DbgValsToSink(DbgValsToSinkMap.begin(), + DbgValsToSinkMap.end()); // Clear the kill flag if SrcReg is killed between MI and the end of the // block. diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp --- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp +++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp @@ -185,7 +185,8 @@ break; if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter()) continue; - if (MI.getDebugOperand(0).isFI()) { + if (any_of(MI.debug_operands(), + [](const MachineOperand &MO) { return MO.isFI(); })) { // We can only emit valid locations for frame indices after the frame // setup, so do not stash away them. FrameIndexValues.push_back(&MI); @@ -1203,16 +1204,19 @@ // way with simply the frame index and offset rather than any // target-specific addressing mode. if (MI.isDebugValue()) { - assert(i == 0 && "Frame indices can only appear as the first " - "operand of a DBG_VALUE machine instruction"); + MachineOperand &Op = MI.getOperand(i); + assert( + MI.isDebugOperand(&Op) && + "Frame indices can only appear as a debug operand in a DBG_VALUE*" + " machine instruction"); Register Reg; - unsigned FrameIdx = MI.getOperand(0).getIndex(); + unsigned FrameIdx = Op.getIndex(); unsigned Size = MF.getFrameInfo().getObjectSize(FrameIdx); StackOffset Offset = TFI->getFrameIndexReference(MF, FrameIdx, Reg); - MI.getOperand(0).ChangeToRegister(Reg, false /*isDef*/); - MI.getOperand(0).setIsDebug(); + Op.ChangeToRegister(Reg, false /*isDef*/); + Op.setIsDebug(); const DIExpression *DIExpr = MI.getDebugExpression(); @@ -1243,7 +1247,7 @@ // The debug operand at DebugOpIndex was a frame index at offset // `Offset`; now the operand has been replaced with the frame // register, we must add Offset with `register x, plus Offset`. - unsigned DebugOpIndex = MI.getDebugOperandIndex(&MI.getOperand(i)); + unsigned DebugOpIndex = MI.getDebugOperandIndex(&Op); SmallVector Ops; TRI.getOffsetOpcodes(Offset, Ops); DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, DebugOpIndex); diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp --- a/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -3565,8 +3565,12 @@ // After collecting a block of DBG_VALUEs into ToInsert, enter them into the // vreg => DbgValueLoc map. auto CloseNewDVRange = [this, &ToInsert](SlotIndex Slot) { - for (auto *X : ToInsert) - DbgVRegToValues[X->getDebugOperand(0).getReg()].push_back({Slot, X}); + for (auto *X : ToInsert) { + for (auto Op : X->debug_operands()) { + if (Op.isReg() && Op.getReg().isVirtual()) + DbgVRegToValues[Op.getReg()].push_back({Slot, X}); + } + } ToInsert.clear(); }; @@ -3578,9 +3582,11 @@ SlotIndex CurrentSlot = Slots.getMBBStartIdx(&MBB); for (auto &MI : MBB) { - if (MI.isDebugValue() && MI.getDebugOperand(0).isReg() && - MI.getDebugOperand(0).getReg().isVirtual()) { - ToInsert.push_back(&MI); + if (MI.isDebugValue()) { + if (any_of(MI.debug_operands(), [](const MachineOperand &MO) { + return MO.isReg() && MO.getReg().isVirtual(); + })) + ToInsert.push_back(&MI); } else if (!MI.isDebugInstr()) { CurrentSlot = Slots.getInstructionIndex(MI); CloseNewDVRange(CurrentSlot); @@ -3677,12 +3683,14 @@ if (DbgValueSetIt->first < SegmentIt->end) { // "Other" is live and there is a DBG_VALUE of Reg: test if we should // set it undef. - if (DbgValueSetIt->first >= SegmentIt->start && - DbgValueSetIt->second->getDebugOperand(0).getReg() != 0 && - ShouldUndef(DbgValueSetIt->first)) { - // Mark undef, erase record of this DBG_VALUE to avoid revisiting. - DbgValueSetIt->second->setDebugValueUndef(); - continue; + if (DbgValueSetIt->first >= SegmentIt->start) { + bool HasReg = DbgValueSetIt->second->hasDebugOperandForReg(Reg); + bool ShouldUndefReg = ShouldUndef(DbgValueSetIt->first); + if (HasReg && ShouldUndefReg) { + // Mark undef, erase record of this DBG_VALUE to avoid revisiting. + DbgValueSetIt->second->setDebugValueUndef(); + continue; + } } ++DbgValueSetIt; } else { diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -768,11 +768,15 @@ OS << MI->getDebugVariable()->getName(); OS << " <- "; // Frame address. Currently handles register +- offset only. - assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm()); + assert(MI->isIndirectDebugValue()); OS << '['; - printOperand(MI, 0, OS); - OS << '+'; - printOperand(MI, 1, OS); + for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(), + MI->debug_operands().end()); + I < E; ++I) { + if (I != 0) + OS << ", "; + printOperand(MI, I, OS); + } OS << ']'; OS << "+"; printOperand(MI, NOps - 2, OS); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h b/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h --- a/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h @@ -23,6 +23,7 @@ class WebAssemblyDebugValueManager { SmallVector DbgValues; + unsigned CurrentReg; public: WebAssemblyDebugValueManager(MachineInstr *Instr); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp --- a/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp @@ -24,6 +24,7 @@ // the whole BB, not just contiguous DBG_VALUEs. if (!Instr->getOperand(0).isReg()) return; + CurrentReg = Instr->getOperand(0).getReg(); MachineBasicBlock::iterator DI = *Instr; ++DI; @@ -43,7 +44,9 @@ void WebAssemblyDebugValueManager::updateReg(unsigned Reg) { for (auto *DBI : DbgValues) - DBI->getDebugOperand(0).setReg(Reg); + for (auto &MO : DBI->getDebugOperandsForReg(CurrentReg)) + MO.setReg(Reg); + CurrentReg = Reg; } void WebAssemblyDebugValueManager::clone(MachineInstr *Insert, @@ -52,14 +55,15 @@ MachineFunction *MF = MBB->getParent(); for (MachineInstr *DBI : reverse(DbgValues)) { MachineInstr *Clone = MF->CloneMachineInstr(DBI); - Clone->getDebugOperand(0).setReg(NewReg); + for (auto &MO : Clone->getDebugOperandsForReg(CurrentReg)) + MO.setReg(NewReg); MBB->insert(Insert, Clone); } } void WebAssemblyDebugValueManager::replaceWithLocal(unsigned LocalId) { for (auto *DBI : DbgValues) { - MachineOperand &Op = DBI->getDebugOperand(0); - Op.ChangeToTargetIndex(llvm::WebAssembly::TI_LOCAL, LocalId); + for (auto &MO : DBI->getDebugOperandsForReg(CurrentReg)) + MO.ChangeToTargetIndex(llvm::WebAssembly::TI_LOCAL, LocalId); } }