[DebugInfo] Handle DBG_VALUES with multiple variable location operands in MIR

[StephenTozer]

Summary

Another patch neede for the implementation of the variadic debug values change[0], this patch adds handling for multiple location operands in the MIR-passes, excluding the debug liveness passes and DWARF emission.

Apologies for this patch only going up now, it was accidentally excluded from the patch stack until now. The changes here are relatively simple compared to the IR, debug liveness, and DWARF patches; MachineSink has slightly larger changes than the rest, but most cases are simply replacing accesses of the first debug operand, i.e. getDebugOperand(0), with an iteration over all debug operands.

[0] http://lists.llvm.org/pipermail/llvm-dev/2020-February/139376.html

[StephenTozer]

Rebased onto recent master.

[aprantl]

Mechanically LGTM to me (one question inline).

llvm/lib/CodeGen/MachineSink.cpp

40

why is this no longer calling the helper?

[scott.linder]

llvm/lib/CodeGen/PrologEpilogInserter.cpp

0–1

Couldn't this assert be retained as:

(isNotList && i ==0) || (isList && i > 1)

?

4–1

I think this belongs to the parent commit?

[StephenTozer]

Rebased onto recent master, fixed up MachineSink implementation.

[yanok]

Just wanted to mention that this causes resulting assembly to be non-deterministic, see https://bugs.llvm.org/show_bug.cgi?id=52441

llvm/include/llvm/CodeGen/MachineInstr.h

 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CODEGEN_MACHINEINSTR_H
 #define LLVM_CODEGEN_MACHINEINSTR_H
 
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/PointerSumType.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/InlineAsm.h"
MachineOperand &getDebugOperand(unsigned Index) {
     assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!");
     return *(debug_operands().begin() + Index);
   }
   const MachineOperand &getDebugOperand(unsigned Index) const {
     assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!");
     return *(debug_operands().begin() + Index);
   }
 
+  SmallSet<Register, 4> getUsedDebugRegs() const {
+    assert(isDebugValue() && "not a DBG_VALUE*");
+    SmallSet<Register, 4> UsedRegs;
+    for (auto MO : debug_operands())
+      if (MO.isReg() && MO.getReg())
+        UsedRegs.insert(MO.getReg());
+    return UsedRegs;
+  }
+
   /// Returns whether this debug value has at least one debug operand with the
   /// register \p Reg.
   bool hasDebugOperandForReg(Register Reg) const {
     return any_of(debug_operands(), [Reg](const MachineOperand &Op) {
       return Op.isReg() && Op.getReg() == Reg;
     });
   }
 

llvm/include/llvm/CodeGen/MachineInstrBuilder.h

MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
                             MachineBasicBlock::iterator I, const DebugLoc &DL,
                             const MCInstrDesc &MCID, bool IsIndirect,
                             ArrayRef<MachineOperand> MOs,
                             const MDNode *Variable, const MDNode *Expr);
 
 /// Clone a DBG_VALUE whose value has been spilled to FrameIndex.
 MachineInstr *buildDbgValueForSpill(MachineBasicBlock &BB,
                                     MachineBasicBlock::iterator I,
-                                    const MachineInstr &Orig, int FrameIndex);
+                                    const MachineInstr &Orig, int FrameIndex,
+                                    Register SpillReg);
+MachineInstr *
+buildDbgValueForSpill(MachineBasicBlock &BB, MachineBasicBlock::iterator I,
+                      const MachineInstr &Orig, int FrameIndex,
+                      SmallVectorImpl<const MachineOperand *> &SpilledOperands);
 
 /// Update a DBG_VALUE whose value has been spilled to FrameIndex. Useful when
 /// modifying an instruction in place while iterating over a basic block.
-void updateDbgValueForSpill(MachineInstr &Orig, int FrameIndex);
+void updateDbgValueForSpill(MachineInstr &Orig, int FrameIndex, Register Reg);
 
 inline unsigned getDefRegState(bool B) {
   return B ? RegState::Define : 0;
 }
 inline unsigned getImplRegState(bool B) {
   return B ? RegState::Implicit : 0;
 }
 inline unsigned getKillRegState(bool B) {

llvm/lib/CodeGen/InlineSpiller.cpp

for (MachineRegisterInfo::reg_bundle_iterator
        RegI != E; ) {
     MachineInstr *MI = &*(RegI++);
 
     // Debug values are not allowed to affect codegen.
     if (MI->isDebugValue()) {
       // Modify DBG_VALUE now that the value is in a spill slot.
       MachineBasicBlock *MBB = MI->getParent();
       LLVM_DEBUG(dbgs() << "Modifying debug info due to spill:\t" << *MI);
-      buildDbgValueForSpill(*MBB, MI, *MI, StackSlot);
+      buildDbgValueForSpill(*MBB, MI, *MI, StackSlot, Reg);
       MBB->erase(MI);
       continue;
     }
 
     assert(!MI->isDebugInstr() && "Did not expect to find a use in debug "
            "instruction that isn't a DBG_VALUE");
 
     // Ignore copies to/from snippets. We'll delete them.

llvm/lib/CodeGen/MachineInstr.cpp

MachineInstrBuilder llvm::BuildMI(MachineBasicBlock &BB,
   MachineFunction &MF = *BB.getParent();
   MachineInstr *MI = BuildMI(MF, DL, MCID, IsIndirect, MOs, Variable, Expr);
   BB.insert(I, MI);
   return MachineInstrBuilder(MF, *MI);
 }
 
 /// Compute the new DIExpression to use with a DBG_VALUE for a spill slot.
 /// This prepends DW_OP_deref when spilling an indirect DBG_VALUE.
-static const DIExpression *computeExprForSpill(const MachineInstr &MI,
-                                               Register SpillReg) {
-  assert(MI.hasDebugOperandForReg(SpillReg) && "Spill Reg is not used in MI.");
+static const DIExpression *
+computeExprForSpill(const MachineInstr &MI,
+                    SmallVectorImpl<const MachineOperand *> &SpilledOperands) {
   assert(MI.getDebugVariable()->isValidLocationForIntrinsic(MI.getDebugLoc()) &&
          "Expected inlined-at fields to agree");
 
   const DIExpression *Expr = MI.getDebugExpression();
   if (MI.isIndirectDebugValue()) {
     assert(MI.getDebugOffset().getImm() == 0 &&
            "DBG_VALUE with nonzero offset");
     Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
   } else if (MI.isDebugValueList()) {
     // We will replace the spilled register with a frame index, so
     // immediately deref all references to the spilled register.
     std::array<uint64_t, 1> Ops{{dwarf::DW_OP_deref}};
-    for (const MachineOperand &Op : MI.getDebugOperandsForReg(SpillReg)) {
-      unsigned OpIdx = MI.getDebugOperandIndex(&Op);
+    for (const MachineOperand *Op : SpilledOperands) {
+      unsigned OpIdx = MI.getDebugOperandIndex(Op);
       Expr = DIExpression::appendOpsToArg(Expr, Ops, OpIdx);
     }
   }
   return Expr;
 }
+static const DIExpression *computeExprForSpill(const MachineInstr &MI,
+                                               Register SpillReg) {
+  assert(MI.hasDebugOperandForReg(SpillReg) && "Spill Reg is not used in MI.");
+  SmallVector<const MachineOperand *> SpillOperands;
+  for (const MachineOperand &Op : MI.getDebugOperandsForReg(SpillReg))
+    SpillOperands.push_back(&Op);
+  return computeExprForSpill(MI, SpillOperands);
+}
 
 MachineInstr *llvm::buildDbgValueForSpill(MachineBasicBlock &BB,
                                           MachineBasicBlock::iterator I,
                                           const MachineInstr &Orig,
-                                          int FrameIndex) {
-  Register SpillReg = Orig.getDebugOperand(0).getReg();
+                                          int FrameIndex, Register SpillReg) {
   const DIExpression *Expr = computeExprForSpill(Orig, SpillReg);
   MachineInstrBuilder NewMI =
       BuildMI(BB, I, Orig.getDebugLoc(), Orig.getDesc());
   // Non-Variadic Operands: Location, Offset, Variable, Expression
   // Variadic Operands:     Variable, Expression, Locations...
   if (Orig.isNonListDebugValue())
     NewMI.addFrameIndex(FrameIndex).addImm(0U);
   NewMI.addMetadata(Orig.getDebugVariable()).addMetadata(Expr);
   if (Orig.isDebugValueList()) {
     for (const MachineOperand &Op : Orig.debug_operands())
       if (Op.isReg() && Op.getReg() == SpillReg)
         NewMI.addFrameIndex(FrameIndex);
       else
         NewMI.add(MachineOperand(Op));
   }
   return NewMI;
 }
+MachineInstr *llvm::buildDbgValueForSpill(
+    MachineBasicBlock &BB, MachineBasicBlock::iterator I,
+    const MachineInstr &Orig, int FrameIndex,
+    SmallVectorImpl<const MachineOperand *> &SpilledOperands) {
+  const DIExpression *Expr = computeExprForSpill(Orig, SpilledOperands);
+  MachineInstrBuilder NewMI =
+      BuildMI(BB, I, Orig.getDebugLoc(), Orig.getDesc());
+  // Non-Variadic Operands: Location, Offset, Variable, Expression
+  // Variadic Operands:     Variable, Expression, Locations...
+  if (Orig.isNonListDebugValue())
+    NewMI.addFrameIndex(FrameIndex).addImm(0U);
+  NewMI.addMetadata(Orig.getDebugVariable()).addMetadata(Expr);
+  if (Orig.isDebugValueList()) {
+    for (const MachineOperand &Op : Orig.debug_operands())
+      if (is_contained(SpilledOperands, &Op))
+        NewMI.addFrameIndex(FrameIndex);
+      else
+        NewMI.add(MachineOperand(Op));
+  }
+  return NewMI;
+}
 
-void llvm::updateDbgValueForSpill(MachineInstr &Orig, int FrameIndex) {
-  Register SpillReg = Orig.getDebugOperand(0).getReg();
-  const DIExpression *Expr = computeExprForSpill(Orig, SpillReg);
+void llvm::updateDbgValueForSpill(MachineInstr &Orig, int FrameIndex,
+                                  Register Reg) {
+  const DIExpression *Expr = computeExprForSpill(Orig, Reg);
   if (Orig.isNonListDebugValue())
     Orig.getDebugOffset().ChangeToImmediate(0U);
-  for (MachineOperand &Op : Orig.getDebugOperandsForReg(SpillReg))
+  for (MachineOperand &Op : Orig.getDebugOperandsForReg(Reg))
     Op.ChangeToFrameIndex(FrameIndex);
   Orig.getDebugExpressionOp().setMetadata(Expr);
 }
 
 void MachineInstr::collectDebugValues(
                                 SmallVectorImpl<MachineInstr *> &DbgValues) {
   MachineInstr &MI = *this;
   if (!MI.getOperand(0).isReg())

llvm/lib/CodeGen/MachineSink.cpp

 //
 // This pass is not intended to be a replacement or a complete alternative
 // for an LLVM-IR-level sinking pass. It is only designed to sink simple
 // constructs that are not exposed before lowering and instruction selection.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SparseBitVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/RegisterPressure.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Pass.h"
void MachineSinking::ProcessDbgInst(MachineInstr &MI) {
   // When we see DBG_VALUEs for registers, record any vreg it reads, so that
   // we know what to sink if the vreg def sinks.
   assert(MI.isDebugValue() && "Expected DBG_VALUE for processing");
 
   DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
                     MI.getDebugLoc()->getInlinedAt());
   bool SeenBefore = SeenDbgVars.contains(Var);
 
-  MachineOperand &MO = MI.getDebugOperand(0);
+  for (MachineOperand &MO : MI.debug_operands()) {
     if (MO.isReg() && MO.getReg().isVirtual())
       SeenDbgUsers[MO.getReg()].push_back(SeenDbgUser(&MI, SeenBefore));
+  }
 
   // Record the variable for any DBG_VALUE, to avoid re-ordering any of them.
   SeenDbgVars.insert(Var);
 }
 
 bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr &MI,
                                                  MachineBasicBlock *From,
                                                  MachineBasicBlock *To) {
return MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
           MBP.Predicate == MachineBranchPredicate::PRED_EQ) &&
          MBP.LHS.getReg() == BaseOp->getReg();
 }
 
 /// If the sunk instruction is a copy, try to forward the copy instead of
 /// leaving an 'undef' DBG_VALUE in the original location. Don't do this if
 /// there's any subregister weirdness involved. Returns true if copy
 /// propagation occurred.
-static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI) {
+static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI,
+                                 Register Reg) {
   const MachineRegisterInfo &MRI = SinkInst.getMF()->getRegInfo();
   const TargetInstrInfo &TII = *SinkInst.getMF()->getSubtarget().getInstrInfo();
 
   // Copy DBG_VALUE operand and set the original to undef. We then check to
   // see whether this is something that can be copy-forwarded. If it isn't,
   // continue around the loop.
-  MachineOperand &DbgMO = DbgMI.getDebugOperand(0);
 
   const MachineOperand *SrcMO = nullptr, *DstMO = nullptr;
   auto CopyOperands = TII.isCopyInstr(SinkInst);
   if (!CopyOperands)
     return false;
   SrcMO = CopyOperands->Source;
   DstMO = CopyOperands->Destination;
 
   // Check validity of forwarding this copy.
   bool PostRA = MRI.getNumVirtRegs() == 0;
 
   // Trying to forward between physical and virtual registers is too hard.
-  if (DbgMO.getReg().isVirtual() != SrcMO->getReg().isVirtual())
+  if (Reg.isVirtual() != SrcMO->getReg().isVirtual())
     return false;
 
   // Only try virtual register copy-forwarding before regalloc, and physical
   // register copy-forwarding after regalloc.
-  bool arePhysRegs = !DbgMO.getReg().isVirtual();
+  bool arePhysRegs = !Reg.isVirtual();
   if (arePhysRegs != PostRA)
     return false;
 
   // Pre-regalloc, only forward if all subregisters agree (or there are no
   // subregs at all). More analysis might recover some forwardable copies.
-  if (!PostRA && (DbgMO.getSubReg() != SrcMO->getSubReg() ||
-                  DbgMO.getSubReg() != DstMO->getSubReg()))
+  if (!PostRA)
+    for (auto &DbgMO : DbgMI.getDebugOperandsForReg(Reg))
+      if (DbgMO.getSubReg() != SrcMO->getSubReg() ||
+          DbgMO.getSubReg() != DstMO->getSubReg())
         return false;
 
   // Post-regalloc, we may be sinking a DBG_VALUE of a sub or super-register
   // of this copy. Only forward the copy if the DBG_VALUE operand exactly
   // matches the copy destination.
-  if (PostRA && DbgMO.getReg() != DstMO->getReg())
+  if (PostRA && Reg != DstMO->getReg())
     return false;
 
+  for (auto &DbgMO : DbgMI.getDebugOperandsForReg(Reg)) {
     DbgMO.setReg(SrcMO->getReg());
     DbgMO.setSubReg(SrcMO->getSubReg());
+  }
   return true;
 }
 
+using MIRegs = std::pair<MachineInstr *, SmallVector<unsigned, 2>>;
 /// Sink an instruction and its associated debug instructions.
 static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo,
                         MachineBasicBlock::iterator InsertPos,
-                        SmallVectorImpl<MachineInstr *> &DbgValuesToSink) {
+                        SmallVectorImpl<MIRegs> &DbgValuesToSink) {
 
   // If we cannot find a location to use (merge with), then we erase the debug
   // location to prevent debug-info driven tools from potentially reporting
   // wrong location information.
   if (!SuccToSinkTo.empty() && InsertPos != SuccToSinkTo.end())
     MI.setDebugLoc(DILocation::getMergedLocation(MI.getDebugLoc(),
                                                  InsertPos->getDebugLoc()));
   else
     MI.setDebugLoc(DebugLoc());
 
   // Move the instruction.
   MachineBasicBlock *ParentBlock = MI.getParent();
   SuccToSinkTo.splice(InsertPos, ParentBlock, MI,
                       ++MachineBasicBlock::iterator(MI));
 
   // Sink a copy of debug users to the insert position. Mark the original
   // DBG_VALUE location as 'undef', indicating that any earlier variable
   // location should be terminated as we've optimised away the value at this
   // point.
-  for (MachineInstr *DbgMI : DbgValuesToSink) {
+  for (auto DbgValueToSink : DbgValuesToSink) {
+    MachineInstr *DbgMI = DbgValueToSink.first;
     MachineInstr *NewDbgMI = DbgMI->getMF()->CloneMachineInstr(DbgMI);
     SuccToSinkTo.insert(InsertPos, NewDbgMI);
 
-    if (!attemptDebugCopyProp(MI, *DbgMI))
+    bool PropagatedAllSunkOps = true;
+    for (unsigned Reg : DbgValueToSink.second) {
+      if (DbgMI->hasDebugOperandForReg(Reg)) {
+        if (!attemptDebugCopyProp(MI, *DbgMI, Reg)) {
+          PropagatedAllSunkOps = false;
+          break;
+        }
+      }
+    }
+    if (!PropagatedAllSunkOps)
       DbgMI->setDebugValueUndef();
   }
 }
 
 /// hasStoreBetween - check if there is store betweeen straight line blocks From
 /// and To.
 bool MachineSinking::hasStoreBetween(MachineBasicBlock *From,
                                      MachineBasicBlock *To, MachineInstr &MI) {
bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
   }
 
   // Determine where to insert into. Skip phi nodes.
   MachineBasicBlock::iterator InsertPos = SuccToSinkTo->begin();
   while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
     ++InsertPos;
 
   // Collect debug users of any vreg that this inst defines.
-  SmallVector<MachineInstr *, 4> DbgUsersToSink;
+  SmallVector<MIRegs, 4> DbgUsersToSink;
   for (auto &MO : MI.operands()) {
     if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual())
       continue;
     if (!SeenDbgUsers.count(MO.getReg()))
       continue;
 
     // Sink any users that don't pass any other DBG_VALUEs for this variable.
     auto &Users = SeenDbgUsers[MO.getReg()];
     for (auto &User : Users) {
       MachineInstr *DbgMI = User.getPointer();
       if (User.getInt()) {
         // This DBG_VALUE would re-order assignments. If we can't copy-propagate
         // it, it can't be recovered. Set it undef.
-        if (!attemptDebugCopyProp(MI, *DbgMI))
+        if (!attemptDebugCopyProp(MI, *DbgMI, MO.getReg()))
           DbgMI->setDebugValueUndef();
       } else {
-        DbgUsersToSink.push_back(DbgMI);
+        DbgUsersToSink.push_back(
+            {DbgMI, SmallVector<unsigned, 2>(1, MO.getReg())});
       }
     }
   }
 
   // After sinking, some debug users may not be dominated any more. If possible,
   // copy-propagate their operands. As it's expensive, don't do this if there's
   // no debuginfo in the program.
   if (MI.getMF()->getFunction().getSubprogram() && MI.isCopy())
void MachineSinking::SalvageUnsunkDebugUsersOfCopy(
     MachineInstr &MI, MachineBasicBlock *TargetBlock) {
   assert(MI.isCopy());
   assert(MI.getOperand(1).isReg());
 
   // Enumerate all users of vreg operands that are def'd. Skip those that will
   // be sunk. For the rest, if they are not dominated by the block we will sink
   // MI into, propagate the copy source to them.
   SmallVector<MachineInstr *, 4> DbgDefUsers;
+  SmallVector<Register, 4> DbgUseRegs;
   const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
   for (auto &MO : MI.operands()) {
     if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual())
       continue;
+    DbgUseRegs.push_back(MO.getReg());
     for (auto &User : MRI.use_instructions(MO.getReg())) {
       if (!User.isDebugValue() || DT->dominates(TargetBlock, User.getParent()))
         continue;
 
       // If is in same block, will either sink or be use-before-def.
       if (User.getParent() == MI.getParent())
         continue;
 
-      assert(User.getDebugOperand(0).isReg() &&
-             "DBG_VALUE user of vreg, but non reg operand?");
+      assert(User.hasDebugOperandForReg(MO.getReg()) &&
+             "DBG_VALUE user of vreg, but has no operand for it?");
       DbgDefUsers.push_back(&User);
     }
   }
 
   // Point the users of this copy that are no longer dominated, at the source
   // of the copy.
   for (auto *User : DbgDefUsers) {
-    User->getDebugOperand(0).setReg(MI.getOperand(1).getReg());
-    User->getDebugOperand(0).setSubReg(MI.getOperand(1).getSubReg());
+    for (auto &Reg : DbgUseRegs) {
+      for (auto &DbgOp : User->getDebugOperandsForReg(Reg)) {
+        DbgOp.setReg(MI.getOperand(1).getReg());
+        DbgOp.setSubReg(MI.getOperand(1).getSubReg());
+      }
+    }
   }
 }
 
 //===----------------------------------------------------------------------===//
 // This pass is not intended to be a replacement or a complete alternative
 // for the pre-ra machine sink pass. It is only designed to sink COPY
 // instructions which should be handled after RA.
 //
return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::NoVRegs);
   }
 
 private:
   /// Track which register units have been modified and used.
   LiveRegUnits ModifiedRegUnits, UsedRegUnits;
 
   /// Track DBG_VALUEs of (unmodified) register units. Each DBG_VALUE has an
-  /// entry in this map for each unit it touches.
-  DenseMap<unsigned, TinyPtrVector<MachineInstr *>> SeenDbgInstrs;
+  /// entry in this map for each unit it touches. The DBG_VALUE's entry
+  /// consists of a pointer to the instruction itself, and a vector of registers
+  /// referred to by the instruction that overlap the key register unit.
+  DenseMap<unsigned, SmallVector<MIRegs, 2>> SeenDbgInstrs;
 
   /// Sink Copy instructions unused in the same block close to their uses in
   /// successors.
   bool tryToSinkCopy(MachineBasicBlock &BB, MachineFunction &MF,
                      const TargetRegisterInfo *TRI, const TargetInstrInfo *TII);
 };
 } // namespace
 
for (auto I = CurBB.rbegin(), E = CurBB.rend(); I != E;) {
     // Track the operand index for use in Copy.
     SmallVector<unsigned, 2> UsedOpsInCopy;
     // Track the register number defed in Copy.
     SmallVector<unsigned, 2> DefedRegsInCopy;
 
     // We must sink this DBG_VALUE if its operand is sunk. To avoid searching
     // for DBG_VALUEs later, record them when they're encountered.
     if (MI->isDebugValue()) {
-      auto &MO = MI->getDebugOperand(0);
+      SmallDenseMap<MCRegister, SmallVector<unsigned, 2>, 4> MIUnits;
+      bool IsValid = true;
+      for (MachineOperand &MO : MI->debug_operands()) {
         if (MO.isReg() && Register::isPhysicalRegister(MO.getReg())) {
           // Bail if we can already tell the sink would be rejected, rather
           // than needlessly accumulating lots of DBG_VALUEs.
           if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy,
-                                  ModifiedRegUnits, UsedRegUnits))
-          continue;
+                                    ModifiedRegUnits, UsedRegUnits)) {
+            IsValid = false;
+            break;
+          }
 
           // Record debug use of each reg unit.
-        SmallSet<MCRegister, 4> Units = getRegUnits(MO.getReg(), TRI);
-        for (MCRegister Reg : Units)
-          SeenDbgInstrs[Reg].push_back(MI);
+          SmallSet<MCRegister, 4> RegUnits = getRegUnits(MO.getReg(), TRI);
+          for (MCRegister Reg : RegUnits)
+            MIUnits[Reg].push_back(MO.getReg());
+        }
+      }
+      if (IsValid) {
+        for (auto RegOps : MIUnits)
+          SeenDbgInstrs[RegOps.first].push_back({MI, RegOps.second});
       }
       continue;
     }
 
     if (MI->isDebugInstr())
       continue;
 
     // Do not move any instruction across function call.
if (!SuccBB) {
       continue;
     }
     assert((SuccBB->pred_size() == 1 && *SuccBB->pred_begin() == &CurBB) &&
            "Unexpected predecessor");
 
     // Collect DBG_VALUEs that must sink with this copy. We've previously
     // recorded which reg units that DBG_VALUEs read, if this instruction
     // writes any of those units then the corresponding DBG_VALUEs must sink.
-    SetVector<MachineInstr *> DbgValsToSinkSet;
+    MapVector<MachineInstr *, MIRegs::second_type> DbgValsToSinkMap;
     for (auto &MO : MI->operands()) {
       if (!MO.isReg() || !MO.isDef())
         continue;
 
       SmallSet<MCRegister, 4> Units = getRegUnits(MO.getReg(), TRI);
-      for (MCRegister Reg : Units)
-        for (auto *MI : SeenDbgInstrs.lookup(Reg))
-          DbgValsToSinkSet.insert(MI);
+      for (MCRegister Reg : Units) {
+        for (auto MIRegs : SeenDbgInstrs.lookup(Reg)) {
+          auto &Regs = DbgValsToSinkMap[MIRegs.first];
+          for (unsigned Reg : MIRegs.second)
+            Regs.push_back(Reg);
+        }
+      }
     }
-    SmallVector<MachineInstr *, 4> DbgValsToSink(DbgValsToSinkSet.begin(),
-                                                 DbgValsToSinkSet.end());
+    SmallVector<MIRegs, 4> DbgValsToSink(DbgValsToSinkMap.begin(),
+                                         DbgValsToSinkMap.end());
 
     // Clear the kill flag if SrcReg is killed between MI and the end of the
     // block.
     clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegUnits, TRI);
     MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI();
     performSink(*MI, *SuccBB, InsertPos, DbgValsToSink);
     updateLiveIn(MI, SuccBB, UsedOpsInCopy, DefedRegsInCopy);
 

llvm/lib/CodeGen/PrologEpilogInserter.cpp

 //===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This pass is responsible for finalizing the functions frame layout, saving
static void stashEntryDbgValues(MachineBasicBlock &MBB,
                                 SavedDbgValuesMap &EntryDbgValues) {
   SmallVector<const MachineInstr *, 4> FrameIndexValues;
 
   for (auto &MI : MBB) {
     if (!MI.isDebugInstr())
       break;
     if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter())
       continue;
-    if (MI.getDebugOperand(0).isFI()) {
+    if (any_of(MI.debug_operands(),
+               [](const MachineOperand &MO) { return MO.isFI(); })) {
       // We can only emit valid locations for frame indices after the frame
       // setup, so do not stash away them.
       FrameIndexValues.push_back(&MI);
       continue;
     }
     const DILocalVariable *Var = MI.getDebugVariable();
     const DIExpression *Expr = MI.getDebugExpression();
     auto Overlaps = [Var, Expr](const MachineInstr *DV) {
for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) {
     for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
       if (!MI.getOperand(i).isFI())
         continue;
 
       // Frame indices in debug values are encoded in a target independent
       // way with simply the frame index and offset rather than any
       // target-specific addressing mode.
       if (MI.isDebugValue()) {
-        assert(i == 0 && "Frame indices can only appear as the first "
-                         "operand of a DBG_VALUE machine instruction");
+        MachineOperand &Op = MI.getOperand(i);
+        assert(
+            MI.isDebugOperand(&Op) &&
+            "Frame indices can only appear as a debug operand in a DBG_VALUE*"
+            " machine instruction");
         Register Reg;
-        unsigned FrameIdx = MI.getOperand(0).getIndex();
+        unsigned FrameIdx = Op.getIndex();
         unsigned Size = MF.getFrameInfo().getObjectSize(FrameIdx);
 
         StackOffset Offset =
             TFI->getFrameIndexReference(MF, FrameIdx, Reg);
-        MI.getOperand(0).ChangeToRegister(Reg, false /*isDef*/);
-        MI.getOperand(0).setIsDebug();
+        Op.ChangeToRegister(Reg, false /*isDef*/);
+        Op.setIsDebug();
 
         const DIExpression *DIExpr = MI.getDebugExpression();
 
         // If we have a direct DBG_VALUE, and its location expression isn't
         // currently complex, then adding an offset will morph it into a
         // complex location that is interpreted as being a memory address.
         // This changes a pointer-valued variable to dereference that pointer,
         // which is incorrect. Fix by adding DW_OP_stack_value.
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
             // Make the DBG_VALUE direct.
             MI.getDebugOffset().ChangeToRegister(0, false);
           }
           DIExpr = TRI.prependOffsetExpression(DIExpr, PrependFlags, Offset);
         } else {
           // The debug operand at DebugOpIndex was a frame index at offset
           // `Offset`; now the operand has been replaced with the frame
           // register, we must add Offset with `register x, plus Offset`.
-          unsigned DebugOpIndex = MI.getDebugOperandIndex(&MI.getOperand(i));
+          unsigned DebugOpIndex = MI.getDebugOperandIndex(&Op);
           SmallVector<uint64_t, 3> Ops;
           TRI.getOffsetOpcodes(Offset, Ops);
           DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, DebugOpIndex);
         }
         MI.getDebugExpressionOp().setMetadata(DIExpr);
         continue;
       }
 

llvm/lib/CodeGen/RegAllocFast.cpp

private:
     using LiveRegMap = SparseSet<LiveReg>;
     /// This map contains entries for each virtual register that is currently
     /// available in a physical register.
     LiveRegMap LiveVirtRegs;
 
     /// Stores assigned virtual registers present in the bundle MI.
     DenseMap<Register, MCPhysReg> BundleVirtRegsMap;
 
-    DenseMap<unsigned, SmallVector<MachineInstr *, 2>> LiveDbgValueMap;
+    DenseMap<unsigned, SmallVector<MachineOperand *, 2>> LiveDbgValueMap;
     /// List of DBG_VALUE that we encountered without the vreg being assigned
     /// because they were placed after the last use of the vreg.
     DenseMap<unsigned, SmallVector<MachineInstr *, 1>> DanglingDbgValues;
 
     /// Has a bit set for every virtual register for which it was determined
     /// that it is alive across blocks.
     BitVector MayLiveAcrossBlocks;
 
void RegAllocFast::spill(MachineBasicBlock::iterator Before, Register VirtReg,
   TII->storeRegToStackSlot(*MBB, Before, AssignedReg, Kill, FI, &RC, TRI);
   ++NumStores;
 
   MachineBasicBlock::iterator FirstTerm = MBB->getFirstTerminator();
 
   // When we spill a virtual register, we will have spill instructions behind
   // every definition of it, meaning we can switch all the DBG_VALUEs over
   // to just reference the stack slot.
-  SmallVectorImpl<MachineInstr *> &LRIDbgValues = LiveDbgValueMap[VirtReg];
-  for (MachineInstr *DBG : LRIDbgValues) {
-    MachineInstr *NewDV = buildDbgValueForSpill(*MBB, Before, *DBG, FI);
+  SmallVectorImpl<MachineOperand *> &LRIDbgOperands = LiveDbgValueMap[VirtReg];
+  SmallDenseMap<MachineInstr *, SmallVector<const MachineOperand *>>
+      SpilledOperandsMap;
+  for (MachineOperand *MO : LRIDbgOperands)
+    SpilledOperandsMap[MO->getParent()].push_back(MO);
+  for (auto MISpilledOperands : SpilledOperandsMap) {
+    MachineInstr &DBG = *MISpilledOperands.first;
+    MachineInstr *NewDV = buildDbgValueForSpill(
+        *MBB, Before, *MISpilledOperands.first, FI, MISpilledOperands.second);
     assert(NewDV->getParent() == MBB && "dangling parent pointer");
     (void)NewDV;
     LLVM_DEBUG(dbgs() << "Inserting debug info due to spill:\n" << *NewDV);
 
     if (LiveOut) {
       // We need to insert a DBG_VALUE at the end of the block if the spill slot
       // is live out, but there is another use of the value after the
       // spill. This will allow LiveDebugValues to see the correct live out
       // value to propagate to the successors.
       MachineInstr *ClonedDV = MBB->getParent()->CloneMachineInstr(NewDV);
       MBB->insert(FirstTerm, ClonedDV);
       LLVM_DEBUG(dbgs() << "Cloning debug info due to live out spill\n");
     }
 
     // Rewrite unassigned dbg_values to use the stack slot.
     // TODO We can potentially do this for list debug values as well if we know
     // how the dbg_values are getting unassigned.
-    if (DBG->isNonListDebugValue()) {
-      MachineOperand &MO = DBG->getDebugOperand(0);
+    if (DBG.isNonListDebugValue()) {
+      MachineOperand &MO = DBG.getDebugOperand(0);
       if (MO.isReg() && MO.getReg() == 0) {
-        updateDbgValueForSpill(*DBG, FI);
+        updateDbgValueForSpill(DBG, FI, 0);
       }
     }
   }
   // Now this register is spilled there is should not be any DBG_VALUE
   // pointing to this register because they are all pointing to spilled value
   // now.
-  LRIDbgValues.clear();
+  LRIDbgOperands.clear();
 }
 
 /// Insert reload instruction for \p PhysReg before \p Before.
 void RegAllocFast::reload(MachineBasicBlock::iterator Before, Register VirtReg,
                           MCPhysReg PhysReg) {
   LLVM_DEBUG(dbgs() << "Reloading " << printReg(VirtReg, TRI) << " into "
                     << printReg(PhysReg, TRI) << '\n');
   int FI = getStackSpaceFor(VirtReg);
void RegAllocFast::allocateInstruction(MachineInstr &MI) {
   if (MI.isCopy() && MI.getOperand(0).getReg() == MI.getOperand(1).getReg() &&
       MI.getNumOperands() == 2) {
     LLVM_DEBUG(dbgs() << "Mark identity copy for removal\n");
     Coalesced.push_back(&MI);
   }
 }
 
 void RegAllocFast::handleDebugValue(MachineInstr &MI) {
-  SmallSet<Register, 4> SeenRegisters;
-  for (MachineOperand &MO : MI.debug_operands()) {
-    // Ignore DBG_VALUEs that aren't based on virtual registers. These are
-    // mostly constants and frame indices.
-    if (!MO.isReg())
-      continue;
-    Register Reg = MO.getReg();
+  for (Register Reg : MI.getUsedDebugRegs()) {
     if (!Register::isVirtualRegister(Reg))
       continue;
-    // Only process each register once per MI, each use of that register will
-    // be updated if necessary.
-    if (!SeenRegisters.insert(Reg).second)
-      continue;
 
     // Already spilled to a stackslot?
     int SS = StackSlotForVirtReg[Reg];
     if (SS != -1) {
       // Modify DBG_VALUE now that the value is in a spill slot.
-      updateDbgValueForSpill(MI, SS);
+      updateDbgValueForSpill(MI, SS, Reg);
       LLVM_DEBUG(dbgs() << "Rewrite DBG_VALUE for spilled memory: " << MI);
       continue;
     }
 
     // See if this virtual register has already been allocated to a physical
     // register or spilled to a stack slot.
     LiveRegMap::iterator LRI = findLiveVirtReg(Reg);
+    SmallVector<MachineOperand *> DbgOps;
+    for (MachineOperand &Op : MI.getDebugOperandsForReg(Reg))
+      DbgOps.push_back(&Op);
+
     if (LRI != LiveVirtRegs.end() && LRI->PhysReg) {
       // Update every use of Reg within MI.
-      for (auto &RegMO : MI.getDebugOperandsForReg(Reg))
-        setPhysReg(MI, RegMO, LRI->PhysReg);
+      for (auto &RegMO : DbgOps)
+        setPhysReg(MI, *RegMO, LRI->PhysReg);
     } else {
       DanglingDbgValues[Reg].push_back(&MI);
     }
 
     // If Reg hasn't been spilled, put this DBG_VALUE in LiveDbgValueMap so
     // that future spills of Reg will have DBG_VALUEs.
-    LiveDbgValueMap[Reg].push_back(&MI);
+    LiveDbgValueMap[Reg].append(DbgOps.begin(), DbgOps.end());
   }
 }
 
 void RegAllocFast::handleBundle(MachineInstr &MI) {
   MachineBasicBlock::instr_iterator BundledMI = MI.getIterator();
   ++BundledMI;
   while (BundledMI->isBundledWithPred()) {
     for (unsigned I = 0; I < BundledMI->getNumOperands(); ++I) {

llvm/lib/CodeGen/RegisterCoalescer.cpp

 void RegisterCoalescer::buildVRegToDbgValueMap(MachineFunction &MF)
 {
   const SlotIndexes &Slots = *LIS->getSlotIndexes();
   SmallVector<MachineInstr *, 8> ToInsert;
 
   // After collecting a block of DBG_VALUEs into ToInsert, enter them into the
   // vreg => DbgValueLoc map.
   auto CloseNewDVRange = [this, &ToInsert](SlotIndex Slot) {
-    for (auto *X : ToInsert)
-      DbgVRegToValues[X->getDebugOperand(0).getReg()].push_back({Slot, X});
+    for (auto *X : ToInsert) {
+      for (auto Op : X->debug_operands()) {
+        if (Op.isReg() && Op.getReg().isVirtual())
+          DbgVRegToValues[Op.getReg()].push_back({Slot, X});
+      }
+    }
 
     ToInsert.clear();
   };
 
   // Iterate over all instructions, collecting them into the ToInsert vector.
   // Once a non-debug instruction is found, record the slot index of the
   // collected DBG_VALUEs.
   for (auto &MBB : MF) {
     SlotIndex CurrentSlot = Slots.getMBBStartIdx(&MBB);
 
     for (auto &MI : MBB) {
-      if (MI.isDebugValue() && MI.getDebugOperand(0).isReg() &&
-          MI.getDebugOperand(0).getReg().isVirtual()) {
+      if (MI.isDebugValue()) {
+        if (any_of(MI.debug_operands(), [](const MachineOperand &MO) {
+              return MO.isReg() && MO.getReg().isVirtual();
+            }))
           ToInsert.push_back(&MI);
       } else if (!MI.isDebugInstr()) {
         CurrentSlot = Slots.getInstructionIndex(MI);
         CloseNewDVRange(CurrentSlot);
       }
     }
 
     // Close range of DBG_VALUEs at the end of blocks.
     CloseNewDVRange(Slots.getMBBEndIdx(&MBB));
void RegisterCoalescer::checkMergingChangesDbgValuesImpl(Register Reg,
 
   // Iterate over both the live-range of the "Other" register, and the set of
   // DBG_VALUEs for Reg at the same time. Advance whichever one has the lowest
   // slot index. This relies on the DbgValueSet being ordered.
   while (DbgValueSetIt != DbgValueSet.end() && SegmentIt != OtherLR.end()) {
     if (DbgValueSetIt->first < SegmentIt->end) {
       // "Other" is live and there is a DBG_VALUE of Reg: test if we should
       // set it undef.
-      if (DbgValueSetIt->first >= SegmentIt->start &&
-          DbgValueSetIt->second->getDebugOperand(0).getReg() != 0 &&
-          ShouldUndef(DbgValueSetIt->first)) {
+      if (DbgValueSetIt->first >= SegmentIt->start) {
+        bool HasReg = DbgValueSetIt->second->hasDebugOperandForReg(Reg);
+        bool ShouldUndefReg = ShouldUndef(DbgValueSetIt->first);
+        if (HasReg && ShouldUndefReg) {
           // Mark undef, erase record of this DBG_VALUE to avoid revisiting.
           DbgValueSetIt->second->setDebugValueUndef();
           continue;
         }
+      }
       ++DbgValueSetIt;
     } else {
       ++SegmentIt;
     }
   }
 }
 
 namespace {

llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
                                                raw_ostream &OS) {
   unsigned NOps = MI->getNumOperands();
   assert(NOps == 4);
   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
   // cast away const; DIetc do not take const operands for some reason.
   OS << MI->getDebugVariable()->getName();
   OS << " <- ";
   // Frame address.  Currently handles register +- offset only.
-  assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm());
+  assert(MI->isIndirectDebugValue());
   OS << '[';
-  printOperand(MI, 0, OS);
-  OS << '+';
-  printOperand(MI, 1, OS);
+  for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
+                                         MI->debug_operands().end());
+       I < E; ++I) {
+    if (I != 0)
+      OS << ", ";
+    printOperand(MI, I, OS);
+  }
   OS << ']';
   OS << "+";
   printOperand(MI, NOps - 2, OS);
 }
 
 void AArch64AsmPrinter::emitJumpTableInfo() {
   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
   if (!MJTI) return;

llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h

 #include "llvm/ADT/SmallVector.h"
 
 namespace llvm {
 
 class MachineInstr;
 
 class WebAssemblyDebugValueManager {
   SmallVector<MachineInstr *, 2> DbgValues;
+  unsigned CurrentReg;
 
 public:
   WebAssemblyDebugValueManager(MachineInstr *Instr);
 
   void move(MachineInstr *Insert);
   void updateReg(unsigned Reg);
   void clone(MachineInstr *Insert, unsigned NewReg);
   void replaceWithLocal(unsigned LocalId);
 };
 
 } // end namespace llvm
 
 #endif

llvm/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp

 using namespace llvm;
 
 WebAssemblyDebugValueManager::WebAssemblyDebugValueManager(
     MachineInstr *Instr) {
   // This code differs from MachineInstr::collectDebugValues in that it scans
   // the whole BB, not just contiguous DBG_VALUEs.
   if (!Instr->getOperand(0).isReg())
     return;
+  CurrentReg = Instr->getOperand(0).getReg();
 
   MachineBasicBlock::iterator DI = *Instr;
   ++DI;
   for (MachineBasicBlock::iterator DE = Instr->getParent()->end(); DI != DE;
        ++DI) {
     if (DI->isDebugValue() &&
         DI->hasDebugOperandForReg(Instr->getOperand(0).getReg()))
       DbgValues.push_back(&*DI);
   }
 }
 
 void WebAssemblyDebugValueManager::move(MachineInstr *Insert) {
   MachineBasicBlock *MBB = Insert->getParent();
   for (MachineInstr *DBI : reverse(DbgValues))
     MBB->splice(Insert, DBI->getParent(), DBI);
 }
 
 void WebAssemblyDebugValueManager::updateReg(unsigned Reg) {
   for (auto *DBI : DbgValues)
-    DBI->getDebugOperand(0).setReg(Reg);
+    for (auto &MO : DBI->getDebugOperandsForReg(CurrentReg))
+      MO.setReg(Reg);
+  CurrentReg = Reg;
 }
 
 void WebAssemblyDebugValueManager::clone(MachineInstr *Insert,
                                          unsigned NewReg) {
   MachineBasicBlock *MBB = Insert->getParent();
   MachineFunction *MF = MBB->getParent();
   for (MachineInstr *DBI : reverse(DbgValues)) {
     MachineInstr *Clone = MF->CloneMachineInstr(DBI);
-    Clone->getDebugOperand(0).setReg(NewReg);
+    for (auto &MO : Clone->getDebugOperandsForReg(CurrentReg))
+      MO.setReg(NewReg);
     MBB->insert(Insert, Clone);
   }
 }
 
 void WebAssemblyDebugValueManager::replaceWithLocal(unsigned LocalId) {
   for (auto *DBI : DbgValues) {
-    MachineOperand &Op0 = DBI->getDebugOperand(0);
-    MachineOperand &Op1 = DBI->getOperand(1);
-    bool Indirect = Op1.isImm() && Op1.getImm() == 0;
-    Op0.ChangeToTargetIndex(Indirect ? llvm::WebAssembly::TI_LOCAL_INDIRECT
-                                     : llvm::WebAssembly::TI_LOCAL,
-                            LocalId);
+    auto IndexType = DBI->isIndirectDebugValue()
+                         ? llvm::WebAssembly::TI_LOCAL_INDIRECT
+                         : llvm::WebAssembly::TI_LOCAL;
+    for (auto &MO : DBI->getDebugOperandsForReg(CurrentReg))
+      MO.ChangeToTargetIndex(IndexType, LocalId);
   }
 }