[CSSPGO] Exclude pseudo probes from slot index

[hoy]

Summary

Pseudo probe are currently given a slot index like other regular instructions. This affects register pressure and lifetime weight computation because of enlarged lifetime length with pseudo probe instructions. As a consequence, program could get different code generated w/ and w/o pseudo probes. I'm closing the gap by excluding pseudo probes from stack index and downstream register allocation related passes.

[hoy]

Updating D100334: [CSSPGO] Exclude pseudo probes from slot index

[wmi]

LGTM.

llvm/lib/CodeGen/LexicalScopes.cpp

void LexicalScopes::extractLexicalScopes(
     SmallVectorImpl<InsnRange> &MIRanges,
     DenseMap<const MachineInstr *, LexicalScope *> &MI2ScopeMap) {
   // Scan each instruction and create scopes. First build working set of scopes.
   for (const auto &MBB : *MF) {
     const MachineInstr *RangeBeginMI = nullptr;
     const MachineInstr *PrevMI = nullptr;
     const DILocation *PrevDL = nullptr;
     for (const auto &MInsn : MBB) {
+      // Ignore DBG_VALUE and similar instruction that do not contribute to any
+      // instruction in the output.
+      if (MInsn.isMetaInstruction())
+        continue;
+
       // Check if instruction has valid location information.
       const DILocation *MIDL = MInsn.getDebugLoc();
       if (!MIDL) {
         PrevMI = &MInsn;
         continue;
       }
 
       // If scope has not changed then skip this instruction.
       if (MIDL == PrevDL) {
         PrevMI = &MInsn;
         continue;
       }
 
-      // Ignore DBG_VALUE and similar instruction that do not contribute to any
-      // instruction in the output.
-      if (MInsn.isMetaInstruction())
-        continue;
-
       if (RangeBeginMI) {
         // If we have already seen a beginning of an instruction range and
         // current instruction scope does not match scope of first instruction
         // in this range then create a new instruction range.
         InsnRange R(RangeBeginMI, PrevMI);
         MI2ScopeMap[RangeBeginMI] = getOrCreateLexicalScope(PrevDL);
         MIRanges.push_back(R);
       }

llvm/lib/CodeGen/LiveDebugVariables.cpp

 
 bool LDVImpl::collectDebugValues(MachineFunction &mf) {
   bool Changed = false;
   for (MachineBasicBlock &MBB : mf) {
     for (MachineBasicBlock::iterator MBBI = MBB.begin(), MBBE = MBB.end();
          MBBI != MBBE;) {
       // Use the first debug instruction in the sequence to get a SlotIndex
       // for following consecutive debug instructions.
-      if (!MBBI->isDebugInstr()) {
+      if (!MBBI->isDebugOrPseudoInstr()) {
         ++MBBI;
         continue;
       }
       // Debug instructions has no slot index. Use the previous
       // non-debug instruction's SlotIndex as its SlotIndex.
       SlotIndex Idx =
           MBBI == MBB.begin()
               ? LIS->getMBBStartIdx(&MBB)
               : LIS->getInstructionIndex(*std::prev(MBBI)).getRegSlot();
       // Handle consecutive debug instructions with the same slot index.
       do {
         // Only handle DBG_VALUE in handleDebugValue(). Skip all other
         // kinds of debug instructions.
         if ((MBBI->isDebugValue() && handleDebugValue(*MBBI, Idx)) ||
             (MBBI->isDebugRef() && handleDebugInstrRef(*MBBI, Idx)) ||
             (MBBI->isDebugLabel() && handleDebugLabel(*MBBI, Idx))) {
           MBBI = MBB.erase(MBBI);
           Changed = true;
         } else
           ++MBBI;
-      } while (MBBI != MBBE && MBBI->isDebugInstr());
+      } while (MBBI != MBBE && MBBI->isDebugOrPseudoInstr());
     }
   }
   return Changed;
 }
 
 void UserValue::extendDef(
     SlotIndex Idx, DbgVariableValue DbgValue,
     SmallDenseMap<unsigned, std::pair<LiveRange *, const VNInfo *>>

llvm/lib/CodeGen/LiveIntervals.cpp

SlotIndex findLastUseBefore(SlotIndex Before, Register Reg,
     MachineBasicBlock::iterator MII = MBB->end();
     if (MachineInstr *MI = Indexes->getInstructionFromIndex(
                            Indexes->getNextNonNullIndex(OldIdx)))
       if (MI->getParent() == MBB)
         MII = MI;
 
     MachineBasicBlock::iterator Begin = MBB->begin();
     while (MII != Begin) {
-      if ((--MII)->isDebugInstr())
+      if ((--MII)->isDebugOrPseudoInstr())
         continue;
       SlotIndex Idx = Indexes->getInstructionIndex(*MII);
 
       // Stop searching when Before is reached.
       if (!SlotIndex::isEarlierInstr(Before, Idx))
         return Before;
 
       // Check if MII uses Reg.
void LiveIntervals::repairOldRegInRange(const MachineBasicBlock::iterator Begin,
   if (LII != LR.end() && LII->start < EndIdx)
     lastUseIdx = LII->end;
   else
     --LII;
 
   for (MachineBasicBlock::iterator I = End; I != Begin;) {
     --I;
     MachineInstr &MI = *I;
-    if (MI.isDebugInstr())
+    if (MI.isDebugOrPseudoInstr())
       continue;
 
     SlotIndex instrIdx = getInstructionIndex(MI);
     bool isStartValid = getInstructionFromIndex(LII->start);
     bool isEndValid = getInstructionFromIndex(LII->end);
 
     // FIXME: This doesn't currently handle early-clobber or multiple removed
     // defs inside of the region to repair.
LiveIntervals::repairIntervalsInRange(MachineBasicBlock *MBB,
   else
     EndIdx = getInstructionIndex(*End);
 
   Indexes->repairIndexesInRange(MBB, Begin, End);
 
   for (MachineBasicBlock::iterator I = End; I != Begin;) {
     --I;
     MachineInstr &MI = *I;
-    if (MI.isDebugInstr())
+    if (MI.isDebugOrPseudoInstr())
       continue;
     for (MachineInstr::const_mop_iterator MOI = MI.operands_begin(),
                                           MOE = MI.operands_end();
          MOI != MOE; ++MOI) {
       if (MOI->isReg() && Register::isVirtualRegister(MOI->getReg()) &&
           !hasInterval(MOI->getReg())) {
         createAndComputeVirtRegInterval(MOI->getReg());
       }

llvm/lib/CodeGen/LiveRangeShrink.cpp

if (MBB.empty())
       continue;
     bool SawStore = false;
     BuildInstOrderMap(MBB.begin(), IOM);
     UseMap.clear();
 
     for (MachineBasicBlock::iterator Next = MBB.begin(); Next != MBB.end();) {
       MachineInstr &MI = *Next;
       ++Next;
-      if (MI.isPHI() || MI.isDebugInstr())
+      if (MI.isPHI() || MI.isDebugOrPseudoInstr())
         continue;
       if (MI.mayStore())
         SawStore = true;
 
       unsigned CurrentOrder = IOM[&MI];
       unsigned Barrier = 0;
       MachineInstr *BarrierMI = nullptr;
       for (const MachineOperand &MO : MI.operands()) {
for (MachineBasicBlock::iterator Next = MBB.begin(); Next != MBB.end();) {
         if (I == BarrierMI) {
           Insert = nullptr;
           break;
         }
       // Move the instruction when # of shrunk live range > 1.
       if (DefMO && Insert && NumEligibleUse > 1 && Barrier <= IOM[Insert]) {
         MachineBasicBlock::iterator I = std::next(Insert->getIterator());
         // Skip all the PHI and debug instructions.
-        while (I != MBB.end() && (I->isPHI() || I->isDebugInstr()))
+        while (I != MBB.end() && (I->isPHI() || I->isDebugOrPseudoInstr()))
           I = std::next(I);
         if (I == MI.getIterator())
           continue;
 
         // Update the dominator order to be the same as the insertion point.
         // We do this to maintain a non-decreasing order without need to update
         // all instruction orders after the insertion point.
         unsigned NewOrder = IOM[&*I];

llvm/lib/CodeGen/LiveVariables.cpp

for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
       PhysRegDef[SubReg] = &MI;
       PhysRegUse[SubReg]  = nullptr;
     }
   }
 }
 
 void LiveVariables::runOnInstr(MachineInstr &MI,
                                SmallVectorImpl<unsigned> &Defs) {
-  assert(!MI.isDebugInstr());
+  assert(!MI.isDebugOrPseudoInstr());
   // Process all of the operands of the instruction...
   unsigned NumOperandsToProcess = MI.getNumOperands();
 
   // Unless it is a PHI node.  In this case, ONLY process the DEF, not any
   // of the uses.  They will be handled in other basic blocks.
   if (MI.isPHI())
     NumOperandsToProcess = 1;
 
assert(Register::isPhysicalRegister(LI.PhysReg) &&
            "Cannot have a live-in virtual register!");
     HandlePhysRegDef(LI.PhysReg, nullptr, Defs);
   }
 
   // Loop over all of the instructions, processing them.
   DistanceMap.clear();
   unsigned Dist = 0;
   for (MachineInstr &MI : *MBB) {
-    if (MI.isDebugInstr())
+    if (MI.isDebugOrPseudoInstr())
       continue;
     DistanceMap.insert(std::make_pair(&MI, Dist++));
 
     runOnInstr(MI, Defs);
   }
 
   // Handle any virtual assignments from PHI nodes which might be at the
   // bottom of this basic block.  We check all of our successor blocks to see

llvm/lib/CodeGen/MachineBasicBlock.cpp

 MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI,
                                            MCRegister Reg, const_iterator Before,
                                            unsigned Neighborhood) const {
   unsigned N = Neighborhood;
 
   // Try searching forwards from Before, looking for reads or defs.
   const_iterator I(Before);
   for (; I != end() && N > 0; ++I) {
-    if (I->isDebugInstr())
+    if (I->isDebugOrPseudoInstr())
       continue;
 
     --N;
 
     PhysRegInfo Info = AnalyzePhysRegInBundle(*I, Reg, TRI);
 
     // Register is live when we read it here.
     if (Info.Read)
MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI,
 
   // Start by searching backwards from Before, looking for kills, reads or defs.
   I = const_iterator(Before);
   // If this is the first insn in the block, don't search backwards.
   if (I != begin()) {
     do {
       --I;
 
-      if (I->isDebugInstr())
+      if (I->isDebugOrPseudoInstr())
         continue;
 
       --N;
 
       PhysRegInfo Info = AnalyzePhysRegInBundle(*I, Reg, TRI);
 
       // Defs happen after uses so they take precedence if both are present.
 
do {
       if (Info.Read)
         return LQR_Live;
 
     } while (I != begin() && N > 0);
   }
 
   // If all the instructions before this in the block are debug instructions,
   // skip over them.
-  while (I != begin() && std::prev(I)->isDebugInstr())
+  while (I != begin() && std::prev(I)->isDebugOrPseudoInstr())
     --I;
 
   // Did we get to the start of the block?
   if (I == begin()) {
     // If so, the register's state is definitely defined by the live-in state.
     for (const MachineBasicBlock::RegisterMaskPair &LI : liveins())
       if (TRI->regsOverlap(LI.PhysReg, Reg))
         return LQR_Live;

llvm/lib/CodeGen/MachineScheduler.cpp

static cl::opt<bool> EnablePostRAMachineSched(
     cl::init(true), cl::Hidden);
 
 /// Decrement this iterator until reaching the top or a non-debug instr.
 static MachineBasicBlock::const_iterator
 priorNonDebug(MachineBasicBlock::const_iterator I,
               MachineBasicBlock::const_iterator Beg) {
   assert(I != Beg && "reached the top of the region, cannot decrement");
   while (--I != Beg) {
-    if (!I->isDebugInstr())
+    if (!I->isDebugOrPseudoInstr())
       break;
   }
   return I;
 }
 
 /// Non-const version.
 static MachineBasicBlock::iterator
 priorNonDebug(MachineBasicBlock::iterator I,
               MachineBasicBlock::const_iterator Beg) {
   return priorNonDebug(MachineBasicBlock::const_iterator(I), Beg)
       .getNonConstIterator();
 }
 
 /// If this iterator is a debug value, increment until reaching the End or a
 /// non-debug instruction.
 static MachineBasicBlock::const_iterator
 nextIfDebug(MachineBasicBlock::const_iterator I,
             MachineBasicBlock::const_iterator End) {
   for(; I != End; ++I) {
-    if (!I->isDebugInstr())
+    if (!I->isDebugOrPseudoInstr())
       break;
   }
   return I;
 }
 
 /// Non-const version.
 static MachineBasicBlock::iterator
 nextIfDebug(MachineBasicBlock::iterator I,
for(MachineBasicBlock::iterator RegionEnd = MBB->end();
     // The next region starts above the previous region. Look backward in the
     // instruction stream until we find the nearest boundary.
     unsigned NumRegionInstrs = 0;
     I = RegionEnd;
     for (;I != MBB->begin(); --I) {
       MachineInstr &MI = *std::prev(I);
       if (isSchedBoundary(&MI, &*MBB, MF, TII))
         break;
-      if (!MI.isDebugInstr()) {
+      if (!MI.isDebugOrPseudoInstr()) {
         // MBB::size() uses instr_iterator to count. Here we need a bundle to
         // count as a single instruction.
         ++NumRegionInstrs;
       }
     }
 
     // It's possible we found a scheduling region that only has debug
     // instructions. Don't bother scheduling these.

llvm/lib/CodeGen/MachineSink.cpp

do {
     MachineInstr &MI = *I;  // The instruction to sink.
 
     // Predecrement I (if it's not begin) so that it isn't invalidated by
     // sinking.
     ProcessedBegin = I == MBB.begin();
     if (!ProcessedBegin)
       --I;
 
-    if (MI.isDebugInstr()) {
+    if (MI.isDebugOrPseudoInstr()) {
       if (MI.isDebugValue())
         ProcessDbgInst(MI);
       continue;
     }
 
     bool Joined = PerformTrivialForwardCoalescing(MI, &MBB);
     if (Joined) {
       MadeChange = true;
MachineSinking::getBBRegisterPressure(MachineBasicBlock &MBB) {
   // Initialize the register pressure tracker.
   RPTracker.init(MBB.getParent(), &RegClassInfo, nullptr, &MBB, MBB.end(),
                  /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
 
   for (MachineBasicBlock::iterator MII = MBB.instr_end(),
                                    MIE = MBB.instr_begin();
        MII != MIE; --MII) {
     MachineInstr &MI = *std::prev(MII);
-    if (MI.isDebugValue() || MI.isDebugLabel())
+    if (MI.isDebugValue() || MI.isDebugLabel() || MI.isPseudoProbe())
       continue;
     RegisterOperands RegOpers;
     RegOpers.collect(MI, *TRI, *MRI, false, false);
     RPTracker.recedeSkipDebugValues();
     assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
     RPTracker.recede(RegOpers);
   }
 
if (MI->isDebugValue()) {
       }
       if (IsValid) {
         for (auto RegOps : MIUnits)
           SeenDbgInstrs[RegOps.first].push_back({MI, RegOps.second});
       }
       continue;
     }
 
-    if (MI->isDebugInstr())
+    if (MI->isDebugOrPseudoInstr())
       continue;
 
     // Do not move any instruction across function call.
     if (MI->isCall())
       return false;
 
     if (!MI->isCopy() || !MI->getOperand(0).isRenamable()) {
       LiveRegUnits::accumulateUsedDefed(*MI, ModifiedRegUnits, UsedRegUnits,

llvm/lib/CodeGen/RegAllocGreedy.cpp

for (unsigned Number : Blocks) {
     }
 
     assert(B < GroupSize && "Array overflow");
     BCS[B].Number = Number;
 
     // Abort if the spill cannot be inserted at the MBB' start
     MachineBasicBlock *MBB = MF->getBlockNumbered(Number);
     if (!MBB->empty() &&
-        SlotIndex::isEarlierInstr(LIS->getInstructionIndex(MBB->instr_front()),
-                                  SA->getFirstSplitPoint(Number)))
+        SlotIndex::isEarlierInstr(
+            LIS->getInstructionIndex(*MBB->getFirstNonDebugInstr()),
       return false;
     // Interference for the live-in value.
     if (Intf.first() <= Indexes->getMBBStartIdx(Number))
       BCS[B].Entry = SpillPlacement::MustSpill;
     else
       BCS[B].Entry = SpillPlacement::PrefSpill;
 
     // Interference for the live-out value.

llvm/lib/CodeGen/RegisterCoalescer.cpp

do {
     if (!OV.RedefVNI)
       break;
   } while (TaintedLanes.any());
   return true;
 }
 
 bool JoinVals::usesLanes(const MachineInstr &MI, Register Reg, unsigned SubIdx,
                          LaneBitmask Lanes) const {
-  if (MI.isDebugInstr())
+  if (MI.isDebugOrPseudoInstr())
     return false;
   for (const MachineOperand &MO : MI.operands()) {
     if (!MO.isReg() || MO.isDef() || MO.getReg() != Reg)
       continue;
     if (!MO.readsReg())
       continue;
     unsigned S = TRI->composeSubRegIndices(SubIdx, MO.getSubReg());
     if ((Lanes & TRI->getSubRegIndexLaneMask(S)).any())
for (auto &MBB : MF) {
     SlotIndex CurrentSlot = Slots.getMBBStartIdx(&MBB);
 
     for (auto &MI : MBB) {
       if (MI.isDebugValue()) {
         if (any_of(MI.debug_operands(), [](const MachineOperand &MO) {
               return MO.isReg() && MO.getReg().isVirtual();
             }))
           ToInsert.push_back(&MI);
-      } else if (!MI.isDebugInstr()) {
+      } else if (!MI.isDebugOrPseudoInstr()) {
         CurrentSlot = Slots.getInstructionIndex(MI);
         CloseNewDVRange(CurrentSlot);
       }
     }
 
     // Close range of DBG_VALUEs at the end of blocks.
     CloseNewDVRange(Slots.getMBBEndIdx(&MBB));
   }

llvm/lib/CodeGen/RegisterPressure.cpp

 
 /// Recede across the previous instruction. If LiveUses is provided, record any
 /// RegUnits that are made live by the current instruction's uses. This includes
 /// registers that are both defined and used by the instruction.  If a pressure
 /// difference pointer is provided record the changes is pressure caused by this
 /// instruction independent of liveness.
 void RegPressureTracker::recede(const RegisterOperands &RegOpers,
                                 SmallVectorImpl<RegisterMaskPair> *LiveUses) {
-  assert(!CurrPos->isDebugInstr());
+  assert(!CurrPos->isDebugOrPseudoInstr());
 
   // Boost pressure for all dead defs together.
   bumpDeadDefs(RegOpers.DeadDefs);
 
   // Kill liveness at live defs.
   // TODO: consider earlyclobbers?
   for (const RegisterMaskPair &Def : RegOpers.Defs) {
     Register Reg = Def.RegUnit;
void RegPressureTracker::recedeSkipDebugValues() {
   // Open the top of the region using block iterators.
   if (!RequireIntervals && isTopClosed())
     static_cast<RegionPressure&>(P).openTop(CurrPos);
 
   // Find the previous instruction.
   CurrPos = prev_nodbg(CurrPos, MBB->begin());
 
   SlotIndex SlotIdx;
-  if (RequireIntervals && !CurrPos->isDebugInstr())
+  if (RequireIntervals && !CurrPos->isDebugOrPseudoInstr())
     SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
 
   // Open the top of the region using slot indexes.
   if (RequireIntervals && isTopClosed())
     static_cast<IntervalPressure&>(P).openTop(SlotIdx);
 }
 
 void RegPressureTracker::recede(SmallVectorImpl<RegisterMaskPair> *LiveUses) {
   recedeSkipDebugValues();
-  if (CurrPos->isDebugValue()) {
-    // It's possible to only have debug_value instructions and hit the start of
-    // the block.
+  if (CurrPos->isDebugValue() || CurrPos->isPseudoProbe()) {
+    // It's possible to only have debug_value and pseudo probe instructions and
+    // hit the start of the block.
     assert(CurrPos == MBB->begin());
     return;
   }
 
   const MachineInstr &MI = *CurrPos;
   RegisterOperands RegOpers;
   RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false);
   if (TrackLaneMasks) {

 
 /// Record the upward impact of a single instruction on current register
 /// pressure. Unlike the advance/recede pressure tracking interface, this does
 /// not discover live in/outs.
 ///
 /// This is intended for speculative queries. It leaves pressure inconsistent
 /// with the current position, so must be restored by the caller.
 void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
-  assert(!MI->isDebugInstr() && "Expect a nondebug instruction.");
+  assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
 
   SlotIndex SlotIdx;
   if (RequireIntervals)
     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
 
   // Account for register pressure similar to RegPressureTracker::recede().
   RegisterOperands RegOpers;
   RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/true);

 
 /// Record the downward impact of a single instruction on current register
 /// pressure. Unlike the advance/recede pressure tracking interface, this does
 /// not discover live in/outs.
 ///
 /// This is intended for speculative queries. It leaves pressure inconsistent
 /// with the current position, so must be restored by the caller.
 void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
-  assert(!MI->isDebugInstr() && "Expect a nondebug instruction.");
+  assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");
 
   SlotIndex SlotIdx;
   if (RequireIntervals)
     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
 
   // Account for register pressure similar to RegPressureTracker::recede().
   RegisterOperands RegOpers;
   RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, false);

llvm/lib/CodeGen/RegisterScavenging.cpp

void RegScavenger::forward() {
   for (ScavengedInfo &I : Scavenged) {
     if (I.Restore != &MI)
       continue;
 
     I.Reg = 0;
     I.Restore = nullptr;
   }
 
-  if (MI.isDebugInstr())
+  if (MI.isDebugOrPseudoInstr())
     return;
 
   determineKillsAndDefs();
 
   // Verify uses and defs.
 #ifndef NDEBUG
   for (const MachineOperand &MO : MI.operands()) {
     if (!MO.isReg())
Register RegScavenger::findSurvivorReg(MachineBasicBlock::iterator StartMI,
 
   MachineBasicBlock::iterator ME = MBB->getFirstTerminator();
   assert(StartMI != ME && "MI already at terminator");
   MachineBasicBlock::iterator RestorePointMI = StartMI;
   MachineBasicBlock::iterator MI = StartMI;
 
   bool inVirtLiveRange = false;
   for (++MI; InstrLimit > 0 && MI != ME; ++MI, --InstrLimit) {
-    if (MI->isDebugInstr()) {
+    if (MI->isDebugOrPseudoInstr()) {
       ++InstrLimit; // Don't count debug instructions
       continue;
     }
     bool isVirtKillInsn = false;
     bool isVirtDefInsn = false;
     // Remove any candidates touched by instruction.
     for (const MachineOperand &MO : MI->operands()) {
       if (MO.isRegMask())

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

 /// Adds a register data dependency if the instruction that defines the
 /// virtual register used at OperIdx is mapped to an SUnit. Add a register
 /// antidependency from this SUnit to instructions that occur later in the same
 /// scheduling region if they write the virtual register.
 ///
 /// TODO: Handle ExitSU "uses" properly.
 void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) {
   const MachineInstr *MI = SU->getInstr();
-  assert(!MI->isDebugInstr());
+  assert(!MI->isDebugOrPseudoInstr());
 
   const MachineOperand &MO = MI->getOperand(OperIdx);
   Register Reg = MO.getReg();
 
   // Remember the use. Data dependencies will be added when we find the def.
   LaneBitmask LaneMask = TrackLaneMasks ? getLaneMaskForMO(MO)
                                         : LaneBitmask::getAll();
   CurrentVRegUses.insert(VReg2SUnitOperIdx(Reg, LaneMask, OperIdx, SU));

 /// MachineScheduler relies on initSUnits numbering the nodes by their order in
 /// the original instruction list.
 void ScheduleDAGInstrs::initSUnits() {
   // We'll be allocating one SUnit for each real instruction in the region,
   // which is contained within a basic block.
   SUnits.reserve(NumRegionInstrs);
 
   for (MachineInstr &MI : make_range(RegionBegin, RegionEnd)) {
-    if (MI.isDebugInstr())
+    if (MI.isDebugOrPseudoInstr())
       continue;
 
     SUnit *SU = newSUnit(&MI);
     MISUnitMap[&MI] = SU;
 
     SU->isCall = MI.isCall();
     SU->isCommutable = MI.isCommutable();
 
for (MachineBasicBlock::iterator MII = RegionEnd, MIE = RegionBegin;
 
     if (MI.isDebugValue() || MI.isDebugRef()) {
       DbgMI = &MI;
       continue;
     }
     if (MI.isDebugLabel())
       continue;
 
+    if (MI.isPseudoProbe())
+      continue;
+
     SUnit *SU = MISUnitMap[&MI];
     assert(SU && "No SUnit mapped to this MI");
 
     if (RPTracker) {
       RegisterOperands RegOpers;
       RegOpers.collect(MI, *TRI, MRI, TrackLaneMasks, false);
       if (TrackLaneMasks) {
         SlotIndex SlotIdx = LIS->getInstructionIndex(MI);

 void ScheduleDAGInstrs::fixupKills(MachineBasicBlock &MBB) {
   LLVM_DEBUG(dbgs() << "Fixup kills for " << printMBBReference(MBB) << '\n');
 
   LiveRegs.init(*TRI);
   LiveRegs.addLiveOuts(MBB);
 
   // Examine block from end to start...
   for (MachineInstr &MI : make_range(MBB.rbegin(), MBB.rend())) {
-    if (MI.isDebugInstr())
+    if (MI.isDebugOrPseudoInstr())
       continue;
 
     // Update liveness.  Registers that are defed but not used in this
     // instruction are now dead. Mark register and all subregs as they
     // are completely defined.
     for (ConstMIBundleOperands O(MI); O.isValid(); ++O) {
       const MachineOperand &MO = *O;
       if (MO.isReg()) {
if (!MI.isBundled()) {
 
       // Some targets make the (questionable) assumtion that the instructions
       // inside the bundle are ordered and consequently only the last use of
       // a register inside the bundle can kill it.
       MachineBasicBlock::instr_iterator I = std::next(Bundle);
       while (I->isBundledWithSucc())
         ++I;
       do {
-        if (!I->isDebugInstr())
+        if (!I->isDebugOrPseudoInstr())
           toggleKills(MRI, LiveRegs, *I, true);
         --I;
       } while (I != Bundle);
     }
   }
 }
 
 void ScheduleDAGInstrs::dumpNode(const SUnit &SU) const {

llvm/lib/CodeGen/SlotIndexes.cpp

bool SlotIndexes::runOnMachineFunction(MachineFunction &fn) {
   indexList.push_back(createEntry(nullptr, index));
 
   // Iterate over the function.
   for (MachineBasicBlock &MBB : *mf) {
     // Insert an index for the MBB start.
     SlotIndex blockStartIndex(&indexList.back(), SlotIndex::Slot_Block);
 
     for (MachineInstr &MI : MBB) {
-      if (MI.isDebugInstr())
+      if (MI.isDebugOrPseudoInstr())
         continue;
 
       // Insert a store index for the instr.
       indexList.push_back(createEntry(&MI, index += SlotIndex::InstrDist));
 
       // Save this base index in the maps.
       mi2iMap.insert(std::make_pair(
           &MI, SlotIndex(&indexList.back(), SlotIndex::Slot_Block)));
while (ListI != ListB || MBBI != Begin || (includeStart && !pastStart)) {
     }
   }
 
   // In theory this could be combined with the previous loop, but it is tricky
   // to update the IndexList while we are iterating it.
   for (MachineBasicBlock::iterator I = End; I != Begin;) {
     --I;
     MachineInstr &MI = *I;
-    if (!MI.isDebugInstr() && mi2iMap.find(&MI) == mi2iMap.end())
+    if (!MI.isDebugOrPseudoInstr() && mi2iMap.find(&MI) == mi2iMap.end())
       insertMachineInstrInMaps(MI);
   }
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void SlotIndexes::dump() const {
   for (const IndexListEntry &ILE : indexList) {
     dbgs() << ILE.getIndex() << " ";

llvm/lib/CodeGen/SplitKit.cpp

for (const VNInfo *C : Copies) {
     SlotIndex Def = C->def;
     MachineInstr *MI = LIS.getInstructionFromIndex(Def);
     assert(MI && "No instruction for back-copy");
 
     MachineBasicBlock *MBB = MI->getParent();
     MachineBasicBlock::iterator MBBI(MI);
     bool AtBegin;
     do AtBegin = MBBI == MBB->begin();
-    while (!AtBegin && (--MBBI)->isDebugInstr());
+    while (!AtBegin && (--MBBI)->isDebugOrPseudoInstr());
 
     LLVM_DEBUG(dbgs() << "Removing " << Def << '\t' << *MI);
     LIS.removeVRegDefAt(*LI, Def);
     LIS.RemoveMachineInstrFromMaps(*MI);
     MI->eraseFromParent();
 
     // Adjust RegAssign if a register assignment is killed at Def. We want to
     // avoid calculating the live range of the source register if possible.

llvm/test/Transforms/SampleProfile/pseudo-probe-slotindex.ll

+; REQUIRES: x86_64-linux
+; RUN: llc -print-after=slotindexes -stop-after=slotindexes -mtriple=x86_64-- %s -filetype=asm -o %t 2>&1 | FileCheck %s
+
+define void @foo(i32* %p) {
+  store i32 0, i32* %p
+  call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+  store i32 0, i32* %p
+  ret void
+}
+
+;; Check the pseudo probe instruction isn't assigned a slot index.
+;CHECK: IR Dump {{.*}}
+;CHECK: # Machine code for function foo{{.*}}
+;CHECK: {{[0-9]+}}B  bb.0 (%ir-block.0)
+;CHECK: {{[0-9]+}}B  %0:gr64 = COPY killed $rdi
+;CHECK: {{^}}        PSEUDO_PROBE 5116412291814990879
+;CHECK: {{[0-9]+}}B  MOV32mi
+;CHECK: {{[0-9]+}}B  RET 0
+
+declare void @llvm.pseudoprobe(i64, i64, i32, i64) #0
+
+attributes #0 = { inaccessiblememonly nounwind willreturn }
 No newline at end of file