MachineScheduler: Allow tracking of subregister lane masks.

[MatzeB]

Summary

Before this patch the machine scheduler was unable to reorder sequences like the following:

%vreg42:sub0 = ...
%vreg42:sub1 = ...

This patch series enables this. It is however disabled by default. Targets that benefit from this (probably R600) can enable it by change the "shouldTrackLaneMasks" flag in the SchedulingPolicy/Strategy.

This patch requires http://reviews.llvm.org/D9067 and http://reviews.llvm.org/D9068 to be applied first.

About the patch:

Tracking lanemasks allows the scheduler to differentiate between
different subregister defs and schedule them independently even when
they go into the same vreg.
For this to work we have to take care that the subregister defs count as
register pressure neutral except for the first one. We also have to attach the
reads-undef marker to the first subregister def and remove it from all
others.

[MatzeB]

Abandoning in favor of http://reviews.llvm.org/D14969 (which has the same goal but works differently internally)

include/llvm/CodeGen/MachineInstr.h

bool addRegisterDead(unsigned Reg, const TargetRegisterInfo *RegInfo,
                        bool AddIfNotFound = false);
 
   /// Clear all dead flags on operands defining register @p Reg.
   void clearRegisterDeads(unsigned Reg);
 
   /// Mark all subregister defs of register @p Reg with the undef flag.
   /// This function is used when we determined to have a subregister def in an
   /// otherwise undefined super register.
-  void addRegisterDefReadUndef(unsigned Reg);
+  void setRegisterDefReadUndef(unsigned Reg, bool IsUndef = true);
 
   /// addRegisterDefined - We have determined MI defines a register. Make sure
   /// there is an operand defining Reg.
   void addRegisterDefined(unsigned Reg,
                           const TargetRegisterInfo *RegInfo = nullptr);
 
   /// setPhysRegsDeadExcept - Mark every physreg used by this instruction as
   /// dead except those in the UsedRegs list.

include/llvm/CodeGen/MachineScheduler.h

 class ScheduleDAGMI;
 
 /// Define a generic scheduling policy for targets that don't provide their own
 /// MachineSchedStrategy. This can be overriden for each scheduling region
 /// before building the DAG.
 struct MachineSchedPolicy {
   // Allow the scheduler to disable register pressure tracking.
   bool ShouldTrackPressure;
+  // Allow the schedule to track subregister defs to different lanes
+  // independently.
+  bool ShouldTrackLaneMasks;
 
   // Allow the scheduler to force top-down or bottom-up scheduling. If neither
   // is true, the scheduler runs in both directions and converges.
   bool OnlyTopDown;
   bool OnlyBottomUp;
 
-  MachineSchedPolicy(): ShouldTrackPressure(false), OnlyTopDown(false),
-    OnlyBottomUp(false) {}
+  MachineSchedPolicy(): ShouldTrackPressure(false), ShouldTrackLaneMasks(false),
+    OnlyTopDown(false), OnlyBottomUp(false) {}
 };
 
 /// MachineSchedStrategy - Interface to the scheduling algorithm used by
 /// ScheduleDAGMI.
 ///
 /// Initialization sequence:
 ///   initPolicy -> shouldTrackPressure -> initialize(DAG) -> registerRoots
 class MachineSchedStrategy {
   virtual void anchor();
 public:
   virtual ~MachineSchedStrategy() {}
 
   /// Optionally override the per-region scheduling policy.
   virtual void initPolicy(MachineBasicBlock::iterator Begin,
                           MachineBasicBlock::iterator End,
                           unsigned NumRegionInstrs) {}
 
   /// Check if pressure tracking is needed before building the DAG and
   /// initializing this strategy. Called after initPolicy.
   virtual bool shouldTrackPressure() const { return true; }
 
+  /// Check if lane masks should be tracked to schedule subregister defs of the
+  /// same vreg independently. Called after initPolicy().
+  virtual bool shouldTrackLaneMasks() const { return false; }
+
   /// Initialize the strategy after building the DAG for a new region.
   virtual void initialize(ScheduleDAGMI *DAG) = 0;
 
   /// Notify this strategy that all roots have been released (including those
   /// that depend on EntrySU or ExitSU).
   virtual void registerRoots() {}
 
   /// Pick the next node to schedule, or return NULL. Set IsTopNode to true to
protected:
   void scheduleMI(SUnit *SU, bool IsTopNode);
 
   // Lesser helpers...
 
   void initRegPressure();
 
   void updatePressureDiffs(ArrayRef<unsigned> LiveUses);
 
+  void updateFirstDef(const VRegInfo &Info, const MachineInstr *ToBeScheduled);
+
   void updateScheduledPressure(const SUnit *SU,
                                const std::vector<unsigned> &NewMaxPressure);
 };
 
 //===----------------------------------------------------------------------===//
 ///
 /// Helpers for implementing custom MachineSchedStrategy classes. These take
 /// care of the book-keeping associated with list scheduling heuristics.
public:
   void initPolicy(MachineBasicBlock::iterator Begin,
                   MachineBasicBlock::iterator End,
                   unsigned NumRegionInstrs) override;
 
   bool shouldTrackPressure() const override {
     return RegionPolicy.ShouldTrackPressure;
   }
 
+  bool shouldTrackLaneMasks() const override {
+    return RegionPolicy.ShouldTrackLaneMasks;
+  }
+
   void initialize(ScheduleDAGMI *dag) override;
 
   SUnit *pickNode(bool &IsTopNode) override;
 
   void schedNode(SUnit *SU, bool IsTopNode) override;
 
   void releaseTopNode(SUnit *SU) override {
     Top.releaseTopNode(SU);

include/llvm/CodeGen/RegisterPressure.h

public:
   void setPos(MachineBasicBlock::const_iterator Pos) { CurrPos = Pos; }
 
   /// \brief Get the SlotIndex for the first nondebug instruction including or
   /// after the current position.
   SlotIndex getCurrSlot() const;
 
   /// Recede across the previous instruction.
   bool recede(SmallVectorImpl<unsigned> *LiveUses = nullptr,
+              SmallVectorImpl<unsigned> *Defs = nullptr,
               PressureDiff *PDiff = nullptr);
 
   /// Advance across the current instruction.
-  bool advance();
+  bool advance(SmallVectorImpl<unsigned> *Defs = nullptr);
 
   /// Finalize the region boundaries and recored live ins and live outs.
   void closeRegion();
 
   /// Initialize the LiveThru pressure set based on the untied defs found in
   /// RPTracker.
   void initLiveThru(const RegPressureTracker &RPTracker);
 

include/llvm/CodeGen/ScheduleDAGInstrs.h

struct VReg2SUnit {
     VReg2SUnit(unsigned VReg, unsigned LaneMask, SUnit *SU)
       : VirtReg(VReg), LaneMask(LaneMask), SU(SU) {}
 
     unsigned getSparseSetIndex() const {
       return TargetRegisterInfo::virtReg2Index(VirtReg);
     }
   };
 
+  struct VRegInfo {
+    unsigned VirtReg;
+    unsigned NumUnscheduledDefSUs : 31;
+    bool     LiveIn               :  1;
+
+    VRegInfo(unsigned VReg)
+      : VirtReg(VReg), NumUnscheduledDefSUs(0), LiveIn(false) {}
+
+    unsigned getSparseSetIndex() const {
+      return TargetRegisterInfo::virtReg2Index(VirtReg);
+    }
+  };
+
   /// Record a physical register access.
   /// For non-data-dependent uses, OpIdx == -1.
   struct PhysRegSUOper {
     SUnit *SU;
     int OpIdx;
     unsigned Reg;
 
     PhysRegSUOper(SUnit *su, int op, unsigned R): SU(su), OpIdx(op), Reg(R) {}
namespace llvm {
   /// require a destructor.
   typedef SparseSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMap;
 
   /// Track local uses of virtual registers. These uses are gathered by the DAG
   /// builder and may be consulted by the scheduler to avoid iterating an entire
   /// vreg use list.
   typedef SparseMultiSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMultiMap;
 
+  typedef SparseSet<VRegInfo, VirtReg2IndexFunctor> VRegInfoMap;
+
   /// ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of
   /// MachineInstrs.
   class ScheduleDAGInstrs : public ScheduleDAG {
   protected:
     const MachineLoopInfo *MLI;
     const MachineFrameInfo *MFI;
 
     /// TargetSchedModel provides an interface to the machine model.
protected:
     /// After calling BuildSchedGraph, each machine instruction in the current
     /// scheduling region is mapped to an SUnit.
     DenseMap<MachineInstr*, SUnit*> MISUnitMap;
 
     /// After calling BuildSchedGraph, each vreg used in the scheduling region
     /// is mapped to a set of SUnits. These include all local vreg uses, not
     /// just the uses for a singly defined vreg.
     VReg2SUnitMultiMap VRegUses;
+    /// After calling BuildSchedGraph, each vreg def in the scheduling region
+    /// is mapped to a set of SUnits.
+    VReg2SUnitMultiMap VRegDefs;
+
+    /// Information (mostly the number of unscheduled defs left) about a vreg.
+    VRegInfoMap VRegInfos;
 
     /// State internal to DAG building.
     /// -------------------------------
 
     /// Defs, Uses - Remember where defs and uses of each register are as we
     /// iterate upward through the instructions. This is allocated here instead
     /// of inside BuildSchedGraph to avoid the need for it to be initialized and
     /// destructed for each block.

lib/CodeGen/LiveIntervalAnalysis.cpp

for (auto VNI : LI.valnos) {
     LiveRange::iterator I = LI.FindSegmentContaining(Def);
     assert(I != LI.end() && "Missing segment for VNI");
 
     // Is the register live before? Otherwise we may have to add a read-undef
     // flag for subregister defs.
     if (MRI->shouldTrackSubRegLiveness(LI.reg)) {
       if ((I == LI.begin() || std::prev(I)->end < Def) && !VNI->isPHIDef()) {
         MachineInstr *MI = getInstructionFromIndex(Def);
-        MI->addRegisterDefReadUndef(LI.reg);
+        MI->setRegisterDefReadUndef(LI.reg);
       }
     }
 
     if (I->end != Def.getDeadSlot())
       continue;
     if (VNI->isPHIDef()) {
       // This is a dead PHI. Remove it.
       VNI->markUnused();

lib/CodeGen/MachineInstr.cpp

 void MachineInstr::clearRegisterDeads(unsigned Reg) {
   for (MachineOperand &MO : operands()) {
     if (!MO.isReg() || !MO.isDef() || MO.getReg() != Reg)
       continue;
     MO.setIsDead(false);
   }
 }
 
-void MachineInstr::addRegisterDefReadUndef(unsigned Reg) {
+void MachineInstr::setRegisterDefReadUndef(unsigned Reg, bool IsUndef) {
   for (MachineOperand &MO : operands()) {
     if (!MO.isReg() || !MO.isDef() || MO.getReg() != Reg || MO.getSubReg() == 0)
       continue;
-    MO.setIsUndef();
+    MO.setIsUndef(IsUndef);
   }
 }
 
 void MachineInstr::addRegisterDefined(unsigned Reg,
                                       const TargetRegisterInfo *RegInfo) {
   if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
     MachineOperand *MO = findRegisterDefOperand(Reg, false, RegInfo);
     if (MO)

lib/CodeGen/MachineScheduler.cpp

void ScheduleDAGMILive::enterRegion(MachineBasicBlock *bb,
   ScheduleDAGMI::enterRegion(bb, begin, end, regioninstrs);
 
   // For convenience remember the end of the liveness region.
   LiveRegionEnd = (RegionEnd == bb->end()) ? RegionEnd : std::next(RegionEnd);
 
   SUPressureDiffs.clear();
 
   ShouldTrackPressure = SchedImpl->shouldTrackPressure();
+  TrackLaneMasks = SchedImpl->shouldTrackLaneMasks();
 }
 
 // Setup the register pressure trackers for the top scheduled top and bottom
 // scheduled regions.
 void ScheduleDAGMILive::initRegPressure() {
   TopRPTracker.init(&MF, RegClassInfo, LIS, BB, RegionBegin);
   BotRPTracker.init(&MF, RegClassInfo, LIS, BB, LiveRegionEnd);
 
if (RegionPressure[i] > Limit) {
       RegionCriticalPSets.push_back(PressureChange(i));
     }
   }
   DEBUG(dbgs() << "Excess PSets: ";
         for (unsigned i = 0, e = RegionCriticalPSets.size(); i != e; ++i)
           dbgs() << TRI->getRegPressureSetName(
             RegionCriticalPSets[i].getPSet()) << " ";
         dbgs() << "\n");
+
+  if (TrackLaneMasks) {
+    // Set read-undef flag for VRegs with exactly one (subregister) def.
+    for (const VRegInfo &Info : VRegInfos)
+      updateFirstDef(Info, nullptr);
+  }
 }
 
 void ScheduleDAGMILive::
 updateScheduledPressure(const SUnit *SU,
                         const std::vector<unsigned> &NewMaxPressure) {
   const PressureDiff &PDiff = getPressureDiff(SU);
   unsigned CritIdx = 0, CritEnd = RegionCriticalPSets.size();
   for (PressureDiff::const_iterator I = PDiff.begin(), E = PDiff.end();
for (VReg2SUnitMultiMap::iterator
           = LI.Query(LIS->getInstructionIndex(SU->getInstr()));
         if (LRQ.valueIn() == VNI)
           getPressureDiff(SU).addPressureChange(Reg, true, &MRI);
       }
     }
   }
 }
 
+/// Register pressure is tracked for full vregs. This means a subregister def
+/// does not change the register pressure as it appears to be reading+writing
+/// the vreg, except for the first which just writes the vreg. This means we
+/// have to change the pressure diffs as soon as we only have 1 def left to
+/// schedule and know it will become the first.
+/// If @p ToBeScheduled is given this node is considered as already scheduled.
+void ScheduleDAGMILive::updateFirstDef(const VRegInfo &Info,
+                                       const MachineInstr *ToBeScheduled) {
+  if (Info.LiveIn || Info.NumUnscheduledDefSUs != 1)
+    return;
+
+  unsigned VReg = Info.VirtReg;
+#ifndef NDEBUG
+  unsigned Unscheduled = 0;
+  for (const VReg2SUnit &V2SU
+       : make_range(VRegDefs.find(VReg), VRegDefs.end())) {
+    if (!V2SU.SU->isScheduled && V2SU.SU->getInstr() != ToBeScheduled)
+      ++Unscheduled;
+  }
+  assert(Unscheduled == 1 &&
+         "There should be exactly one other unscheduled instruction left");
+#endif
+  // There should be exactly one other unscheduled MI left which needs its
+  // pressure changed.
+  for (const VReg2SUnit &V2SU
+       : make_range(VRegDefs.find(VReg), VRegDefs.end())) {
+    SUnit *SU = V2SU.SU;
+    if (SU->isScheduled)
+      continue;
+    MachineInstr &MI = *SU->getInstr();
+    if (&MI == ToBeScheduled)
+      continue;
+    // The remaining def will be the first in line. This means it does not read
+    // earlier values: Change the pressure diff and add the read-undef flag.
+    getPressureDiff(SU).addPressureChange(VReg, false, &MRI);
+    MI.setRegisterDefReadUndef(VReg, true);
+    DEBUG(dbgs() << "  SU(" << SU->NodeNum << ") will be first def of "
+          << PrintVRegOrUnit(VReg, TRI) << ": " << MI);
+    break;
+  }
+}
+
 /// schedule - Called back from MachineScheduler::runOnMachineFunction
 /// after setting up the current scheduling region. [RegionBegin, RegionEnd)
 /// only includes instructions that have DAG nodes, not scheduling boundaries.
 ///
 /// This is a skeletal driver, with all the functionality pushed into helpers,
 /// so that it can be easilly extended by experimental schedulers. Generally,
 /// implementing MachineSchedStrategy should be sufficient to implement a new
 /// scheduling algorithm. However, if a scheduler further subclasses
if (&*CurrentTop == MI)
       CurrentTop = nextIfDebug(++CurrentTop, CurrentBottom);
     else {
       moveInstruction(MI, CurrentTop);
       TopRPTracker.setPos(MI);
     }
 
     if (ShouldTrackPressure) {
       // Update top scheduled pressure.
-      TopRPTracker.advance();
+      SmallVector<unsigned, 2> Defs;
+      TopRPTracker.advance(&Defs);
       assert(TopRPTracker.getPos() == CurrentTop && "out of sync");
       updateScheduledPressure(SU, TopRPTracker.getPressure().MaxSetPressure);
+
+      if (TrackLaneMasks) {
+        // If we schedule the first of a series of subregister defs, then we
+        // have to mark it with the read-undef flag.
+        for (unsigned Reg : Defs) {
+          if (!TRI->isVirtualRegister(Reg))
+            continue;
+
+          VRegInfo &Info = *VRegInfos.find(Reg);
+          if (Info.LiveIn)
+            continue;
+          MI->setRegisterDefReadUndef(Reg, true);
+          Info.LiveIn = true;
+        }
+      }
     }
   }
   else {
     assert(SU->isBottomReady() && "node still has unscheduled dependencies");
     MachineBasicBlock::iterator priorII =
       priorNonDebug(CurrentBottom, CurrentTop);
     if (&*priorII == MI)
       CurrentBottom = priorII;
     else {
       if (&*CurrentTop == MI) {
         CurrentTop = nextIfDebug(++CurrentTop, priorII);
         TopRPTracker.setPos(CurrentTop);
       }
       moveInstruction(MI, CurrentBottom);
       CurrentBottom = MI;
     }
     if (ShouldTrackPressure) {
       // Update bottom scheduled pressure.
       SmallVector<unsigned, 8> LiveUses;
-      BotRPTracker.recede(&LiveUses);
+      SmallVector<unsigned, 2> Defs;
+      BotRPTracker.recede(&LiveUses, &Defs);
       assert(BotRPTracker.getPos() == CurrentBottom && "out of sync");
       updateScheduledPressure(SU, BotRPTracker.getPressure().MaxSetPressure);
       updatePressureDiffs(LiveUses);
+
+      if (TrackLaneMasks) {
+        for (unsigned Reg : Defs) {
+          if (!TRI->isVirtualRegister(Reg))
+            continue;
+
+          VRegInfo &Info = *VRegInfos.find(Reg);
+          assert(Info.NumUnscheduledDefSUs > 0);
+          --Info.NumUnscheduledDefSUs;
+          updateFirstDef(Info, MI);
+        }
+      }
     }
   }
 }
 
 //===----------------------------------------------------------------------===//
 // LoadClusterMutation - DAG post-processing to cluster loads.
 //===----------------------------------------------------------------------===//
 

lib/CodeGen/RegisterPressure.cpp

 }
 
 /// Recede across the previous instruction. If LiveUses is provided, record any
 /// RegUnits that are made live by the current instruction's uses. This includes
 /// registers that are both defined and used by the instruction.  If a pressure
 /// difference pointer is provided record the changes is pressure caused by this
 /// instruction independent of liveness.
 bool RegPressureTracker::recede(SmallVectorImpl<unsigned> *LiveUses,
+                                SmallVectorImpl<unsigned> *Defs,
                                 PressureDiff *PDiff) {
   // Check for the top of the analyzable region.
   if (CurrPos == MBB->begin()) {
     closeRegion();
     return false;
   }
   if (!isBottomClosed())
     closeBottom();
bool RegPressureTracker::recede(SmallVectorImpl<unsigned> *LiveUses,
   // Boost pressure for all dead defs together.
   increaseRegPressure(RegOpers.DeadDefs);
   decreaseRegPressure(RegOpers.DeadDefs);
 
   // Kill liveness at live defs.
   // TODO: consider earlyclobbers?
   for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
     unsigned Reg = RegOpers.Defs[i];
+    if (Defs != nullptr && !containsReg(*Defs, Reg))
+      Defs->push_back(Reg);
+
     bool DeadDef = false;
     if (RequireIntervals) {
       const LiveRange *LR = getLiveRange(Reg);
       if (LR) {
         LiveQueryResult LRQ = LR->Query(SlotIdx);
         DeadDef = LRQ.isDeadDef();
       }
     }
for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
       if (TargetRegisterInfo::isVirtualRegister(Reg) && !LiveRegs.contains(Reg))
         UntiedDefs.insert(Reg);
     }
   }
   return true;
 }
 
 /// Advance across the current instruction.
-bool RegPressureTracker::advance() {
+bool RegPressureTracker::advance(SmallVectorImpl<unsigned> *Defs) {
   assert(!TrackUntiedDefs && "unsupported mode");
 
   // Check for the bottom of the analyzable region.
   if (CurrPos == MBB->end()) {
     closeRegion();
     return false;
   }
   if (!isTopClosed())
for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
     }
     else if (!lastUse && !isLive)
       increaseRegPressure(Reg);
   }
 
   // Generate liveness for defs.
   for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
     unsigned Reg = RegOpers.Defs[i];
+    if (Defs != nullptr && !containsReg(*Defs, Reg))
+      Defs->push_back(Reg);
     if (LiveRegs.insert(Reg))
       increaseRegPressure(Reg);
   }
 
   // Boost pressure for all dead defs together.
   increaseRegPressure(RegOpers.DeadDefs);
   decreaseRegPressure(RegOpers.DeadDefs);
 

lib/CodeGen/ScheduleDAGInstrs.cpp

 
 /// addVRegDefDeps - Add register output and data dependencies from this SUnit
 /// to instructions that occur later in the same scheduling region if they read
 /// from or write to the virtual register defined at OperIdx.
 ///
 /// TODO: Hoist loop induction variable increments. This has to be
 /// reevaluated. Generally, IV scheduling should be done before coalescing.
 void ScheduleDAGInstrs::addVRegDefDeps(SUnit *SU, unsigned OperIdx) {
-  const MachineInstr *MI = SU->getInstr();
-  const MachineOperand &MO = MI->getOperand(OperIdx);
+  MachineInstr *MI = SU->getInstr();
+  MachineOperand &MO = MI->getOperand(OperIdx);
   unsigned Reg = MO.getReg();
   unsigned DefLaneMask = getLaneMaskForMO(MO);
+  // Remember the isUndef flag, as we may reset it in a bit.
+  bool IsKill = MO.getSubReg() == 0 || MO.isUndef();
 
+  VRegInfo *RegInfo = nullptr;
+  if (TrackLaneMasks) {
+    auto IteratorBoolPair = VRegInfos.insert(VRegInfo(Reg));
+    RegInfo = &*(IteratorBoolPair.first);
+    RegInfo->LiveIn = !IsKill;
+    // Clear undef flag, we'll re-add it later once we know which Def is first.
+    MO.setIsUndef(false);
+  }
+
+  // Record this local VReg Def.
   if (MO.isDead()) {
     assert(CurrentVRegUses.find(Reg) == CurrentVRegUses.end() &&
            "Dead defs should have no uses");
   } else {
+    if (TrackLaneMasks) {
+      VReg2SUnitMultiMap::iterator UI = VRegDefs.find(Reg);
+      for (; UI != VRegDefs.end(); ++UI) {
+        if (UI->SU == SU)
+          break;
+      }
+      if (UI == VRegDefs.end()) {
+        VRegDefs.insert(VReg2SUnit(Reg, 0, SU));
+        ++RegInfo->NumUnscheduledDefSUs;
+      }
+    }
+
     // If we have a <read-undef> flag, none of the lane values comes from an
     // earlier instruction.
-    unsigned KillLaneMask = MO.isUndef() ? ~0u : DefLaneMask;
+    unsigned KillLaneMask = IsKill ? ~0u : DefLaneMask;
     // Add data dependence to all uses we found so far.
     const TargetSubtargetInfo &ST = MF.getSubtarget();
     for (VReg2SUnitMultiMap::iterator I = CurrentVRegUses.find(Reg),
          E = CurrentVRegUses.end(); I != E; /*empty*/) {
       // Ignore uses of other lanes.
       if ((I->LaneMask & KillLaneMask) == 0) {
         ++I;
         continue;
void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
   assert(CurrentVRegDefs.empty() && "nobody else should use CurrentVRegDefs");
   assert(CurrentVRegUses.empty() && "nobody else should use CurrentVRegUses");
   unsigned NumVirtRegs = MRI.getNumVirtRegs();
   CurrentVRegDefs.setUniverse(NumVirtRegs);
   CurrentVRegUses.setUniverse(NumVirtRegs);
 
   VRegUses.clear();
   VRegUses.setUniverse(NumVirtRegs);
+  VRegDefs.clear();
+  VRegInfos.clear();
+  if (TrackLaneMasks) {
+    VRegDefs.setUniverse(NumVirtRegs);
+    VRegInfos.setUniverse(NumVirtRegs);
+  }
 
   // Model data dependencies between instructions being scheduled and the
   // ExitSU.
   addSchedBarrierDeps();
 
   // Walk the list of instructions, from bottom moving up.
   MachineInstr *DbgMI = nullptr;
   for (MachineBasicBlock::iterator MII = RegionEnd, MIE = RegionBegin;
        MII != MIE; --MII) {
     MachineInstr *MI = std::prev(MII);
     if (MI && DbgMI) {
       DbgValues.push_back(std::make_pair(DbgMI, MI));
       DbgMI = nullptr;
     }
 
     if (MI->isDebugValue()) {
       DbgMI = MI;
       continue;
     }
     SUnit *SU = MISUnitMap[MI];
     assert(SU && "No SUnit mapped to this MI");
 
-    if (RPTracker) {
-      PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : nullptr;
-      RPTracker->recede(/*LiveUses=*/nullptr, PDiff);
-      assert(RPTracker->getPos() == std::prev(MII) &&
-             "RPTracker can't find MI");
-    }
-
     assert(
         (CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) &&
         "Cannot schedule terminators or labels!");
 
     // Add register-based dependencies (data, anti, and output).
     bool HasVRegDef = false;
     for (unsigned j = 0, n = MI->getNumOperands(); j != n; ++j) {
       const MachineOperand &MO = MI->getOperand(j);
new_alias_chain:
         // Add dependencies on alias and barrier chains, if needed.
         if (MayAlias && AliasChain)
           addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain,
                              RejectMemNodes);
         if (BarrierChain)
           BarrierChain->addPred(SDep(SU, SDep::Barrier));
       }
     }
+
+    if (RPTracker) {
+      PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : nullptr;
+      RPTracker->recede(/*LiveUses=*/nullptr, /*Defs=*/nullptr, PDiff);
+      assert(RPTracker->getPos() == std::prev(MII) &&
+             "RPTracker can't find MI");
+    }
   }
   if (DbgMI)
     FirstDbgValue = DbgMI;
 
   Defs.clear();
   Uses.clear();
   CurrentVRegDefs.clear();
   CurrentVRegUses.clear();