Let targets to adjust operand latency of bundles

[rampitec]

Summary

This reverts the AMDGPU DAG mutation implemented in D72487 and gives
a more general way of adjusting BUNDLE operand latency.

It also replaces FixBundleLatencyMutation with adjustSchedDependency
callback in the AMDGPU, fixing not only successor latencies but
predecessors' as well.

[foad]

LGTM.

I'd prefer to call adjustSchedDependency unconditionally, but we can do that later.

Grammar nit: remove "to" from the summary. It's either "let them eat cake" or "allow them to eat cake".

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

for (Reg2SUnitsMap::iterator I = Uses.find(*Alias); I != Uses.end(); ++I) {
           (RegUse ? &UseSU->getInstr()->getDesc() : nullptr);
       bool ImplicitPseudoUse =
           (UseMIDesc && UseOp >= ((int)UseMIDesc->getNumOperands()) &&
            !UseMIDesc->hasImplicitUseOfPhysReg(*Alias));
       if (!ImplicitPseudoDef && !ImplicitPseudoUse) {
         Dep.setLatency(SchedModel.computeOperandLatency(SU->getInstr(), OperIdx,
                                                         RegUse, UseOp));
         ST.adjustSchedDependency(SU, UseSU, Dep);
-      } else
+      } else {
         Dep.setLatency(0);
+        // FIXME: We could always let target to adjustSchedDependency(), and
+        // remove this condition, but that currently asserts in Hexagon BE.
+        if (SU->getInstr()->isBundle() || (RegUse && RegUse->isBundle()))
+          ST.adjustSchedDependency(SU, UseSU, Dep);
+      }
 
       UseSU->addPred(Dep);
     }
   }
 }
 
 /// Adds register dependencies (data, anti, and output) from this SUnit
 /// to following instructions in the same scheduling region that depend the

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h

unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
     return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
   }
 
   /// \returns Minimum number of waves per execution unit supported by the
   /// subtarget.
   unsigned getMinWavesPerEU() const override {
     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
   }
+
+  void adjustSchedDependency(SUnit *Src, SUnit *Dst, SDep &Dep) const override;
 };
 
 class R600Subtarget final : public R600GenSubtargetInfo,
                             public AMDGPUSubtarget {
 private:
   R600InstrInfo InstrInfo;
   R600FrameLowering FrameLowering;
   bool FMA;

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

if (F.hasFnAttribute("amdgpu-num-vgpr")) {
 
     if (Requested)
       MaxNumVGPRs = Requested;
   }
 
   return MaxNumVGPRs;
 }
 
+void GCNSubtarget::adjustSchedDependency(SUnit *Src, SUnit *Dst,
+                                         SDep &Dep) const {
+  if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() ||
+      !Src->isInstr() || !Dst->isInstr())
+    return;
+
+  MachineInstr *SrcI = Src->getInstr();
+  MachineInstr *DstI = Dst->getInstr();
+
+  if (SrcI->isBundle()) {
+    const SIRegisterInfo *TRI = getRegisterInfo();
+    auto Reg = Dep.getReg();
+    MachineBasicBlock::const_instr_iterator I(SrcI->getIterator());
+    MachineBasicBlock::const_instr_iterator E(SrcI->getParent()->instr_end());
+    for (++I; I != E && I->isBundledWithPred(); ++I) {
+      if (!I->modifiesRegister(Reg, TRI))
+        continue;
+      Dep.setLatency(InstrInfo.getInstrLatency(getInstrItineraryData(), *I));
+      break;
+    }
+  } else if (DstI->isBundle()) {
+    Dep.setLatency(InstrInfo.getInstrLatency(getInstrItineraryData(), *SrcI));
+  }
+}
+
 namespace {
 struct MemOpClusterMutation : ScheduleDAGMutation {
   const SIInstrInfo *TII;
 
   MemOpClusterMutation(const SIInstrInfo *tii) : TII(tii) {}
 
   void apply(ScheduleDAGInstrs *DAG) override {
     SUnit *SUa = nullptr;
for (SUnit &SU : DAG->SUnits) {
         }
       }
 
       SUa = &SU;
     }
   }
 };
 
-struct FixBundleLatencyMutation : ScheduleDAGMutation {
-  const SIInstrInfo *TII;
-
-  const TargetSchedModel *TSchedModel;
-
-  FixBundleLatencyMutation(const SIInstrInfo *tii) : TII(tii) {}
-
-  unsigned computeLatency(const MachineInstr &MI, unsigned Reg) const {
-    const SIRegisterInfo &TRI = TII->getRegisterInfo();
-    MachineBasicBlock::const_instr_iterator I(MI.getIterator());
-    MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
-    unsigned Lat = 0;
-    for (++I; I != E && I->isBundledWithPred(); ++I) {
-      if (!I->modifiesRegister(Reg, &TRI))
-        continue;
-      Lat = TSchedModel->computeInstrLatency(&*I);
-      break;
-    }
-    return Lat;
-  }
-
-  void apply(ScheduleDAGInstrs *DAGInstrs) override {
-    ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
-    TSchedModel = DAGInstrs->getSchedModel();
-    if (!TSchedModel || DAG->SUnits.empty())
-      return;
-
-    for (SUnit &SU : DAG->SUnits) {
-      if (!SU.isInstr() || !SU.getInstr()->isBundle())
-        continue;
-      for (SDep &Dep : SU.Succs) {
-        if (Dep.getKind() == SDep::Kind::Data && Dep.getReg())
-          if (unsigned Lat = computeLatency(*SU.getInstr(), Dep.getReg()))
-            Dep.setLatency(Lat);
-      }
-    }
-  }
-};
-
 struct FillMFMAShadowMutation : ScheduleDAGMutation {
   const SIInstrInfo *TII;
 
   ScheduleDAGMI *DAG;
 
   FillMFMAShadowMutation(const SIInstrInfo *tii) : TII(tii) {}
 
   bool isSALU(const SUnit *SU) const {
for (SUnit &SU : DAG->SUnits) {
       }
     }
   }
 };
 } // namespace
 
 void GCNSubtarget::getPostRAMutations(
     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
-  Mutations.push_back(std::make_unique<FixBundleLatencyMutation>(&InstrInfo));
   Mutations.push_back(std::make_unique<MemOpClusterMutation>(&InstrInfo));
   Mutations.push_back(std::make_unique<FillMFMAShadowMutation>(&InstrInfo));
 }
 
 const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
   if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn)
     return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
   else

llvm/lib/Target/AMDGPU/SIInstrInfo.h

MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
                                       ArrayRef<unsigned> Ops,
                                       MachineBasicBlock::iterator InsertPt,
                                       int FrameIndex,
                                       LiveIntervals *LIS = nullptr,
                                       VirtRegMap *VRM = nullptr) const override;
 
   unsigned getInstrLatency(const InstrItineraryData *ItinData,
                            const MachineInstr &MI,
-                           unsigned *PredCost) const override;
+                           unsigned *PredCost = nullptr) const override;
 };
 
 /// \brief Returns true if a reg:subreg pair P has a TRC class
 inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
                          const TargetRegisterClass &TRC,
                          MachineRegisterInfo &MRI) {
   auto *RC = MRI.getRegClass(P.Reg);
   if (!P.SubReg)