diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h --- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -224,9 +224,13 @@ virtual void overrideSchedPolicy(MachineSchedPolicy &Policy, unsigned NumRegionInstrs) const {} - // Perform target specific adjustments to the latency of a schedule + // Perform target-specific adjustments to the latency of a schedule // dependency. - virtual void adjustSchedDependency(SUnit *def, SUnit *use, SDep &dep) const {} + // If a pair of operands is associated with the schedule dependency, DefOpIdx + // and UseOpIdx are the indices of the operands in Def and Use, respectively. + // Otherwise, either may be -1. + virtual void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, + int UseOpIdx, SDep &Dep) const {} // For use with PostRAScheduling: get the anti-dependence breaking that should // be performed before post-RA scheduling. diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -809,7 +809,7 @@ if (!MI->isPHI()) { SDep Dep(SU, SDep::Data, Reg); Dep.setLatency(0); - ST.adjustSchedDependency(SU, &I, Dep); + ST.adjustSchedDependency(SU, 0, &I, MI->getOperandNo(MOI), Dep); I.addPred(Dep); } else { HasPhiUse = Reg; diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp --- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -269,13 +269,13 @@ if (!ImplicitPseudoDef && !ImplicitPseudoUse) { Dep.setLatency(SchedModel.computeOperandLatency(SU->getInstr(), OperIdx, RegUse, UseOp)); - ST.adjustSchedDependency(SU, UseSU, Dep); + ST.adjustSchedDependency(SU, OperIdx, UseSU, UseOp, Dep); } else { Dep.setLatency(0); // FIXME: We could always let target to adjustSchedDependency(), and // remove this condition, but that currently asserts in Hexagon BE. if (SU->getInstr()->isBundle() || (RegUse && RegUse->isBundle())) - ST.adjustSchedDependency(SU, UseSU, Dep); + ST.adjustSchedDependency(SU, OperIdx, UseSU, UseOp, Dep); } UseSU->addPred(Dep); @@ -444,7 +444,7 @@ SDep Dep(SU, SDep::Data, Reg); Dep.setLatency(SchedModel.computeOperandLatency(MI, OperIdx, Use, I->OperandIndex)); - ST.adjustSchedDependency(SU, UseSU, Dep); + ST.adjustSchedDependency(SU, OperIdx, UseSU, I->OperandIndex, Dep); UseSU->addPred(Dep); } diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -474,6 +474,7 @@ for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { SDNode *OpN = N->getOperand(i).getNode(); + unsigned DefIdx = N->getOperand(i).getResNo(); if (isPassiveNode(OpN)) continue; // Not scheduled. SUnit *OpSU = &SUnits[OpN->getNodeId()]; assert(OpSU && "Node has no SUnit!"); @@ -508,7 +509,7 @@ Dep.setLatency(OpLatency); if (!isChain && !UnitLatencies) { computeOperandLatency(OpN, N, i, Dep); - ST.adjustSchedDependency(OpSU, SU, Dep); + ST.adjustSchedDependency(OpSU, DefIdx, SU, i, Dep); } if (!SU->addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h @@ -1193,7 +1193,8 @@ return AMDGPU::IsaInfo::getMinWavesPerEU(this); } - void adjustSchedDependency(SUnit *Src, SUnit *Dst, SDep &Dep) const override; + void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, + SDep &Dep) const override; }; class R600Subtarget final : public R600GenSubtargetInfo, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -722,20 +722,20 @@ return MaxNumVGPRs; } -void GCNSubtarget::adjustSchedDependency(SUnit *Src, SUnit *Dst, - SDep &Dep) const { +void GCNSubtarget::adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, + int UseOpIdx, SDep &Dep) const { if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() || - !Src->isInstr() || !Dst->isInstr()) + !Def->isInstr() || !Use->isInstr()) return; - MachineInstr *SrcI = Src->getInstr(); - MachineInstr *DstI = Dst->getInstr(); + MachineInstr *DefI = Def->getInstr(); + MachineInstr *UseI = Use->getInstr(); - if (SrcI->isBundle()) { + if (DefI->isBundle()) { const SIRegisterInfo *TRI = getRegisterInfo(); auto Reg = Dep.getReg(); - MachineBasicBlock::const_instr_iterator I(SrcI->getIterator()); - MachineBasicBlock::const_instr_iterator E(SrcI->getParent()->instr_end()); + MachineBasicBlock::const_instr_iterator I(DefI->getIterator()); + MachineBasicBlock::const_instr_iterator E(DefI->getParent()->instr_end()); unsigned Lat = 0; for (++I; I != E && I->isBundledWithPred(); ++I) { if (I->modifiesRegister(Reg, TRI)) @@ -744,12 +744,12 @@ --Lat; } Dep.setLatency(Lat); - } else if (DstI->isBundle()) { + } else if (UseI->isBundle()) { const SIRegisterInfo *TRI = getRegisterInfo(); auto Reg = Dep.getReg(); - MachineBasicBlock::const_instr_iterator I(DstI->getIterator()); - MachineBasicBlock::const_instr_iterator E(DstI->getParent()->instr_end()); - unsigned Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *SrcI); + MachineBasicBlock::const_instr_iterator I(UseI->getIterator()); + MachineBasicBlock::const_instr_iterator E(UseI->getParent()->instr_end()); + unsigned Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *DefI); for (++I; I != E && I->isBundledWithPred() && Lat; ++I) { if (I->readsRegister(Reg, TRI)) break; diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.h b/llvm/lib/Target/Hexagon/HexagonSubtarget.h --- a/llvm/lib/Target/Hexagon/HexagonSubtarget.h +++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.h @@ -258,7 +258,8 @@ /// Perform target specific adjustments to the latency of a schedule /// dependency. - void adjustSchedDependency(SUnit *def, SUnit *use, SDep& dep) const override; + void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, + SDep &Dep) const override; unsigned getVectorLength() const { assert(useHVXOps()); diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp --- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp +++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp @@ -315,7 +315,8 @@ /// Perform target specific adjustments to the latency of a schedule /// dependency. -void HexagonSubtarget::adjustSchedDependency(SUnit *Src, SUnit *Dst, +void HexagonSubtarget::adjustSchedDependency(SUnit *Src, int SrcOpIdx, + SUnit *Dst, int DstOpIdx, SDep &Dep) const { MachineInstr *SrcInst = Src->getInstr(); MachineInstr *DstInst = Dst->getInstr();