Index: lib/CodeGen/MachinePipeliner.cpp =================================================================== --- lib/CodeGen/MachinePipeliner.cpp +++ lib/CodeGen/MachinePipeliner.cpp @@ -1025,76 +1025,80 @@ /// Add a chain edge between a load and store if the store can be an /// alias of the load on a subsequent iteration, i.e., a loop carried -/// dependence. This code is very similar to the code in ScheduleDAGInstrs +/// dependence. +/// Similarly, add a chain edge between a store and load or between a +/// store and store. +/// This code is very similar to the code in ScheduleDAGInstrs /// but that code doesn't create loop carried dependences. void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) { - MapVector> PendingLoads; + MapVector> PendingLoadsStores; for (auto &SU : SUnits) { MachineInstr &MI = *SU.getInstr(); if (isDependenceBarrier(MI, AA)) - PendingLoads.clear(); - else if (MI.mayLoad()) { - SmallVector Objs; - getUnderlyingObjects(&MI, Objs, MF.getDataLayout()); - for (auto V : Objs) { - SmallVector &SUs = PendingLoads[V]; - SUs.push_back(&SU); - } - } else if (MI.mayStore()) { - SmallVector Objs; - getUnderlyingObjects(&MI, Objs, MF.getDataLayout()); - for (auto V : Objs) { - MapVector>::iterator I = - PendingLoads.find(V); - if (I == PendingLoads.end()) + PendingLoadsStores.clear(); + else if (!MI.mayLoad() && !MI.mayStore()) + continue; + + SmallVector Objs; + getUnderlyingObjects(&MI, Objs, MF.getDataLayout()); + for (auto V : Objs) { + SmallVector &SUs = PendingLoadsStores[V]; + SUs.push_back(&SU); + + MapVector>::iterator I = + PendingLoadsStores.find(V); + if (I == PendingLoadsStores.end()) + continue; + for (auto LdSt : I->second) { + if (isSuccOrder(LdSt, &SU)) + continue; + + MachineInstr &LdStMI = *LdSt->getInstr(); + if ((LdStMI.mayLoad() && MI.mayLoad()) || &LdStMI == &MI) + continue; + + // First, perform the cheaper check that compares the base register. + // If they are the same and the load offset is less than the store + // offset, then mark the dependence as loop carried potentially. + unsigned BaseReg1, BaseReg2; + int64_t Offset1, Offset2; + if (!TII->getMemOpBaseRegImmOfs(LdStMI, BaseReg1, Offset1, TRI) || + !TII->getMemOpBaseRegImmOfs(MI, BaseReg2, Offset2, TRI)) { + SU.addPred(SDep(LdSt, SDep::Barrier)); + continue; + } + if (BaseReg1 == BaseReg2 && (int)Offset1 < (int)Offset2) { + assert(TII->areMemAccessesTriviallyDisjoint(LdStMI, MI, AA) && + "What happened to the chain edge?"); + SU.addPred(SDep(LdSt, SDep::Barrier)); + continue; + } + // Second, the more expensive check that uses alias analysis on the + // base registers. If they alias, and the load offset is less than + // the store offset, the mark the dependence as loop carried. + if (!AA) { + SU.addPred(SDep(LdSt, SDep::Barrier)); + continue; + } + MachineMemOperand *MMO1 = *LdStMI.memoperands_begin(); + MachineMemOperand *MMO2 = *MI.memoperands_begin(); + if (!MMO1->getValue() || !MMO2->getValue()) { + SU.addPred(SDep(LdSt, SDep::Barrier)); + continue; + } + if (MMO1->getValue() == MMO2->getValue() && + MMO1->getOffset() <= MMO2->getOffset()) { + SU.addPred(SDep(LdSt, SDep::Barrier)); continue; - for (auto Load : I->second) { - if (isSuccOrder(Load, &SU)) - continue; - MachineInstr &LdMI = *Load->getInstr(); - // First, perform the cheaper check that compares the base register. - // If they are the same and the load offset is less than the store - // offset, then mark the dependence as loop carried potentially. - unsigned BaseReg1, BaseReg2; - int64_t Offset1, Offset2; - if (!TII->getMemOpBaseRegImmOfs(LdMI, BaseReg1, Offset1, TRI) || - !TII->getMemOpBaseRegImmOfs(MI, BaseReg2, Offset2, TRI)) { - SU.addPred(SDep(Load, SDep::Barrier)); - continue; - } - if (BaseReg1 == BaseReg2 && (int)Offset1 < (int)Offset2) { - assert(TII->areMemAccessesTriviallyDisjoint(LdMI, MI, AA) && - "What happened to the chain edge?"); - SU.addPred(SDep(Load, SDep::Barrier)); - continue; - } - // Second, the more expensive check that uses alias analysis on the - // base registers. If they alias, and the load offset is less than - // the store offset, the mark the dependence as loop carried. - if (!AA) { - SU.addPred(SDep(Load, SDep::Barrier)); - continue; - } - MachineMemOperand *MMO1 = *LdMI.memoperands_begin(); - MachineMemOperand *MMO2 = *MI.memoperands_begin(); - if (!MMO1->getValue() || !MMO2->getValue()) { - SU.addPred(SDep(Load, SDep::Barrier)); - continue; - } - if (MMO1->getValue() == MMO2->getValue() && - MMO1->getOffset() <= MMO2->getOffset()) { - SU.addPred(SDep(Load, SDep::Barrier)); - continue; - } - AliasResult AAResult = AA->alias( - MemoryLocation(MMO1->getValue(), MemoryLocation::UnknownSize, - MMO1->getAAInfo()), - MemoryLocation(MMO2->getValue(), MemoryLocation::UnknownSize, - MMO2->getAAInfo())); - - if (AAResult != NoAlias) - SU.addPred(SDep(Load, SDep::Barrier)); } + AliasResult AAResult = AA->alias( + MemoryLocation(MMO1->getValue(), MemoryLocation::UnknownSize, + MMO1->getAAInfo()), + MemoryLocation(MMO2->getValue(), MemoryLocation::UnknownSize, + MMO2->getAAInfo())); + + if (AAResult != NoAlias) + SU.addPred(SDep(LdSt, SDep::Barrier)); } } } Index: test/CodeGen/Hexagon/vect/vect-v4i16.ll =================================================================== --- test/CodeGen/Hexagon/vect/vect-v4i16.ll +++ test/CodeGen/Hexagon/vect/vect-v4i16.ll @@ -1,8 +1,6 @@ ; RUN: llc -march=hexagon -mcpu=hexagonv5 -disable-hsdr < %s | FileCheck %s ; Check that store is post-incremented. -; CHECK: memuh(r{{[0-9]+}}+#6) -; CHECK: combine(r{{[0-9]+}},r{{[0-9]+}}) ; CHECK: vaddh target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"