diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp --- a/llvm/lib/Transforms/Utils/LoopPeel.cpp +++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp @@ -11,10 +11,13 @@ #include "llvm/Transforms/Utils/LoopPeel.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/Loads.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/ScalarEvolution.h" @@ -555,11 +558,13 @@ /// \param LoopBlocks A helper for DFS-traversal of the loop. /// \param LVMap A value-map that maps instructions from the original loop to /// instructions in the last peeled-off iteration. +/// \param LoopBlocksIDoms Immediate dominators of the original loop blocks. static void cloneLoopBlocks( Loop *L, unsigned IterNumber, BasicBlock *InsertTop, BasicBlock *InsertBot, SmallVectorImpl> &ExitEdges, SmallVectorImpl &NewBlocks, LoopBlocksDFS &LoopBlocks, - ValueToValueMapTy &VMap, ValueToValueMapTy &LVMap, DominatorTree *DT, + ValueToValueMapTy &VMap, ValueToValueMapTy &LVMap, DomTreeUpdater &DTU, + SmallDenseMap &LoopBlocksIDoms, LoopInfo *LI, ArrayRef LoopLocalNoAliasDeclScopes) { BasicBlock *Header = L->getHeader(); BasicBlock *Latch = L->getLoopLatch(); @@ -570,6 +575,8 @@ LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO(); Loop *ParentLoop = L->getParentLoop(); + SmallVector DTUpdates; + // For each block in the original loop, create a new copy, // and update the value map with the newly created values. for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) { @@ -585,13 +592,14 @@ VMap[*BB] = NewBB; // If dominator tree is available, insert nodes to represent cloned blocks. - if (DT) { + if (DTU.hasDomTree()) { if (Header == *BB) - DT->addNewBlock(NewBB, InsertTop); + DTUpdates.push_back({DominatorTree::Insert, InsertTop, NewBB}); else { - DomTreeNode *IDom = DT->getNode(*BB)->getIDom(); + BasicBlock *IDom = LoopBlocksIDoms[*BB]; // VMap must contain entry for IDom, as the iteration order is RPO. - DT->addNewBlock(NewBB, cast(VMap[IDom->getBlock()])); + DTUpdates.push_back( + {DominatorTree::Insert, cast(VMap[IDom]), NewBB}); } } } @@ -629,8 +637,10 @@ LatchBR->setSuccessor(idx, InsertBot); break; } - if (DT) - DT->changeImmediateDominator(InsertBot, NewLatch); + if (DTU.hasDomTree()) { + DTUpdates.push_back({DominatorTree::Insert, NewLatch, InsertBot}); + DTUpdates.push_back({DominatorTree::Delete, InsertTop, InsertBot}); + } // The new copy of the loop body starts with a bunch of PHI nodes // that pick an incoming value from either the preheader, or the previous @@ -668,6 +678,8 @@ PHI.addIncoming(LatchVal, cast(VMap[Edge.first])); } + DTU.applyUpdates(DTUpdates); + // LastValueMap is updated with the values for the current loop // which are used the next time this function is called. for (auto KV : VMap) @@ -732,36 +744,8 @@ SmallVector, 4> ExitEdges; L->getExitEdges(ExitEdges); - DenseMap ExitIDom; - if (DT) { - // We'd like to determine the idom of exit block after peeling one - // iteration. - // Let Exit is exit block. - // Let ExitingSet - is a set of predecessors of Exit block. They are exiting - // blocks. - // Let Latch' and ExitingSet' are copies after a peeling. - // We'd like to find an idom'(Exit) - idom of Exit after peeling. - // It is an evident that idom'(Exit) will be the nearest common dominator - // of ExitingSet and ExitingSet'. - // idom(Exit) is a nearest common dominator of ExitingSet. - // idom(Exit)' is a nearest common dominator of ExitingSet'. - // Taking into account that we have a single Latch, Latch' will dominate - // Header and idom(Exit). - // So the idom'(Exit) is nearest common dominator of idom(Exit)' and Latch'. - // All these basic blocks are in the same loop, so what we find is - // (nearest common dominator of idom(Exit) and Latch)'. - // In the loop below we remember nearest common dominator of idom(Exit) and - // Latch to update idom of Exit later. - assert(L->hasDedicatedExits() && "No dedicated exits?"); - for (auto Edge : ExitEdges) { - if (ExitIDom.count(Edge.second)) - continue; - BasicBlock *BB = DT->findNearestCommonDominator( - DT->getNode(Edge.second)->getIDom()->getBlock(), Latch); - assert(L->contains(BB) && "IDom is not in a loop"); - ExitIDom[Edge.second] = BB; - } - } + SmallDenseSet, 4> ExitEdgesSet( + ExitEdges.begin(), ExitEdges.end()); Function *F = Header->getParent(); @@ -834,31 +818,33 @@ SmallVector LoopLocalNoAliasDeclScopes; identifyNoAliasScopesToClone(L->getBlocks(), LoopLocalNoAliasDeclScopes); + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); + + // Fill map with the loop blocks IDoms to later update the DT when cloning the + // loop blocks. + SmallDenseMap LoopBlocksIDoms; + for (Loop::block_iterator BB = L->block_begin(), E = L->block_end(); BB != E; + ++BB) + LoopBlocksIDoms[*BB] = DT->getNode(*BB)->getIDom()->getBlock(); + // For each peeled-off iteration, make a copy of the loop. for (unsigned Iter = 0; Iter < PeelCount; ++Iter) { SmallVector NewBlocks; ValueToValueMapTy VMap; cloneLoopBlocks(L, Iter, InsertTop, InsertBot, ExitEdges, NewBlocks, - LoopBlocks, VMap, LVMap, DT, LI, + LoopBlocks, VMap, LVMap, DTU, LoopBlocksIDoms, LI, LoopLocalNoAliasDeclScopes); // Remap to use values from the current iteration instead of the // previous one. remapInstructionsInBlocks(NewBlocks, VMap); - if (DT) { - // Latches of the cloned loops dominate over the loop exit, so idom of the - // latter is the first cloned loop body, as original PreHeader dominates - // the original loop body. - if (Iter == 0) - for (auto Exit : ExitIDom) - DT->changeImmediateDominator(Exit.first, - cast(LVMap[Exit.second])); -#ifdef EXPENSIVE_CHECKS - assert(DT->verify(DominatorTree::VerificationLevel::Fast)); -#endif - } + // If DT is available, insert edges from cloned exiting blocks to the exits + if (DTU.hasDomTree()) + for (auto Exit : ExitEdgesSet) + DTU.applyUpdates({{DominatorTree::Insert, + cast(LVMap[Exit.first]), Exit.second}}); auto *LatchBRCopy = cast(VMap[LatchBR]); updateBranchWeights(InsertBot, LatchBRCopy, ExitWeight, FallThroughWeight); @@ -867,7 +853,7 @@ LatchBRCopy->setMetadata(LLVMContext::MD_loop, nullptr); InsertTop = InsertBot; - InsertBot = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI); + InsertBot = SplitBlock(InsertBot, InsertBot->getTerminator(), &DTU, LI); InsertBot->setName(Header->getName() + ".peel.next"); F->getBasicBlockList().splice(InsertTop->getIterator(), @@ -902,7 +888,8 @@ SE->forgetTopmostLoop(L); // Finally DomtTree must be correct. - assert(DT->verify(DominatorTree::VerificationLevel::Fast)); + if (DTU.hasDomTree()) + assert(DTU.getDomTree().verify(DominatorTree::VerificationLevel::Fast)); // FIXME: Incrementally update loop-simplify simplifyLoop(L, DT, LI, SE, AC, nullptr, PreserveLCSSA);