diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp --- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp @@ -450,7 +450,6 @@ internal::appendLoopsToWorklist(reverse(LI), Worklist); while (!Worklist.empty()) { Loop *L = Worklist.pop_back_val(); - formLCSSA(*L, DT, &LI, &SE); LoopUnrollResult Result = tryToUnrollAndJamLoop(L, DT, &LI, SE, TTI, AC, DI, ORE, OptLevel); if (Result != LoopUnrollResult::Unmodified) diff --git a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp --- a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp @@ -284,8 +284,7 @@ // Move any instructions from fore phi operands from AftBlocks into Fore. moveHeaderPhiOperandsToForeBlocks( - Header, LatchBlock, SubLoop->getLoopPreheader()->getTerminator(), - AftBlocks); + Header, LatchBlock, ForeBlocksLast[0]->getTerminator(), AftBlocks); // The current on-the-fly SSA update requires blocks to be processed in // reverse postorder so that LastValueMap contains the correct value at each @@ -312,7 +311,7 @@ // Copy all blocks for (unsigned It = 1; It != Count; ++It) { - std::vector NewBlocks; + SmallVector NewBlocks; // Maps Blocks[It] -> Blocks[It-1] DenseMap PrevItValueMap; @@ -379,9 +378,9 @@ } // Remap all instructions in the most recent iteration + remapInstructionsInBlocks(NewBlocks, LastValueMap); for (BasicBlock *NewBlock : NewBlocks) { for (Instruction &I : *NewBlock) { - ::remapInstruction(&I, LastValueMap); if (auto *II = dyn_cast(&I)) if (II->getIntrinsicID() == Intrinsic::assume) AC->registerAssumption(II); @@ -447,8 +446,8 @@ // Update ForeBlocks successors and phi nodes BranchInst *ForeTerm = cast(ForeBlocksLast.back()->getTerminator()); - BasicBlock *Dest = SubLoopBlocksFirst[0]; - ForeTerm->setSuccessor(0, Dest); + assert(ForeTerm->getNumSuccessors() == 1 && "Expecting one successor"); + ForeTerm->setSuccessor(0, SubLoopBlocksFirst[0]); if (CompletelyUnroll) { while (PHINode *Phi = dyn_cast(ForeBlocksFirst[0]->begin())) { @@ -465,8 +464,8 @@ // Remap ForeBlock successors from previous iteration to this BranchInst *ForeTerm = cast(ForeBlocksLast[It - 1]->getTerminator()); - BasicBlock *Dest = ForeBlocksFirst[It]; - ForeTerm->setSuccessor(0, Dest); + assert(ForeTerm->getNumSuccessors() == 1 && "Expecting one successor"); + ForeTerm->setSuccessor(0, ForeBlocksFirst[It]); } // Subloop successors and phis @@ -495,12 +494,14 @@ } // Aft blocks successors and phis - BranchInst *Term = cast(AftBlocksLast.back()->getTerminator()); + BranchInst *AftTerm = cast(AftBlocksLast.back()->getTerminator()); if (CompletelyUnroll) { - BranchInst::Create(LoopExit, Term); - Term->eraseFromParent(); + BranchInst::Create(LoopExit, AftTerm); + AftTerm->eraseFromParent(); } else { - Term->setSuccessor(!ContinueOnTrue, ForeBlocksFirst[0]); + AftTerm->setSuccessor(!ContinueOnTrue, ForeBlocksFirst[0]); + assert(AftTerm->getSuccessor(ContinueOnTrue) == LoopExit && + "Expecting the ContinueOnTrue successor of AftTerm to be LoopExit"); } updatePHIBlocks(AftBlocksFirst[0], SubLoopBlocksLast[0], SubLoopBlocksLast.back()); @@ -518,24 +519,16 @@ movePHIs(AftBlocksFirst[It], AftBlocksFirst[0]); } - DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); // Dominator Tree. Remove the old links between Fore, Sub and Aft, adding the // new ones required. if (Count != 1) { - SmallVector DTUpdates; - DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete, ForeBlocksLast[0], - SubLoopBlocksFirst[0]); - DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete, - SubLoopBlocksLast[0], AftBlocksFirst[0]); - - DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert, - ForeBlocksLast.back(), SubLoopBlocksFirst[0]); - DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert, - SubLoopBlocksLast.back(), AftBlocksFirst[0]); - DTU.applyUpdatesPermissive(DTUpdates); + DT->changeImmediateDominator(SubLoopBlocksFirst[0], ForeBlocksLast.back()); + DT->changeImmediateDominator(AftBlocksFirst[0], SubLoopBlocksLast.back()); + DT->changeImmediateDominator(LoopExit, AftBlocksLast.back()); } // Merge adjacent basic blocks, if possible. + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); SmallPtrSet MergeBlocks; MergeBlocks.insert(ForeBlocksLast.begin(), ForeBlocksLast.end()); MergeBlocks.insert(SubLoopBlocksLast.begin(), SubLoopBlocksLast.end()); @@ -568,21 +561,26 @@ NumCompletelyUnrolledAndJammed += CompletelyUnroll; ++NumUnrolledAndJammed; + // Update LoopInfo if the loop is completely removed. + if (CompletelyUnroll) + LI->erase(L); + #ifndef NDEBUG // We shouldn't have done anything to break loop simplify form or LCSSA. - Loop *OuterL = L->getParentLoop(); - Loop *OutestLoop = OuterL ? OuterL : (!CompletelyUnroll ? L : SubLoop); + Loop *OutestLoop = SubLoop->getParentLoop() + ? SubLoop->getParentLoop()->getParentLoop() + ? SubLoop->getParentLoop()->getParentLoop() + : SubLoop->getParentLoop() + : SubLoop; + assert(DT->verify()); + LI->verify(*DT); assert(OutestLoop->isRecursivelyLCSSAForm(*DT, *LI)); if (!CompletelyUnroll) assert(L->isLoopSimplifyForm()); assert(SubLoop->isLoopSimplifyForm()); - assert(DT->verify()); + SE->verify(); #endif - // Update LoopInfo if the loop is completely removed. - if (CompletelyUnroll) - LI->erase(L); - return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled : LoopUnrollResult::PartiallyUnrolled; }