[LV] Do not create separate latch block in VPlan::execute.

[fhahn]

Summary

Now that all dependencies on creating the latch block up-front have been
removed, there is no need to create it early.

Depends on D121618.

[Ayal]

Nice! Looks good to me.

llvm/lib/Transforms/Vectorize/VPlan.cpp

759–760

Rename it State.CFG.ExitBB?
Update above TODO?

923

Only "1" is left, can be removed.

[fhahn]

Address latest comments, thanks! I am planning to push this once the last remaining dependency (D121618) lands.

llvm/lib/Transforms/Vectorize/VPlan.h

struct VPTransformState {
   struct CFGState {
     /// The previous VPBasicBlock visited. Initially set to null.
     VPBasicBlock *PrevVPBB = nullptr;
 
     /// The previous IR BasicBlock created or used. Initially set to the new
     /// header BasicBlock.
     BasicBlock *PrevBB = nullptr;
 
-    /// The last IR BasicBlock in the output IR. Set to the new latch
-    /// BasicBlock, used for placing the newly created BasicBlocks.
-    BasicBlock *LastBB = nullptr;
+    /// The last IR BasicBlock in the output IR. Set to the exit block of the
+    /// vector loop.
+    BasicBlock *ExitBB = nullptr;
 
     /// The IR BasicBlock that is the preheader of the vector loop in the output
     /// IR.
     /// FIXME: The vector preheader should also be modeled in VPlan, so any code
     /// that needs to be added to the preheader gets directly generated by
     /// VPlan. There should be no need to manage a pointer to the IR BasicBlock.
     BasicBlock *VectorPreHeader = nullptr;
 

llvm/lib/Transforms/Vectorize/VPlan.cpp

 }
 
 BasicBlock *
 VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
   // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
   // Pred stands for Predessor. Prev stands for Previous - last visited/created.
   BasicBlock *PrevBB = CFG.PrevBB;
   BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(),
-                                         PrevBB->getParent(), CFG.LastBB);
+                                         PrevBB->getParent(), CFG.ExitBB);
   LLVM_DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n');
 
   // Hook up the new basic block to its predecessors.
   for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
     VPBasicBlock *PredVPBB = PredVPBlock->getExitBasicBlock();
     auto &PredVPSuccessors = PredVPBB->getSuccessors();
     BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
 
if (PrevVPBB && /* A */
         PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
       !(Replica && getPredecessors().empty())) {       /* C */
     NewBB = createEmptyBasicBlock(State->CFG);
     State->Builder.SetInsertPoint(NewBB);
     // Temporarily terminate with unreachable until CFG is rewired.
     UnreachableInst *Terminator = State->Builder.CreateUnreachable();
     State->Builder.SetInsertPoint(Terminator);
     // Register NewBB in its loop. In innermost loops its the same for all BB's.
-    Loop *L = State->LI->getLoopFor(State->CFG.LastBB);
+    Loop *L = State->LI->getLoopFor(State->CFG.PrevBB);
     L->addBasicBlockToLoop(NewBB, *State->LI);
     State->CFG.PrevBB = NewBB;
   }
 
   // 2. Fill the IR basic block with IR instructions.
   LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()
                     << " in BB:" << NewBB->getName() << '\n');
 
case VPInstruction::BranchOnCount: {
     VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
     VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
     if (Header->empty()) {
       assert(EnableVPlanNativePath &&
              "empty entry block only expected in VPlanNativePath");
       Header = cast<VPBasicBlock>(Header->getSingleSuccessor());
     }
     // TODO: Once the exit block is modeled in VPlan, use it instead of going
-    // through State.CFG.LastBB.
-    BasicBlock *Exit =
-        cast<BranchInst>(State.CFG.LastBB->getTerminator())->getSuccessor(0);
+    // through State.CFG.ExitBB.
+    BasicBlock *Exit = State.CFG.ExitBB;
 
     Builder.CreateCondBr(Cond, Exit, State.CFG.VPBB2IRBB[Header]);
     Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
     break;
   }
   default:
     llvm_unreachable("Unsupported opcode for instruction");
   }
void VPlan::execute(VPTransformState *State) {
   for (auto &Entry : Value2VPValue)
     State->VPValue2Value[Entry.second] = Entry.first;
 
   BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
   State->CFG.VectorPreHeader = VectorPreHeaderBB;
   BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
   assert(VectorHeaderBB && "Loop preheader does not have a single successor.");
 
-  // 1. Make room to generate basic-blocks inside loop body if needed.
-  BasicBlock *VectorLatchBB = VectorHeaderBB->splitBasicBlock(
-      VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch");
   Loop *L = State->LI->getLoopFor(VectorHeaderBB);
-  L->addBasicBlockToLoop(VectorLatchBB, *State->LI);
+  State->CFG.ExitBB = L->getExitBlock();
+
   // Remove the edge between Header and Latch to allow other connections.
   // Temporarily terminate with unreachable until CFG is rewired.
   // Note: this asserts the generated code's assumption that
   // getFirstInsertionPt() can be dereferenced into an Instruction.
   VectorHeaderBB->getTerminator()->eraseFromParent();
   State->Builder.SetInsertPoint(VectorHeaderBB);
   UnreachableInst *Terminator = State->Builder.CreateUnreachable();
   State->Builder.SetInsertPoint(Terminator);
 
-  // 2. Generate code in loop body.
+  // Generate code in loop body.
   State->CFG.PrevVPBB = nullptr;
   State->CFG.PrevBB = VectorHeaderBB;
-  State->CFG.LastBB = VectorLatchBB;
 
   for (VPBlockBase *Block : depth_first(Entry))
     Block->execute(State);
 
   // Setup branch terminator successors for VPBBs in VPBBsToFix based on
   // VPBB's successors.
   for (auto VPBB : State->CFG.VPBBsToFix) {
     assert(EnableVPlanNativePath &&
            "Unexpected VPBBsToFix in non VPlan-native path");
     BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
     assert(BB && "Unexpected null basic block for VPBB");
 
     unsigned Idx = 0;
     auto *BBTerminator = BB->getTerminator();
 
     for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
       VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
       BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
       ++Idx;
     }
   }
 
-  // 3. Merge the temporary latch created with the last basic-block filled.
-  BasicBlock *LastBB = State->CFG.PrevBB;
-  assert(isa<BranchInst>(LastBB->getTerminator()) &&
-         "Expected VPlan CFG to terminate with branch");
-
-  // Move both the branch and check from LastBB to VectorLatchBB.
-  auto *LastBranch = cast<BranchInst>(LastBB->getTerminator());
-  LastBranch->moveBefore(VectorLatchBB->getTerminator());
-  VectorLatchBB->getTerminator()->eraseFromParent();
-  // Move condition so it is guaranteed to be next to branch. This is only done
-  // to avoid excessive test updates.
-  // TODO: Remove special handling once the increments for all inductions are
-  // modeled explicitly in VPlan.
-  cast<Instruction>(LastBranch->getCondition())->moveBefore(LastBranch);
-  // Connect LastBB to VectorLatchBB to facilitate their merge.
-  BranchInst::Create(VectorLatchBB, LastBB);
-
-  // Merge LastBB with Latch.
-  bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI);
-  (void)Merged;
-  assert(Merged && "Could not merge last basic block with latch.");
-  VectorLatchBB = LastBB;
+  BasicBlock *VectorLatchBB = State->CFG.PrevBB;
 
   // Fix the latch value of canonical, reduction and first-order recurrences
   // phis in the vector loop.
   VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
   if (Header->empty()) {
     assert(EnableVPlanNativePath);
     Header = cast<VPBasicBlock>(Header->getSingleSuccessor());
   }