[VPlan] Do not create dedicated latch block up front (WIP).

[fhahn]

Summary

This patch updates VPlan execution to not create and rely on the latch
block of the vector loop existing before VPlan execution.

The patch introduces VPInstruction opcodes ICmpEQ and ExitBranch
to model compares with the canonical IV and the branch exiting the loop
directly in VPlan. This is all subject to change pending feedback.

It is still WIP because there are a few remaining test failures, but I'd
like to share it early so the general direction can be reviewed.

[fhahn]

This has landed split up in multiple patches.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

protected:
   /// vectorization factor.
   using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;
 
   /// Set up the values of the IVs correctly when exiting the vector loop.
   void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
                     Value *CountRoundDown, Value *EndValue,
                     BasicBlock *MiddleBlock);
 
-  void createLatchTerminator(Loop *L);
-
   /// Handle all cross-iteration phis in the header.
   void fixCrossIterationPHIs(VPTransformState &State);
 
   /// Create the exit value of first order recurrences in the middle block and
   /// update their users.
   void fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR, VPTransformState &State);
 
   /// Create code for the loop exit value of the reduction.
protected:
   /// Allow subclasses to override and print debug traces before/after vplan
   /// execution, when trace information is requested.
   virtual void printDebugTracesAtStart(){};
   virtual void printDebugTracesAtEnd(){};
 
   /// The original loop.
   Loop *OrigLoop;
 
+  MDNode *OrigLoopID;
+
   /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
   /// dynamic knowledge to simplify SCEV expressions and converts them to a
   /// more usable form.
   PredicatedScalarEvolution &PSE;
 
   /// Loop Info.
   LoopInfo *LI;
 
void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPValue *Def,
   if (auto *II = dyn_cast<AssumeInst>(Cloned))
     AC->registerAssumption(II);
 
   // End if-block.
   if (IfPredicateInstr)
     PredicatedInstructions.push_back(Cloned);
 }
 
-void InnerLoopVectorizer::createLatchTerminator(Loop *L) {
-  BasicBlock *Header = L->getHeader();
-  BasicBlock *Latch = L->getLoopLatch();
-  // As we're just creating this loop, it's possible no latch exists
-  // yet. If so, use the header as this will be a single block loop.
-  if (!Latch)
-    Latch = Header;
-
-  IRBuilder<> B(&*Header->getFirstInsertionPt());
-  Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction);
-
-  B.SetInsertPoint(Latch->getTerminator());
-  setDebugLocFromInst(OldInst, &B);
-
-  // Create the compare.
-  B.CreateCondBr(B.getTrue(), L->getUniqueExitBlock(), Header);
-
-  // Now we have two terminators. Remove the old one from the block.
-  Latch->getTerminator()->eraseFromParent();
-}
-
 Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
   if (TripCount)
     return TripCount;
 
   assert(L && "Create Trip Count for null loop.");
   IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
   // Find the loop boundaries.
   ScalarEvolution *SE = PSE.getSE();
if (!Cost->requiresScalarEpilogue(VF) && !Cost->foldTailByMasking()) {
     cast<BranchInst>(LoopMiddleBlock->getTerminator())->setCondition(CmpN);
   }
 
   // Get ready to start creating new instructions into the vectorized body.
   assert(LoopVectorPreHeader == L->getLoopPreheader() &&
          "Inconsistent vector loop preheader");
   Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt());
 
-  Optional<MDNode *> VectorizedLoopID =
-      makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
-                                      LLVMLoopVectorizeFollowupVectorized});
-  if (VectorizedLoopID.hasValue()) {
-    L->setLoopID(VectorizedLoopID.getValue());
-
-    // Do not setAlreadyVectorized if loop attributes have been defined
-    // explicitly.
-    return LoopVectorPreHeader;
-  }
-
   // Keep all loop hints from the original loop on the vector loop (we'll
   // replace the vectorizer-specific hints below).
-  if (MDNode *LID = OrigLoop->getLoopID())
-    L->setLoopID(LID);
-
-  LoopVectorizeHints Hints(L, true, *ORE);
-  Hints.setAlreadyVectorized();
+  if (OrigLoopID)
+    OrigLoop->setLoopID(OrigLoopID);
 
 #ifdef EXPENSIVE_CHECKS
   assert(DT->verify(DominatorTree::VerificationLevel::Fast));
   LI->verify(*DT);
 #endif
 
   return LoopVectorPreHeader;
 }
(opt)  v      <-- edge from middle to exit iff epilogue is not required.
   //   - counts from zero, stepping by one
   //   - is the size of the widest induction variable type
   // then we create a new one.
 
   // The loop step is equal to the vectorization factor (num of SIMD elements)
   // times the unroll factor (num of SIMD instructions).
   Builder.SetInsertPoint(&*Lp->getHeader()->getFirstInsertionPt());
   Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
-  createLatchTerminator(Lp);
 
   // Emit phis for the new starting index of the scalar loop.
   createInductionResumeValues(Lp, CountRoundDown);
 
   return completeLoopSkeleton(Lp, OrigLoopID);
 }
 
 // Fix up external users of the induction variable. At this point, we are
void LoopVectorizationPlanner::executePlan(ElementCount BestVF, unsigned BestUF,
   //
   //===------------------------------------------------===//
 
   // 2. Copy and widen instructions from the old loop into the new loop.
   BestVPlan.getVectorLoopRegion()->setVFandUF(BestVF, BestUF);
   State.setVFandUF(BestVF, BestUF);
   BestVPlan.execute(&State);
 
+  // Keep all loop hints from the original loop on the vector loop (we'll
+  // replace the vectorizer-specific hints below).
+  MDNode *OrigLoopID = OrigLoop->getLoopID();
+
+  Optional<MDNode *> VectorizedLoopID =
+      makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
+                                      LLVMLoopVectorizeFollowupVectorized});
+
+  Loop *L = LI->getLoopFor(State.CFG.PrevBB);
+  if (VectorizedLoopID.hasValue())
+    L->setLoopID(VectorizedLoopID.getValue());
+
+  LoopVectorizeHints Hints(L, true, *ORE);
+  Hints.setAlreadyVectorized();
+
+  /*  if (OrigLoopID)*/
+  /*OrigLoop->setLoopID(OrigLoopID);*/
+
+  /*LoopVectorizeHints Hints(OrigLoop, true, *ORE);*/
+  /*Hints.setAlreadyVectorized();*/
+
   // 3. Fix the vectorized code: take care of header phi's, live-outs,
   //    predication, updating analyses.
   ILV.fixVectorizedLoop(State);
 
   ILV.printDebugTracesAtEnd();
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() {
   // the epilogue.
   EPI.MainLoopIterationCountCheck =
       emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, false);
 
   // Generate the induction variable.
   OldInduction = Legal->getPrimaryInduction();
   Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
   EPI.VectorTripCount = CountRoundDown;
-  createLatchTerminator(Lp);
 
   // Skip induction resume value creation here because they will be created in
   // the second pass. If we created them here, they wouldn't be used anyway,
   // because the vplan in the second pass still contains the inductions from the
   // original loop.
 
   return completeLoopSkeleton(Lp, OrigLoopID);
 }
EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
   PHINode *EPResumeVal = PHINode::Create(IdxTy, 2, "vec.epilog.resume.val",
                                          LoopVectorPreHeader->getFirstNonPHI());
   EPResumeVal->addIncoming(EPI.VectorTripCount, VecEpilogueIterationCountCheck);
   EPResumeVal->addIncoming(ConstantInt::get(IdxTy, 0),
                            EPI.MainLoopIterationCountCheck);
 
   // Generate the induction variable.
   OldInduction = Legal->getPrimaryInduction();
-  createLatchTerminator(Lp);
 
   // Generate induction resume values. These variables save the new starting
   // indexes for the scalar loop. They are used to test if there are any tail
   // iterations left once the vector loop has completed.
   // Note that when the vectorized epilogue is skipped due to iteration count
   // check, then the resume value for the induction variable comes from
   // the trip count of the main vector loop, hence passing the AdditionalBypass
   // argument.
VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
   auto Plan = std::make_unique<VPlan>();
 
   // Scan the body of the loop in a topological order to visit each basic block
   // after having visited its predecessor basic blocks.
   LoopBlocksDFS DFS(OrigLoop);
   DFS.perform(LI);
 
   VPBasicBlock *VPBB = nullptr;
+  VPRecipeBase *PrimaryInd = nullptr;
   for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
     // Relevant instructions from basic block BB will be grouped into VPRecipe
     // ingredients and fill a new VPBasicBlock.
     unsigned VPBBsForBB = 0;
     auto *FirstVPBBForBB = new VPBasicBlock(BB->getName());
     if (VPBB)
       VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB);
     else {
       auto *TopRegion = new VPRegionBlock("vector loop");
       TopRegion->setEntry(FirstVPBBForBB);
       Plan->setEntry(TopRegion);
+      Type *IdxTy = Legal->getWidestInductionType();
+      Value *StartIdx = ConstantInt::get(IdxTy, 0);
+      auto *StartV = Plan->getOrAddVPValue(StartIdx);
+
+      PrimaryInd = new VPCanonicalIVRecipe(StartV);
+      FirstVPBBForBB->appendRecipe(PrimaryInd);
     }
     VPBB = FirstVPBBForBB;
     Builder.setInsertPoint(VPBB);
 
     // Introduce each ingredient into VPlan.
     // TODO: Model and preserve debug instrinsics in VPlan.
     for (Instruction &I : BB->instructionsWithoutDebug()) {
       Instruction *Instr = &I;
for (Instruction &I : BB->instructionsWithoutDebug()) {
           // If the re-used value is a recipe, register the recipe for the
           // instruction, in case the recipe for Instr needs to be recorded.
           if (auto *R = dyn_cast_or_null<VPRecipeBase>(VPV->getDef()))
             RecipeBuilder.setRecipe(Instr, R);
           continue;
         }
         // Otherwise, add the new recipe.
         VPRecipeBase *Recipe = RecipeOrValue.get<VPRecipeBase *>();
+        if (auto *IndR = dyn_cast<VPWidenIntOrFpInductionRecipe>(Recipe))
+          IndR->addOperand(PrimaryInd->getVPSingleValue());
         for (auto *Def : Recipe->definedValues()) {
           auto *UV = Def->getUnderlyingValue();
           if (UV)
             Plan->addVPValue(UV, Def);
         }
 
         RecipeBuilder.setRecipe(Instr, Recipe);
         if (isa<VPWidenIntOrFpInductionRecipe>(Recipe)) {
for (unsigned i = 0; i < IG->getFactor(); ++i)
         RecipeBuilder.getRecipe(Member)->eraseFromParent();
       }
   }
 
   Type *IdxTy = Legal->getWidestInductionType();
   Value *StartIdx = ConstantInt::get(IdxTy, 0);
   auto *StartV = Plan->getOrAddVPValue(StartIdx);
 
-  auto *PrimaryInd = new VPCanonicalIVRecipe(StartV);
-  PrimaryInd->insertBefore(
-      &*TopRegion->getEntry()->getEntryBasicBlock()->begin());
-
   auto *InductionIncrement = cast<VPInstruction>(new VPInstruction(
       !CM.foldTailByMasking() ? VPInstruction::InductionIncrementNUW
                               : VPInstruction::InductionIncrement,
       {PrimaryInd->getVPSingleValue()}));
   PrimaryInd->addOperand(InductionIncrement->getVPSingleValue());
   VPBB->appendRecipe(InductionIncrement);
 
+  VPValue *BTC = Plan->getOrCreateScalarBackedgeTakenCount();
+  auto *ExitCheck = cast<VPInstruction>(
+      new VPInstruction(VPInstruction::ICmpEQ, {InductionIncrement, BTC}));
+  VPBB->appendRecipe(ExitCheck);
+
+  auto *Branch = cast<VPInstruction>(
+      new VPInstruction(VPInstruction::ExitBranch, {ExitCheck}));
+
+  VPBB->appendRecipe(Branch);
+
   // From this point onwards, VPlan-to-VPlan transformations may change the plan
   // in ways that accessing values using original IR values is incorrect.
   Plan->disableValue2VPValue();
 
   VPlanTransforms::sinkScalarOperands(*Plan);
   VPlanTransforms::mergeReplicateRegions(*Plan);
 
   std::string PlanName;

llvm/lib/Transforms/Vectorize/VPlan.h

struct CFGState {
 
     /// The previous IR BasicBlock created or used. Initially set to the new
     /// header BasicBlock.
     BasicBlock *PrevBB = nullptr;
 
     /// The last IR BasicBlock in the output IR. Set to the new latch
     /// BasicBlock, used for placing the newly created BasicBlocks.
     BasicBlock *LastBB = nullptr;
+    BasicBlock *ExitBB = nullptr;
 
     /// The IR BasicBlock that is the preheader of the vector loop in the output
     /// IR.
     /// FIXME: The vector preheader should also be modeled in VPlan, so any code
     /// that needs to be added to the preheader gets directly generated by
     /// VPlan. There should be no need to manage a pointer to the IR BasicBlock.
     BasicBlock *VectorPreHeader = nullptr;
 

 public:
   /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
   enum {
     FirstOrderRecurrenceSplice =
         Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
                                       // values of a first-order recurrence.
     Not,
     ICmpULE,
+    ICmpEQ,
     SLPLoad,
     SLPStore,
     ActiveLaneMask,
     InductionIncrement,
     InductionIncrementNUW,
+    ExitBranch,
   };
 
 private:
   typedef unsigned char OpcodeTy;
   OpcodeTy Opcode;
 
   /// Utility method serving execute(): generates a single instance of the
   /// modeled instruction.
class VPlan {
   // VPlan. External definitions must be immutable and hold a pointer to its
   // underlying IR that will be used to implement its structural comparison
   // (operators '==' and '<').
   SetVector<VPValue *> VPExternalDefs;
 
   /// Represents the backedge taken count of the original loop, for folding
   /// the tail.
   VPValue *BackedgeTakenCount = nullptr;
+  VPValue *ScalarBackedgeTakenCount = nullptr;
 
   /// Holds a mapping between Values and their corresponding VPValue inside
   /// VPlan.
   Value2VPValueTy Value2VPValue;
 
   /// Contains all VPValues that been allocated by addVPValue directly and need
   /// to be free when the plan's destructor is called.
   SmallVector<VPValue *, 16> VPValuesToFree;
if (Entry) {
         Block->dropAllReferences(&DummyValue);
 
       VPBlockBase::deleteCFG(Entry);
     }
     for (VPValue *VPV : VPValuesToFree)
       delete VPV;
     if (BackedgeTakenCount)
       delete BackedgeTakenCount;
+    if (ScalarBackedgeTakenCount)
+      delete ScalarBackedgeTakenCount;
     for (VPValue *Def : VPExternalDefs)
       delete Def;
   }
 
   /// Generate the IR code for this VPlan.
   void execute(struct VPTransformState *State);
 
   VPBlockBase *getEntry() { return Entry; }
   const VPBlockBase *getEntry() const { return Entry; }
 
   VPBlockBase *setEntry(VPBlockBase *Block) {
     Entry = Block;
     Block->setPlan(this);
     return Entry;
   }
 
   /// The backedge taken count of the original loop.
   VPValue *getOrCreateBackedgeTakenCount() {
     if (!BackedgeTakenCount)
       BackedgeTakenCount = new VPValue();
     return BackedgeTakenCount;
   }
 
+  VPValue *getOrCreateScalarBackedgeTakenCount() {
+    if (!ScalarBackedgeTakenCount)
+      ScalarBackedgeTakenCount = new VPValue();
+    return ScalarBackedgeTakenCount;
+  }
+
   /// Mark the plan to indicate that using Value2VPValue is not safe any
   /// longer, because it may be stale.
   void disableValue2VPValue() { Value2VPValueEnabled = false; }
 
   void addVF(ElementCount VF) { VFs.insert(VF); }
 
   bool hasVF(ElementCount VF) { return VFs.count(VF); }
 

llvm/lib/Transforms/Vectorize/VPlan.cpp

if (PrevVPBB && /* A */
         PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
       !(Replica && getPredecessors().empty())) {       /* C */
     NewBB = createEmptyBasicBlock(State->CFG);
     State->Builder.SetInsertPoint(NewBB);
     // Temporarily terminate with unreachable until CFG is rewired.
     UnreachableInst *Terminator = State->Builder.CreateUnreachable();
     State->Builder.SetInsertPoint(Terminator);
     // Register NewBB in its loop. In innermost loops its the same for all BB's.
-    Loop *L = State->LI->getLoopFor(State->CFG.LastBB);
+    Loop *L = State->LI->getLoopFor(State->CFG.PrevBB);
     L->addBasicBlockToLoop(NewBB, *State->LI);
     State->CFG.PrevBB = NewBB;
   }
 
   // 2. Fill the IR basic block with IR instructions.
   LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()
                     << " in BB:" << NewBB->getName() << '\n');
 
void VPInstruction::generateInstruction(VPTransformState &State,
   }
   case VPInstruction::ICmpULE: {
     Value *IV = State.get(getOperand(0), Part);
     Value *TC = State.get(getOperand(1), Part);
     Value *V = Builder.CreateICmpULE(IV, TC);
     State.set(this, V, Part);
     break;
   }
+  case VPInstruction::ICmpEQ: {
+    if (Part == 0) {
+      Value *IV = State.get(getOperand(0), Part);
+      Value *TC = State.get(getOperand(1), Part);
+      Value *V = Builder.CreateICmpEQ(IV, TC);
+      State.set(this, V, Part);
+    }
+    break;
+  }
+  case VPInstruction::ExitBranch: {
+    if (Part == 0) {
+      Value *C = State.get(getOperand(0), Part);
+      auto *Plan = getParent()->getPlan();
+      VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
+      VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
+      Builder.CreateCondBr(C, State.CFG.ExitBB, State.CFG.VPBB2IRBB[Header]);
+      Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
+    }
+    break;
+  }
   case Instruction::Select: {
     Value *Cond = State.get(getOperand(0), Part);
     Value *Op1 = State.get(getOperand(1), Part);
     Value *Op2 = State.get(getOperand(2), Part);
     Value *V = Builder.CreateSelect(Cond, Op1, Op2);
     State.set(this, V, Part);
     break;
   }
if (Part == 0) {
       Value *Step = createStepForVF(
           Builder, ConstantInt::get(Phi->getType(), State.UF), State.VF);
       Value *Next = Builder.CreateAdd(Phi, Step, "index.next", IsNUW, false);
       State.set(getVPSingleValue(), Next, Part);
     }
 
     break;
   }
+
   default:
     llvm_unreachable("Unsupported opcode for instruction");
   }
 }
 
 void VPInstruction::execute(VPTransformState &State) {
   assert(!State.Instance && "VPInstruction executing an Instance");
   for (unsigned Part = 0; Part < State.UF; ++Part)
if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
     auto *TCMO = Builder.CreateSub(TC, ConstantInt::get(TC->getType(), 1),
                                    "trip.count.minus.1");
     auto VF = State->VF;
     Value *VTCMO =
         VF.isScalar() ? TCMO : Builder.CreateVectorSplat(VF, TCMO, "broadcast");
     for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part)
       State->set(BackedgeTakenCount, VTCMO, Part);
   }
+  if (ScalarBackedgeTakenCount && ScalarBackedgeTakenCount->getNumUsers()) {
+    for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part)
+      State->set(ScalarBackedgeTakenCount, State->VectorTripCount, Part);
+  }
 
   // 0. Set the reverse mapping from VPValues to Values for code generation.
   for (auto &Entry : Value2VPValue)
     State->VPValue2Value[Entry.second] = Entry.first;
 
   BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
   State->CFG.VectorPreHeader = VectorPreHeaderBB;
   BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
   assert(VectorHeaderBB && "Loop preheader does not have a single successor.");
 
   // 1. Make room to generate basic-blocks inside loop body if needed.
-  BasicBlock *VectorLatchBB = VectorHeaderBB->splitBasicBlock(
-      VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch");
   Loop *L = State->LI->getLoopFor(VectorHeaderBB);
-  L->addBasicBlockToLoop(VectorLatchBB, *State->LI);
+  State->CFG.PrevVPBB = nullptr;
+  State->CFG.PrevBB = VectorHeaderBB;
+  State->CFG.LastBB = L->getExitBlock();
+  State->CFG.ExitBB = L->getExitBlock();
+
   // Remove the edge between Header and Latch to allow other connections.
   // Temporarily terminate with unreachable until CFG is rewired.
   // Note: this asserts the generated code's assumption that
   // getFirstInsertionPt() can be dereferenced into an Instruction.
   VectorHeaderBB->getTerminator()->eraseFromParent();
   State->Builder.SetInsertPoint(VectorHeaderBB);
   UnreachableInst *Terminator = State->Builder.CreateUnreachable();
   State->Builder.SetInsertPoint(Terminator);
 
   // 2. Generate code in loop body.
-  State->CFG.PrevVPBB = nullptr;
-  State->CFG.PrevBB = VectorHeaderBB;
-  State->CFG.LastBB = VectorLatchBB;
-
   for (VPBlockBase *Block : depth_first(Entry))
     Block->execute(State);
 
+  if (!State->CFG.VPBBsToFix.empty()) {
+    State->CFG.VPBB2IRBB[cast<VPBasicBlock>(getEntry()->getExitBasicBlock())] =
+        State->CFG.ExitBB;
+    VPBasicBlock *OuterLatch = cast<VPBasicBlock>(
+        getEntry()->getExitBasicBlock()->getSinglePredecessor());
+    BasicBlock *BB = State->CFG.VPBB2IRBB[OuterLatch];
+    assert(BB && "Unexpected null basic block for VPBB");
+
+    unsigned Idx = 0;
+    auto *BBTerminator = BB->getTerminator();
+
+    for (VPBlockBase *SuccVPBlock : OuterLatch->getSuccessors()) {
+      VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
+      BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
+      ++Idx;
+    }
+  }
+
   // Setup branch terminator successors for VPBBs in VPBBsToFix based on
   // VPBB's successors.
   for (auto VPBB : State->CFG.VPBBsToFix) {
     assert(EnableVPlanNativePath &&
            "Unexpected VPBBsToFix in non VPlan-native path");
     BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
     assert(BB && "Unexpected null basic block for VPBB");
 
     unsigned Idx = 0;
     auto *BBTerminator = BB->getTerminator();
 
     for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
       VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
       BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
       ++Idx;
     }
   }
 
-  // 3. Merge the temporary latch created with the last basic-block filled.
-  BasicBlock *LastBB = State->CFG.PrevBB;
-  // Connect LastBB to VectorLatchBB to facilitate their merge.
-  assert((EnableVPlanNativePath ||
-          isa<UnreachableInst>(LastBB->getTerminator())) &&
-         "Expected InnerLoop VPlan CFG to terminate with unreachable");
-  assert((!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) &&
-         "Expected VPlan CFG to terminate with branch in NativePath");
-  LastBB->getTerminator()->eraseFromParent();
-  BranchInst::Create(VectorLatchBB, LastBB);
-
-  // Merge LastBB with Latch.
-  bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI);
-  (void)Merged;
-  assert(Merged && "Could not merge last basic block with latch.");
-  VectorLatchBB = LastBB;
-
+  BasicBlock *VectorLatchBB = State->CFG.PrevBB;
   // Fix the latch value of reduction and first-order recurrences phis in the
   // vector loop.
   VPBasicBlock *Header = Entry->getEntryBasicBlock();
   if (Header->empty()) {
     assert(EnableVPlanNativePath);
     Header = cast<VPBasicBlock>(Header->getSingleSuccessor());
   }
   for (VPRecipeBase &R : Header->phis()) {
     if (auto *Ind = dyn_cast<VPCanonicalIVRecipe>(&R)) {
       auto *BackedgeValue = cast<VPInstruction>(Ind->getOperand(1)->getDef());
       auto *P = cast<PHINode>(State->get(Ind->getVPSingleValue(), 0));
       BasicBlock *LatchBB = State->CFG.VPBB2IRBB[BackedgeValue->getParent()];
       P->addIncoming(State->get(BackedgeValue, 0), LatchBB);
-      auto *Next = cast<Instruction>(P->getIncomingValueForBlock(LatchBB));
-      auto *TermBr = cast<BranchInst>(LatchBB->getTerminator());
-      State->Builder.SetInsertPoint(TermBr);
-      auto *ICmp = cast<Instruction>(
-          State->Builder.CreateICmpEQ(Next, State->VectorTripCount));
-      TermBr->setCondition(ICmp);
-      Next->moveBefore(ICmp);
       continue;
     }
 
     if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
       if (!State->hasAnyVectorValue(IV->getStepValue()))
         continue;
       PHINode *VecInd = cast<PHINode>(State->get(IV->getPhiValue(), 0));
       Instruction *LastInduction =