[WIP][LoopVectorize] Replace VPActiveLaneMaskPHIRecipe with VPWidenPHIRecipe

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

 void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
                                             VPlan &Plan) {
   // Insert truncates and extends for any truncated instructions as hints to
   // InstCombine.
   if (VF.isVector())
     truncateToMinimalBitwidths(State);
 
   // Fix widened non-induction PHIs by setting up the PHI operands.
-  if (EnableVPlanNativePath)
-    fixNonInductionPHIs(Plan, State);
 
   // At this point every instruction in the original loop is widened to a
   // vector form. Now we need to fix the recurrences in the loop. These PHI
   // nodes are currently empty because we did not want to introduce cycles.
   // This is the second stage of vectorizing recurrences.
   fixCrossIterationPHIs(State);
 
   // Forget the original basic block.
if (UseLaneMaskForLoopControlFlow) {
     VPValue *TC = Plan.getOrCreateTripCount();
     auto *EntryALM = new VPInstruction(VPInstruction::ActiveLaneMask,
                                        {CanonicalIVIncrementParts, TC}, DL,
                                        "active.lane.mask.entry");
     Preheader->appendRecipe(EntryALM);
 
     // Now create the ActiveLaneMaskPhi recipe in the main loop using the
     // preheader ActiveLaneMask instruction.
-    auto *LaneMaskPhi = new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc());
+    auto *LaneMaskPhi = new VPWidenPHIRecipe(/*Underlying PHI=*/nullptr);
+    LaneMaskPhi->addIncoming(EntryALM, Preheader);
     Header->insert(LaneMaskPhi, Header->getFirstNonPhi());
 
     // Create the active lane mask for the next iteration of the loop.
     CanonicalIVIncrementParts =
         new VPInstruction(HasNUW ? VPInstruction::CanonicalIVIncrementForPartNUW
                                  : VPInstruction::CanonicalIVIncrementForPart,
                           {CanonicalIVIncrement}, DL);
     EB->appendRecipe(CanonicalIVIncrementParts);
 
     auto *ALM = new VPInstruction(VPInstruction::ActiveLaneMask,
                                   {CanonicalIVIncrementParts, TC}, DL,
                                   "active.lane.mask.next");
     EB->appendRecipe(ALM);
-    LaneMaskPhi->addOperand(ALM);
+    LaneMaskPhi->addIncoming(ALM, EB);
 
     // We have to invert the mask here because a true condition means jumping
     // to the exit block.
     auto *NotMask = new VPInstruction(VPInstruction::Not, ALM, DL);
     EB->appendRecipe(NotMask);
 
     VPInstruction *BranchBack =
         new VPInstruction(VPInstruction::BranchOnCond, {NotMask}, DL);

llvm/lib/Transforms/Vectorize/VPlan.h

if (EntryVPBB->empty()) {
       // VPlan native path.
       EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
     }
     return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
   }
 
   /// Find and return the VPActiveLaneMaskPHIRecipe from the header - there
   /// be only one at most. If there isn't one, then return nullptr.
-  VPActiveLaneMaskPHIRecipe *getActiveLaneMaskPhi();
+  VPWidenPHIRecipe *getActiveLaneMaskPhi();
 
   void addLiveOut(PHINode *PN, VPValue *V);
 
   void clearLiveOuts() {
     for (auto &KV : LiveOuts)
       delete KV.second;
     LiveOuts.clear();
   }

llvm/lib/Transforms/Vectorize/VPlan.cpp

if (getPlan()->getVectorLoopRegion()->getSingleSuccessor() == this) {
     // BB's.
     if (State->CurrentVectorLoop)
       State->CurrentVectorLoop->addBasicBlockToLoop(NewBB, *State->LI);
     State->Builder.SetInsertPoint(Terminator);
     State->CFG.PrevBB = NewBB;
   }
 
   // 2. Fill the IR basic block with IR instructions.
-  LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()
-                    << " in BB:" << NewBB->getName() << '\n');
+  LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB(" << (void*)this << "):" << getName()
+                    << " in BB(" << (void*)NewBB << "):" << NewBB->getName() << '\n');
 
   State->CFG.VPBB2IRBB[this] = NewBB;
   State->CFG.PrevVPBB = this;
 
   for (VPRecipeBase &Recipe : Recipes)
     Recipe.execute(*State);
 
   LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB);
for (auto *BlockBase : depth_first(Entry)) {
     BlockBase->print(O, NewIndent, SlotTracker);
   }
   O << Indent << "}\n";
 
   printSuccessors(O, Indent);
 }
 #endif
 
-VPActiveLaneMaskPHIRecipe *VPlan::getActiveLaneMaskPhi() {
+VPWidenPHIRecipe *VPlan::getActiveLaneMaskPhi() {
   VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
   for (VPRecipeBase &R : Header->phis()) {
-    if (isa<VPActiveLaneMaskPHIRecipe>(&R))
-      return cast<VPActiveLaneMaskPHIRecipe>(&R);
+    if (isa<VPWidenPHIRecipe>(&R) && !cast<VPWidenPHIRecipe>(&R)->getVPSingleValue()->getUnderlyingValue())
+      return cast<VPWidenPHIRecipe>(&R);
   }
   return nullptr;
 }
 
 static bool canSimplifyBranchOnCond(VPInstruction *Term) {
   VPInstruction *Not = dyn_cast<VPInstruction>(Term->getOperand(0));
   if (!Not || Not->getOpcode() != VPInstruction::Not)
     return false;

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
 
   printAsOperand(O, SlotTracker);
   O << " = phi ";
   printOperands(O, SlotTracker);
 }
 #endif
 
 void VPWidenPHIRecipe::execute(VPTransformState &State) {
-  assert(EnableVPlanNativePath &&
-         "Non-native vplans are not expected to have VPWidenPHIRecipes.");
-
   // Currently we enter here in the VPlan-native path for non-induction
   // PHIs where all control flow is uniform. We simply widen these PHIs.
   // Create a vector phi with no operands - the vector phi operands will be
   // set at the end of vector code generation.
   VPBasicBlock *Parent = getParent();
   VPRegionBlock *LoopRegion = Parent->getEnclosingLoopRegion();
   unsigned StartIdx = 0;
   // For phis in header blocks of loop regions, use the index of the value
   // coming from the preheader.
-  if (LoopRegion->getEntryBasicBlock() == Parent) {
+  if (getUnderlyingValue() && LoopRegion->getEntryBasicBlock() == Parent) {
     for (unsigned I = 0; I < getNumOperands(); ++I) {
       if (getIncomingBlock(I) ==
           LoopRegion->getSinglePredecessor()->getExitingBasicBlock())
         StartIdx = I;
     }
   }
+
   for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
     Value *Start = State.get(getOperand(StartIdx), Part);
     Type *VecTy = Start->getType();
     PHINode *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
     VecPhi->addIncoming(Start, State.CFG.VPBB2IRBB[getIncomingBlock(StartIdx)]);
     State.set(this, VecPhi, 0);
   }
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
                              VPSlotTracker &SlotTracker) const {
   O << Indent << "WIDEN-PHI ";
 
+  if (getUnderlyingValue()) {
     auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
     // Unless all incoming values are modeled in VPlan  print the original PHI
     // directly.
     // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming
     // values as VPValues.
     if (getNumOperands() != OriginalPhi->getNumOperands()) {
       O << VPlanIngredient(OriginalPhi);
       return;
     }
+  }
 
   printAsOperand(O, SlotTracker);
   O << " = phi ";
   printOperands(O, SlotTracker);
 }
 #endif
 
 // TODO: It would be good to use the existing VPWidenPHIRecipe instead and