[VPlan] Generalize VPWidenPHIRecipe to generate values for all parts.

[fhahn]

Summary

This extends VPWidenPHIRecipe::execute to generate code for all parts.
This should allow using the recipe also in cases where vector phis are
needed for all parts, as in D125301.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

void InnerLoopVectorizer::fixNonInductionPHIs(VPlan &Plan,
                                               VPTransformState &State) {
   auto Iter = depth_first(
       VPBlockRecursiveTraversalWrapper<VPBlockBase *>(Plan.getEntry()));
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
     for (VPRecipeBase &P : VPBB->phis()) {
       VPWidenPHIRecipe *VPPhi = dyn_cast<VPWidenPHIRecipe>(&P);
       if (!VPPhi)
         continue;
-      PHINode *NewPhi = cast<PHINode>(State.get(VPPhi, 0));
+      for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
+        PHINode *NewPhi = cast<PHINode>(State.get(VPPhi, Part));
         // Make sure the builder has a valid insert point.
         Builder.SetInsertPoint(NewPhi);
         for (unsigned i = 0; i < VPPhi->getNumOperands(); ++i) {
           VPValue *Inc = VPPhi->getIncomingValue(i);
-        VPBasicBlock *VPBB = VPPhi->getIncomingBlock(i);
-        NewPhi->addIncoming(State.get(Inc, 0), State.CFG.VPBB2IRBB[VPBB]);
+          BasicBlock *BB = State.CFG.VPBB2IRBB[VPPhi->getIncomingBlock(i)];
+          if (is_contained(NewPhi->blocks(), BB))
+            continue;
+          NewPhi->addIncoming(State.get(Inc, Part), BB);
+        }
       }
     }
   }
 }
 
 bool InnerLoopVectorizer::useOrderedReductions(
     const RecurrenceDescriptor &RdxDesc) {
   return Cost->useOrderedReductions(RdxDesc);

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

void VPWidenPHIRecipe::execute(VPTransformState &State) {
   // coming from the preheader.
   if (LoopRegion->getEntryBasicBlock() == Parent) {
     for (unsigned I = 0; I < getNumOperands(); ++I) {
       if (getIncomingBlock(I) ==
           LoopRegion->getSinglePredecessor()->getExitingBasicBlock())
         StartIdx = I;
     }
   }
-  Value *Op0 = State.get(getOperand(StartIdx), 0);
-  Type *VecTy = Op0->getType();
-  Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
+  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
+    Value *Start = State.get(getOperand(StartIdx), Part);
+    Type *VecTy = Start->getType();
+    PHINode *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
+    VecPhi->addIncoming(Start, State.CFG.VPBB2IRBB[getIncomingBlock(StartIdx)]);
     State.set(this, VecPhi, 0);
   }
+}
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
                              VPSlotTracker &SlotTracker) const {
   O << Indent << "WIDEN-PHI ";
 
   auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
   // Unless all incoming values are modeled in VPlan  print the original PHI