Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3645,8 +3645,7 @@ truncateToMinimalBitwidths(State); // Fix widened non-induction PHIs by setting up the PHI operands. - if (EnableVPlanNativePath) - fixNonInductionPHIs(Plan, State); + fixNonInductionPHIs(Plan, State); // At this point every instruction in the original loop is widened to a // vector form. Now we need to fix the recurrences in the loop. These PHI @@ -8643,7 +8642,8 @@ // Now create the ActiveLaneMaskPhi recipe in the main loop using the // preheader ActiveLaneMask instruction. - auto *LaneMaskPhi = new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc()); + auto *LaneMaskPhi = new VPWidenPHIRecipe(/*Underlying PHI=*/nullptr); + LaneMaskPhi->addIncoming(EntryALM, Preheader); Header->insert(LaneMaskPhi, Header->getFirstNonPhi()); // Create the active lane mask for the next iteration of the loop. @@ -8657,7 +8657,7 @@ {CanonicalIVIncrementParts, TC}, DL, "active.lane.mask.next"); EB->appendRecipe(ALM); - LaneMaskPhi->addOperand(ALM); + LaneMaskPhi->addIncoming(ALM, EB); // We have to invert the mask here because a true condition means jumping // to the exit block. Index: llvm/lib/Transforms/Vectorize/VPlan.h =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.h +++ llvm/lib/Transforms/Vectorize/VPlan.h @@ -2706,7 +2706,7 @@ /// Find and return the VPActiveLaneMaskPHIRecipe from the header - there /// be only one at most. If there isn't one, then return nullptr. - VPActiveLaneMaskPHIRecipe *getActiveLaneMaskPhi(); + VPWidenPHIRecipe *getActiveLaneMaskPhi(); void addLiveOut(PHINode *PN, VPValue *V); Index: llvm/lib/Transforms/Vectorize/VPlan.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.cpp +++ llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -367,8 +367,8 @@ } // 2. Fill the IR basic block with IR instructions. - LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName() - << " in BB:" << NewBB->getName() << '\n'); + LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB(" << (void*)this << "):" << getName() + << " in BB(" << (void*)NewBB << "):" << NewBB->getName() << '\n'); State->CFG.VPBB2IRBB[this] = NewBB; State->CFG.PrevVPBB = this; @@ -568,11 +568,11 @@ } #endif -VPActiveLaneMaskPHIRecipe *VPlan::getActiveLaneMaskPhi() { +VPWidenPHIRecipe *VPlan::getActiveLaneMaskPhi() { VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock(); for (VPRecipeBase &R : Header->phis()) { - if (isa(&R)) - return cast(&R); + if (isa(&R) && !cast(&R)->getVPSingleValue()->getUnderlyingValue()) + return cast(&R); } return nullptr; } Index: llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1090,9 +1090,6 @@ #endif void VPWidenPHIRecipe::execute(VPTransformState &State) { - assert(EnableVPlanNativePath && - "Non-native vplans are not expected to have VPWidenPHIRecipes."); - // Currently we enter here in the VPlan-native path for non-induction // PHIs where all control flow is uniform. We simply widen these PHIs. // Create a vector phi with no operands - the vector phi operands will be @@ -1102,13 +1099,14 @@ unsigned StartIdx = 0; // For phis in header blocks of loop regions, use the index of the value // coming from the preheader. - if (LoopRegion->getEntryBasicBlock() == Parent) { + if (getUnderlyingValue() && LoopRegion->getEntryBasicBlock() == Parent) { for (unsigned I = 0; I < getNumOperands(); ++I) { if (getIncomingBlock(I) == LoopRegion->getSinglePredecessor()->getExitingBasicBlock()) StartIdx = I; } } + for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { Value *Start = State.get(getOperand(StartIdx), Part); Type *VecTy = Start->getType(); @@ -1123,14 +1121,16 @@ VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN-PHI "; - auto *OriginalPhi = cast(getUnderlyingValue()); - // Unless all incoming values are modeled in VPlan print the original PHI - // directly. - // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming - // values as VPValues. - if (getNumOperands() != OriginalPhi->getNumOperands()) { - O << VPlanIngredient(OriginalPhi); - return; + if (getUnderlyingValue()) { + auto *OriginalPhi = cast(getUnderlyingValue()); + // Unless all incoming values are modeled in VPlan print the original PHI + // directly. + // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming + // values as VPValues. + if (getNumOperands() != OriginalPhi->getNumOperands()) { + O << VPlanIngredient(OriginalPhi); + return; + } } printAsOperand(O, SlotTracker);