diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8750,11 +8750,38 @@ Inst2VPInst[Entry.second] = nullptr; } + VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder); + auto *DummyVal = new VPValue(); OriginalPlan.addExternalDef(DummyVal); for (VPBlockBase *Base : RPOT) { VPBasicBlock *OriginalVPBB = Base->getEntryBasicBlock(); - // Introduce each ingredient into VPlan. + + // First, convert some VPInstructions to recipes, if the conversion does not + // depend on the VF. At the moment, only VPIntOrFpInductionRecipes are + // created here. + for (auto I = OriginalVPBB->begin(), E = OriginalVPBB->end(); I != E;) { + VPRecipeBase *Ingredient = &*I++; + auto *VPhi = dyn_cast(Ingredient); + if (!VPhi) + break; + + auto *Phi = cast(VPhi->getUnderlyingValue()); + if (auto *NewPhi = + RecipeBuilder.tryToOptimizeInductionPHI(Phi, OriginalPlan)) { + for (auto *NewVPV : NewPhi->definedValues()) { + Value *UV = NewVPV->getUnderlyingValue(); + VPValue *OldVPV = OriginalPlan.getVPValue(UV); + OldVPV->replaceAllUsesWith(NewVPV); + OriginalPlan.removeVPValueFor(UV); + OriginalPlan.addVPValue(UV, NewVPV); + } + NewPhi->insertBefore(VPhi); + VPhi->eraseFromParent(); + } + } + + // Remove dead instructions. for (auto I = OriginalVPBB->rbegin(), E = OriginalVPBB->rend(); I != E;) { VPRecipeBase *Ingredient = &*I++; VPInstruction *VPInst = dyn_cast(Ingredient); @@ -8872,6 +8899,18 @@ // Introduce each ingredient into VPlan. for (auto I = OriginalVPBB->begin(), E = OriginalVPBB->end(); I != E;) { VPRecipeBase *Ingredient = &*I++; + if (isa(Ingredient)) { + auto *C = Ingredient->clone(); + for (auto *Def : C->definedValues()) { + auto *UV = Def->getUnderlyingValue(); + Plan->addVPValue(UV, Def); + } + for (unsigned I = 0, E = Ingredient->getNumOperands(); I != E; ++I) + C->setOperand(I, Plan->getOrAddVPValue( + Ingredient->getOperand(I)->getLiveInIRValue())); + VPBB->appendRecipe(C); + continue; + } VPInstruction *VPInst = dyn_cast(Ingredient); Instruction *Instr = VPInst diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -67,11 +67,6 @@ VPRecipeBase *tryToWidenMemory(Instruction *I, VFRange &Range, VPlanPtr &Plan); - /// Check if an induction recipe should be constructed for \I. If so build and - /// return it. If not, return null. - VPWidenIntOrFpInductionRecipe *tryToOptimizeInductionPHI(PHINode *Phi, - VPlan &Plan) const; - /// Optimize the special case where the operand of \p I is a constant integer /// induction variable. VPWidenIntOrFpInductionRecipe * @@ -113,6 +108,11 @@ VPRecipeOrVPValueTy tryToCreateWidenRecipe(Instruction *Instr, VFRange &Range, VPlanPtr &Plan); + /// Check if an induction recipe should be constructed for \I. If so build and + /// return it. If not, return null. + VPWidenIntOrFpInductionRecipe *tryToOptimizeInductionPHI(PHINode *Phi, + VPlan &Plan) const; + /// Set the recipe created for given ingredient. This operation is a no-op for /// ingredients that were not marked using a nullptr entry in the map. void setRecipe(Instruction *I, VPRecipeBase *R) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -234,6 +234,7 @@ } IRDef2VPValue[Inst] = NewVPV; + Plan.addVPValue(Inst, NewVPV); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -51,6 +51,7 @@ VPValue *Start = Plan->getOrAddVPValue(II.getStartValue()); NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, nullptr); } else { + Plan->removeVPValueFor(Phi); Plan->addVPValue(Phi, VPPhi); continue; } diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -659,6 +659,8 @@ void execute(struct VPTransformState &State) override{}; void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override {} + + VPRecipeBase *clone() override { return nullptr; } }; TEST(VPDoubleValueDefTest, traverseUseLists) {