diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8836,6 +8836,32 @@ Inst2VPInst[Entry.second] = nullptr; } + VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder); + + // First, try to convert VPInstructions to recipes, if the conversion does not + // depend on the VF. At the moment, only VPIntOrFpInductionRecipes are + // created here. + for (VPRecipeBase &R : + make_early_inc_range(*TopRegion->getEntryBasicBlock())) { + auto *VPhi = dyn_cast(&R); + if (!VPhi) + break; + + auto *Phi = cast(VPhi->getUnderlyingValue()); + if (auto *NewPhi = RecipeBuilder.tryToOptimizeInductionPHI( + Phi, ArrayRef(R.op_begin(), R.op_end()))) { + for (auto *NewVPV : NewPhi->definedValues()) { + Value *UV = NewVPV->getUnderlyingValue(); + VPValue *OldVPV = OriginalPlan.getVPValue(UV); + OldVPV->replaceAllUsesWith(NewVPV); + OriginalPlan.removeVPValueFor(UV); + OriginalPlan.addVPValue(UV, NewVPV); + } + NewPhi->insertBefore(VPhi); + VPhi->eraseFromParent(); + } + } + // Remove dead instructions from original VPlan. auto *DummyVal = new VPValue(); OriginalPlan.addExternalDef(DummyVal); @@ -8927,8 +8953,8 @@ // Create a dummy pre-entry VPBasicBlock to start building the VPlan. auto Plan = std::make_unique(); - ReversePostOrderTraversal RPOT( - OriginalPlan.getEntry()->getEntryBasicBlock()); + VPBasicBlock *Header = OriginalPlan.getEntry()->getEntryBasicBlock(); + ReversePostOrderTraversal RPOT(Header); VPBasicBlock *VPBB = nullptr; for (VPBlockBase *Base : RPOT) { @@ -8948,6 +8974,21 @@ // Introduce each ingredient into VPlan. for (auto I = OriginalVPBB->begin(), E = OriginalVPBB->end(); I != E;) { VPRecipeBase *Ingredient = &*I++; + // First, clone & add the recipes introduced earlier. + if (isa(Ingredient)) { + auto *C = Ingredient->clone(); + for (unsigned I = 0, E = C->getNumDefinedValues(); I != E; ++I) { + auto *NewDef = C->getVPValue(I); + auto *UV = NewDef->getUnderlyingValue(); + Plan->addVPValue(UV, NewDef); + Ingredient->getVPValue(I)->replaceAllUsesWith(NewDef); + } + for (unsigned I = 0, E = Ingredient->getNumOperands(); I != E; ++I) + C->setOperand(I, Plan->getOrAddVPValue( + Ingredient->getOperand(I)->getLiveInIRValue())); + VPBB->appendRecipe(C); + continue; + } VPInstruction *VPInst = dyn_cast(Ingredient); Instruction *Instr = VPInst diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -67,11 +67,6 @@ VPRecipeBase *tryToWidenMemory(Instruction *I, ArrayRef Operands, VFRange &Range, VPlanPtr &Plan); - /// Check if an induction recipe should be constructed for \I. If so build and - /// return it. If not, return null. - VPWidenIntOrFpInductionRecipe * - tryToOptimizeInductionPHI(PHINode *Phi, ArrayRef Operands) const; - /// Optimize the special case where the operand of \p I is a constant integer /// induction variable. VPWidenIntOrFpInductionRecipe * @@ -115,6 +110,11 @@ ArrayRef Operands, VFRange &Range, VPlanPtr &Plan); + /// Check if an induction recipe should be constructed for \I. If so build and + /// return it. If not, return null. + VPWidenIntOrFpInductionRecipe * + tryToOptimizeInductionPHI(PHINode *Phi, ArrayRef Operands) const; + /// Set the recipe created for given ingredient. This operation is a no-op for /// ingredients that were not marked using a nullptr entry in the map. void setRecipe(Instruction *I, VPRecipeBase *R) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -100,6 +100,19 @@ assert(VPPhi->getNumOperands() == 0 && "Expected VPInstruction with no operands."); + if (Phi->getParent() == TheLoop->getHeader()) { + VPPhi->addIncoming(getOrCreateVPOperand(Phi->getIncomingValueForBlock( + TheLoop->getLoopPreheader())), + BB2VPBB[TheLoop->getLoopPreheader()]); + for (BasicBlock *IncBlock : Phi->blocks()) { + if (IncBlock == TheLoop->getLoopPreheader()) + continue; + VPPhi->addIncoming( + getOrCreateVPOperand(Phi->getIncomingValueForBlock(IncBlock)), + BB2VPBB[IncBlock]); + } + continue; + } for (unsigned I = 0; I != Phi->getNumOperands(); ++I) VPPhi->addIncoming(getOrCreateVPOperand(Phi->getIncomingValue(I)), BB2VPBB[Phi->getIncomingBlock(I)]); @@ -234,6 +247,7 @@ } IRDef2VPValue[Inst] = NewVPV; + Plan.addVPValue(Inst, NewVPV); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -51,6 +51,7 @@ VPValue *Start = Plan->getOrAddVPValue(II.getStartValue()); NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, nullptr); } else { + Plan->removeVPValueFor(Phi); Plan->addVPValue(Phi, VPPhi); continue; }