Index: llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -292,8 +292,7 @@ /// Build a VPlan using VPRecipes according to the information gather by /// Legal. This method is only used for the legacy inner loop vectorizer. VPlanPtr buildVPlanWithVPRecipes( - VFRange &Range, SmallPtrSetImpl &NeedDef, - SmallPtrSetImpl &DeadInstructions, + VFRange &Range, SmallPtrSetImpl &DeadInstructions, const DenseMap &SinkAfter); /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive, Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7220,7 +7220,7 @@ if (!BI->isConditional() || BI->getSuccessor(0) == BI->getSuccessor(1)) return EdgeMaskCache[Edge] = SrcMask; - VPValue *EdgeMask = Plan->getVPValue(BI->getCondition()); + VPValue *EdgeMask = Plan->getOrAddVPValue(BI->getCondition()); assert(EdgeMask && "No Edge Mask found for condition"); if (BI->getSuccessor(0) != Dst) @@ -7258,7 +7258,7 @@ // Start by constructing the desired canonical IV. VPValue *IV = nullptr; if (Legal->getPrimaryInduction()) - IV = Plan->getVPValue(Legal->getPrimaryInduction()); + IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction()); else { auto IVRecipe = new VPWidenCanonicalIVRecipe(); Builder.getInsertBlock()->insert(IVRecipe, NewInsertionPoint); @@ -7605,32 +7605,6 @@ unsigned MaxVF) { assert(OrigLoop->isInnermost() && "Inner loop expected."); - // Collect conditions feeding internal conditional branches; they need to be - // represented in VPlan for it to model masking. - SmallPtrSet NeedDef; - - auto *Latch = OrigLoop->getLoopLatch(); - for (BasicBlock *BB : OrigLoop->blocks()) { - if (BB == Latch) - continue; - BranchInst *Branch = dyn_cast(BB->getTerminator()); - if (Branch && Branch->isConditional()) - NeedDef.insert(Branch->getCondition()); - } - - // If the tail is to be folded by masking, the primary induction variable, if - // exists needs to be represented in VPlan for it to model early-exit masking. - // Also, both the Phi and the live-out instruction of each reduction are - // required in order to introduce a select between them in VPlan. - if (CM.foldTailByMasking()) { - if (Legal->getPrimaryInduction()) - NeedDef.insert(Legal->getPrimaryInduction()); - for (auto &Reduction : Legal->getReductionVars()) { - NeedDef.insert(Reduction.first); - NeedDef.insert(Reduction.second.getLoopExitInstr()); - } - } - // Collect instructions from the original loop that will become trivially dead // in the vectorized loop. We don't need to vectorize these instructions. For // example, original induction update instructions can become dead because we @@ -7654,15 +7628,14 @@ for (unsigned VF = MinVF; VF < MaxVF + 1;) { VFRange SubRange = {VF, MaxVF + 1}; - VPlans.push_back(buildVPlanWithVPRecipes(SubRange, NeedDef, - DeadInstructions, SinkAfter)); + VPlans.push_back( + buildVPlanWithVPRecipes(SubRange, DeadInstructions, SinkAfter)); VF = SubRange.End; } } VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( - VFRange &Range, SmallPtrSetImpl &NeedDef, - SmallPtrSetImpl &DeadInstructions, + VFRange &Range, SmallPtrSetImpl &DeadInstructions, const DenseMap &SinkAfter) { // Hold a mapping from predicated instructions to their recipes, in order to @@ -7731,10 +7704,6 @@ VPBasicBlock *VPBB = new VPBasicBlock("Pre-Entry"); Plan->setEntry(VPBB); - // Represent values that will have defs inside VPlan. - for (Value *V : NeedDef) - Plan->addVPValue(V); - // Scan the body of the loop in a topological order to visit each basic block // after having visited its predecessor basic blocks. LoopBlocksDFS DFS(OrigLoop); @@ -7764,12 +7733,8 @@ // Check if the recipe can be converted to a VPValue. We need the extra // down-casting step until VPRecipeBase inherits from VPValue. VPValue *MaybeVPValue = Recipe->toVPValue(); - if (!Instr->getType()->isVoidTy() && MaybeVPValue) { - if (NeedDef.contains(Instr)) - Plan->addOrReplaceVPValue(Instr, MaybeVPValue); - else - Plan->addVPValue(Instr, MaybeVPValue); - } + if (!Instr->getType()->isVoidTy() && MaybeVPValue) + Plan->addVPValue(Instr, MaybeVPValue); RecipeBuilder.setRecipe(Instr, Recipe); VPBB->appendRecipe(Recipe); @@ -7825,6 +7790,7 @@ NewVPV = new VPValue(Member); Plan->getVPValue(Member)->replaceAllUsesWith(NewVPV); } + Plan->removeVPValueFor(Member); RecipeBuilder.getRecipe(Member)->eraseFromParent(); if (NewVPV) Plan->addVPValue(Member, NewVPV); @@ -7843,8 +7809,8 @@ for (auto &Reduction : Legal->getReductionVars()) { if (CM.isInLoopReduction(Reduction.first)) continue; - VPValue *Phi = Plan->getVPValue(Reduction.first); - VPValue *Red = Plan->getVPValue(Reduction.second.getLoopExitInstr()); + VPValue *Phi = Plan->getOrAddVPValue(Reduction.first); + VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr()); Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi}); } } @@ -7943,6 +7909,11 @@ VPReductionRecipe *RedRecipe = new VPReductionRecipe( &RdxDesc, R, ChainOp, VecOp, CondOp, Legal->hasFunNoNaNAttr(), UseReductionIntrinsic); + if (VPValue* V = WidenRecipe->toVPValue()) { + V->replaceAllUsesWith(RedRecipe); + Plan->removeVPValueFor(R); + Plan->addVPValue(R, RedRecipe); + } WidenRecipe->getParent()->insert(RedRecipe, WidenRecipe->getIterator()); WidenRecipe->eraseFromParent(); @@ -7994,7 +7965,7 @@ } void VPWidenRecipe::execute(VPTransformState &State) { - State.ILV->widenInstruction(Ingredient, *this, State); + State.ILV->widenInstruction(*getUnderlyingInstr(), *this, State); } void VPWidenGEPRecipe::execute(VPTransformState &State) { Index: llvm/lib/Transforms/Vectorize/VPlan.h =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.h +++ llvm/lib/Transforms/Vectorize/VPlan.h @@ -816,14 +816,12 @@ /// VPWidenRecipe is a recipe for producing a copy of vector type its /// ingredient. This recipe covers most of the traditional vectorization cases /// where each ingredient transforms into a vectorized version of itself. -class VPWidenRecipe : public VPRecipeBase, public VPUser { - /// Hold the instruction to be widened. - Instruction &Ingredient; - +class VPWidenRecipe : public VPRecipeBase, public VPValue, public VPUser { public: template VPWidenRecipe(Instruction &I, iterator_range Operands) - : VPRecipeBase(VPWidenSC), VPUser(Operands), Ingredient(I) {} + : VPRecipeBase(VPRecipeBase::VPWidenSC), VPValue(VPValue::VPWidenSC, &I), + VPUser(Operands) {} ~VPWidenRecipe() override = default; @@ -831,6 +829,9 @@ static inline bool classof(const VPRecipeBase *V) { return V->getVPRecipeID() == VPRecipeBase::VPWidenSC; } + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPValue::VPWidenSC; + } /// Produce widened copies of all Ingredients. void execute(VPTransformState &State) override; @@ -1727,15 +1728,6 @@ Value2VPValue[V] = VPV; } - void addOrReplaceVPValue(Value *V, VPValue *VPV) { - assert(V && "Trying to add a null Value to VPlan"); - auto I = Value2VPValue.find(V); - if (I == Value2VPValue.end()) - Value2VPValue[V] = VPV; - else - I->second = VPV; - } - VPValue *getVPValue(Value *V) { assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); Index: llvm/lib/Transforms/Vectorize/VPlan.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.cpp +++ llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -109,6 +109,8 @@ return V; if (auto *V = dyn_cast(this)) return V; + if (auto *V = dyn_cast(this)) + return V; return nullptr; } @@ -119,6 +121,8 @@ return V; if (auto *V = dyn_cast(this)) return V; + if (auto *V = dyn_cast(this)) + return V; return nullptr; } @@ -428,13 +432,6 @@ iplist::iterator VPRecipeBase::eraseFromParent() { assert(getParent() && "Recipe not in any VPBasicBlock"); - // If the recipe is a VPValue and has been added to the containing VPlan, - // remove the mapping. - if (Value *UV = getUnderlyingInstr()) - if (!UV->getType()->isVoidTy()) - if (auto *Plan = getParent()->getPlan()) - Plan->removeVPValueFor(UV); - return getParent()->getRecipeList().erase(getIterator()); } @@ -847,7 +844,7 @@ void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << "\"WIDEN\\l\""; - O << "\" " << VPlanIngredient(&Ingredient); + O << "\" " << VPlanIngredient(getUnderlyingInstr()); } void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent, Index: llvm/lib/Transforms/Vectorize/VPlanValue.h =================================================================== --- llvm/lib/Transforms/Vectorize/VPlanValue.h +++ llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -78,7 +78,13 @@ /// are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPValue objects. They are used for concrete /// type identification. - enum { VPValueSC, VPInstructionSC, VPMemoryInstructionSC, VPReductionSC }; + enum { + VPValueSC, + VPInstructionSC, + VPMemoryInstructionSC, + VPReductionSC, + VPWidenSC, + }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} VPValue(const VPValue &) = delete;