diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -295,20 +295,11 @@ /// exclusive, possibly decreasing \p Range.End. VPlanPtr buildVPlan(const VFRange &Range); - /// Build a VPlan using VPRecipes according to the information gather by - /// Legal. This method is only used for the legacy inner loop vectorizer. - VPlanPtr buildVPlanWithVPRecipes( - VFRange &Range, SmallPtrSetImpl &DeadInstructions, - const DenseMap &SinkAfter); - - VPlanPtr - convertToVPRecipes(VPlan &OriginalPlan, VFRange &Range, - const DenseMap &SinkAfter); + VPlanPtr convertToVPRecipes(VPlan &OriginalPlan, VFRange &Range); /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive, /// according to the information gathered by Legal when it checked if it is /// legal to vectorize the loop. This method creates VPlans using VPRecipes. - void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF); void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF, VPlan &OriginalPlan); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8744,6 +8744,12 @@ ReversePostOrderTraversal RPOT( OriginalPlan.getEntry()->getEntryBasicBlock()); + DenseMap Inst2VPInst; + for (auto &Entry : SinkAfter) { + Inst2VPInst[Entry.first] = nullptr; + Inst2VPInst[Entry.second] = nullptr; + } + auto *DummyVal = new VPValue(); OriginalPlan.addExternalDef(DummyVal); for (VPBlockBase *Base : RPOT) { @@ -8759,21 +8765,31 @@ if (DeadInstructions.contains(Instr) || isa(Instr)) { Ingredient->getVPValue()->replaceAllUsesWith(DummyVal); Ingredient->eraseFromParent(); + continue; } + + auto Iter = Inst2VPInst.find(Instr); + if (Iter != Inst2VPInst.end()) + Iter->second = VPInst; } } + for (auto &Entry : SinkAfter) { + VPRecipeBase *Sink = Inst2VPInst.find(Entry.first)->second; + VPRecipeBase *Target = Inst2VPInst.find(Entry.second)->second; + Sink->moveAfter(Target); + } + auto MaxVFPlusOne = MaxVF.getWithIncrement(1); for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) { VFRange SubRange = {VF, MaxVFPlusOne}; - VPlans.push_back(convertToVPRecipes(OriginalPlan, SubRange, SinkAfter)); + VPlans.push_back(convertToVPRecipes(OriginalPlan, SubRange)); VF = SubRange.End; } } -VPlanPtr LoopVectorizationPlanner::convertToVPRecipes( - VPlan &OriginalPlan, VFRange &Range, - const DenseMap &SinkAfter) { +VPlanPtr LoopVectorizationPlanner::convertToVPRecipes(VPlan &OriginalPlan, + VFRange &Range) { // Hold a mapping from predicated instructions to their recipes, in order to // fix their AlsoPack behavior if a user is determined to replicate and use a @@ -8789,12 +8805,6 @@ // process after constructing the initial VPlan. // --------------------------------------------------------------------------- - // Mark instructions we'll need to sink later and their targets as - // ingredients whose recipe we'll need to record. - for (auto &Entry : SinkAfter) { - RecipeBuilder.recordRecipeOf(Entry.first); - RecipeBuilder.recordRecipeOf(Entry.second); - } for (auto &Reduction : CM.getInLoopReductionChains()) { PHINode *Phi = Reduction.first; RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind(); @@ -8909,24 +8919,6 @@ // bring the VPlan to its final state. // --------------------------------------------------------------------------- - // Apply Sink-After legal constraints. - for (auto &Entry : SinkAfter) { - VPRecipeBase *Sink = RecipeBuilder.getRecipe(Entry.first); - VPRecipeBase *Target = RecipeBuilder.getRecipe(Entry.second); - // If the target is in a replication region, make sure to move Sink to the - // block after it, not into the replication region itself. - if (auto *Region = - dyn_cast_or_null(Target->getParent()->getParent())) { - if (Region->isReplicator()) { - assert(Region->getNumSuccessors() == 1 && "Expected SESE region!"); - VPBasicBlock *NextBlock = - cast(Region->getSuccessors().front()); - Sink->moveBefore(*NextBlock, NextBlock->getFirstNonPhi()); - continue; - } - } - Sink->moveAfter(Target); - } // Interleave memory: for each Interleave Group we marked earlier as relevant // for this VPlan, replace the Recipes widening its memory instructions with a