diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -306,17 +306,9 @@ /// exclusive, possibly decreasing \p Range.End. VPlanPtr buildVPlan(const VFRange &Range); - /// Build a VPlan using VPRecipes according to the information gather by - /// Legal. This method is only used for the legacy inner loop vectorizer. - VPlanPtr buildVPlanWithVPRecipes( - VFRange &Range, SmallPtrSetImpl &DeadInstructions, - const DenseMap &SinkAfter); - /// Returns a new VPlan based on \p OriginalPlan, but with all VPInstruction /// in \p OriginalPlan lowered to widened recipes. - VPlanPtr - convertToVPRecipes(VPlan &OriginalPlan, VFRange &Range, - const DenseMap &SinkAfter); + VPlanPtr convertToVPRecipes(VPlan &OriginalPlan, VFRange &Range); /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive, /// according to the information gathered by Legal when it checked if it is diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8830,6 +8830,12 @@ for (Instruction *I : DeadInstructions) SinkAfter.erase(I); + DenseMap Inst2VPInst; + for (auto &Entry : SinkAfter) { + Inst2VPInst[Entry.first] = nullptr; + Inst2VPInst[Entry.second] = nullptr; + } + // Remove dead instructions from original VPlan. auto *DummyVal = new VPValue(); OriginalPlan.addExternalDef(DummyVal); @@ -8847,20 +8853,29 @@ Ingredient.getVPValue()->replaceAllUsesWith(DummyVal); Ingredient.eraseFromParent(); } + + auto Iter = Inst2VPInst.find(Instr); + if (Iter != Inst2VPInst.end()) + Iter->second = VPInst; } } + for (auto &Entry : SinkAfter) { + VPRecipeBase *Sink = Inst2VPInst.find(Entry.first)->second; + VPRecipeBase *Target = Inst2VPInst.find(Entry.second)->second; + Sink->moveAfter(Target); + } + auto MaxVFPlusOne = MaxVF.getWithIncrement(1); for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) { VFRange SubRange = {VF, MaxVFPlusOne}; - VPlans.push_back(convertToVPRecipes(OriginalPlan, SubRange, SinkAfter)); + VPlans.push_back(convertToVPRecipes(OriginalPlan, SubRange)); VF = SubRange.End; } } -VPlanPtr LoopVectorizationPlanner::convertToVPRecipes( - VPlan &OriginalPlan, VFRange &Range, - const DenseMap &SinkAfter) { +VPlanPtr LoopVectorizationPlanner::convertToVPRecipes(VPlan &OriginalPlan, + VFRange &Range) { SmallPtrSet *, 1> InterleaveGroups; @@ -8871,12 +8886,6 @@ // process after constructing the initial VPlan. // --------------------------------------------------------------------------- - // Mark instructions we'll need to sink later and their targets as - // ingredients whose recipe we'll need to record. - for (auto &Entry : SinkAfter) { - RecipeBuilder.recordRecipeOf(Entry.first); - RecipeBuilder.recordRecipeOf(Entry.second); - } for (auto &Reduction : CM.getInLoopReductionChains()) { PHINode *Phi = Reduction.first; RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind(); @@ -8995,24 +9004,6 @@ // bring the VPlan to its final state. // --------------------------------------------------------------------------- - // Apply Sink-After legal constraints. - for (auto &Entry : SinkAfter) { - VPRecipeBase *Sink = RecipeBuilder.getRecipe(Entry.first); - VPRecipeBase *Target = RecipeBuilder.getRecipe(Entry.second); - // If the target is in a replication region, make sure to move Sink to the - // block after it, not into the replication region itself. - if (auto *Region = - dyn_cast_or_null(Target->getParent()->getParent())) { - if (Region->isReplicator()) { - assert(Region->getNumSuccessors() == 1 && "Expected SESE region!"); - VPBasicBlock *NextBlock = - cast(Region->getSuccessors().front()); - Sink->moveBefore(*NextBlock, NextBlock->getFirstNonPhi()); - continue; - } - } - Sink->moveAfter(Target); - } // Interleave memory: for each Interleave Group we marked earlier as relevant // for this VPlan, replace the Recipes widening its memory instructions with a