diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -291,22 +291,12 @@ /// exclusive, possibly decreasing \p Range.End. VPlanPtr buildVPlan(const VFRange &Range); - /// Build a VPlan using VPRecipes according to the information gather by - /// Legal. This method is only used for the legacy inner loop vectorizer. - VPlanPtr buildVPlanWithVPRecipes( - VFRange &Range, SmallPtrSetImpl &NeedDef, - SmallPtrSetImpl &DeadInstructions, - const DenseMap &SinkAfter); - - VPlanPtr - convertToVPRecipes(VPlan &OriginalPlan, VFRange &Range, - SmallPtrSetImpl &NeedDef, - const DenseMap &SinkAfter); + VPlanPtr convertToVPRecipes(VPlan &OriginalPlan, VFRange &Range, + SmallPtrSetImpl &NeedDef); /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive, /// according to the information gathered by Legal when it checked if it is /// legal to vectorize the loop. This method creates VPlans using VPRecipes. - void buildVPlansWithVPRecipes(unsigned MinVF, unsigned MaxVF); void buildVPlansWithVPRecipes(unsigned MinVF, unsigned MaxVF, VPlan &OriginalPlan); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7692,6 +7692,12 @@ auto *TopRegion = cast(OriginalPlan.getEntry()); ReversePostOrderTraversal RPOT(TopRegion->getEntry()); + DenseMap Inst2VPInst; + for (auto &Entry : SinkAfter) { + Inst2VPInst[Entry.first] = nullptr; + Inst2VPInst[Entry.second] = nullptr; + } + auto *DummyVal = new VPValue(); OriginalPlan.addExternalDef(DummyVal); for (VPBlockBase *Base : RPOT) { @@ -7710,21 +7716,30 @@ if (DeadInstructions.contains(Instr) || isa(Instr)) { Ingredient->getVPValue()->replaceAllUsesWith(DummyVal); Ingredient->eraseFromParent(); + continue; } + + auto Iter = Inst2VPInst.find(Instr); + if (Iter != Inst2VPInst.end()) + Iter->second = VPInst; } } + for (auto &Entry : SinkAfter) { + VPRecipeBase *Sink = Inst2VPInst.find(Entry.first)->second; + VPRecipeBase *Target = Inst2VPInst.find(Entry.second)->second; + Sink->moveAfter(Target); + } + for (unsigned VF = MinVF; VF < MaxVF + 1;) { VFRange SubRange = {VF, MaxVF + 1}; - VPlans.push_back( - convertToVPRecipes(OriginalPlan, SubRange, NeedDef, SinkAfter)); + VPlans.push_back(convertToVPRecipes(OriginalPlan, SubRange, NeedDef)); VF = SubRange.End; } } VPlanPtr LoopVectorizationPlanner::convertToVPRecipes( - VPlan &OriginalPlan, VFRange &Range, SmallPtrSetImpl &NeedDef, - const DenseMap &SinkAfter) { + VPlan &OriginalPlan, VFRange &Range, SmallPtrSetImpl &NeedDef) { // Hold a mapping from predicated instructions to their recipes, in order to // fix their AlsoPack behavior if a user is determined to replicate and use a @@ -7741,12 +7756,6 @@ // process after constructing the initial VPlan. // --------------------------------------------------------------------------- - // Mark instructions we'll need to sink later and their targets as - // ingredients whose recipe we'll need to record. - for (auto &Entry : SinkAfter) { - RecipeBuilder.recordRecipeOf(Entry.first); - RecipeBuilder.recordRecipeOf(Entry.second); - } for (auto &Reduction : CM.getInLoopReductionChains()) { PHINode *Phi = Reduction.first; RecurrenceDescriptor::RecurrenceKind Kind = @@ -7787,21 +7796,6 @@ } }; - auto skipDeadInterleaveMembers = - [&DeadInterleaveGroupMembers](Instruction *I) { - BasicBlock *BB = I->getParent(); - for (auto &I : make_range(I->getIterator(), BB->end())) - if (!DeadInterleaveGroupMembers.contains(&I)) - return &I; - llvm_unreachable("Need to find a valid insert point"); - }; - // Mark instructions we'll need to sink later and their targets as - // ingredients whose recipe we'll need to record. - for (auto &Entry : SinkAfter) { - RecipeBuilder.recordRecipeOf(skipDeadInterleaveMembers(Entry.first)); - RecipeBuilder.recordRecipeOf(skipDeadInterleaveMembers(Entry.second)); - } - // --------------------------------------------------------------------------- // Build initial VPlan: Scan the body of the loop in a topological order to // visit each basic block after having visited its predecessor basic blocks. @@ -7900,15 +7894,6 @@ // bring the VPlan to its final state. // --------------------------------------------------------------------------- - // Apply Sink-After legal constraints. - for (auto &Entry : SinkAfter) { - VPRecipeBase *Sink = - RecipeBuilder.getRecipe(skipDeadInterleaveMembers(Entry.first)); - VPRecipeBase *Target = - RecipeBuilder.getRecipe(skipDeadInterleaveMembers(Entry.second)); - Sink->moveAfter(Target); - } - // Adjust the recipes for any inloop reductions. if (Range.Start > 1) adjustRecipesForInLoopReductions(Plan, RecipeBuilder);