diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8708,8 +8708,12 @@ auto MaxVFTimes2 = MaxVF * 2; for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) { VFRange SubRange = {VF, MaxVFTimes2}; - if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, DeadInstructions)) + if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, DeadInstructions)) { + // Now optimize the initial VPlan. + VPlanTransforms::optimize(*Plan, *PSE.getSE()); + assert(VPlanVerifier::verifyPlanIsValid(*Plan) && "VPlan is invalid"); VPlans.push_back(std::move(Plan)); + } VF = SubRange.End; } } @@ -9073,18 +9077,6 @@ if (!VPlanTransforms::adjustFixedOrderRecurrences(*Plan, Builder)) return nullptr; - VPlanTransforms::removeRedundantCanonicalIVs(*Plan); - VPlanTransforms::removeRedundantInductionCasts(*Plan); - - VPlanTransforms::optimizeInductions(*Plan, *PSE.getSE()); - VPlanTransforms::removeDeadRecipes(*Plan); - - VPlanTransforms::createAndOptimizeReplicateRegions(*Plan); - - VPlanTransforms::removeRedundantExpandSCEVRecipes(*Plan); - VPlanTransforms::mergeBlocksIntoPredecessors(*Plan); - - assert(VPlanVerifier::verifyPlanIsValid(*Plan) && "VPlan is invalid"); return Plan; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -37,12 +37,37 @@ GetIntOrFpInductionDescriptor, ScalarEvolution &SE, const TargetLibraryInfo &TLI); + /// Sink users of fixed-order recurrences after the recipe defining their + /// previous value. Then introduce FirstOrderRecurrenceSplice VPInstructions + /// to combine the value from the recurrence phis and previous values. The + /// current implementation assumes all users can be sunk after the previous + /// value, which is enforced by earlier legality checks. + /// \returns true if all users of fixed-order recurrences could be re-arranged + /// as needed or false if it is not possible. In the latter case, \p Plan is + /// not valid. + static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder); + + /// Clear NSW/NUW flags from reduction instructions if necessary. + static void clearReductionWrapFlags(VPlan &Plan); + + /// Optimize \p Plan based on \p BestVF and \p BestUF. This may restrict the + /// resulting plan to \p BestVF and \p BestUF. + static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, + unsigned BestUF, + PredicatedScalarEvolution &PSE); + + /// Apply VPlan-to-VPlan optimizations to \p Plan, including induction recipe + /// optimizations, dead recipe removal, replicate region optimizations and + /// block merging. + static void optimize(VPlan &Plan, ScalarEvolution &SE); + /// Wrap predicated VPReplicateRecipes with a mask operand in an if-then /// region block and remove the mask operand. Optimize the created regions by /// iteratively sinking scalar operands into the region, followed by merging /// regions until no improvements are remaining. static void createAndOptimizeReplicateRegions(VPlan &Plan); +private: /// Remove redundant VPBasicBlocks by merging them into their predecessor if /// the predecessor has a single successor. static bool mergeBlocksIntoPredecessors(VPlan &Plan); @@ -71,24 +96,6 @@ /// them with already existing recipes expanding the same SCEV expression. static void removeRedundantExpandSCEVRecipes(VPlan &Plan); - /// Sink users of fixed-order recurrences after the recipe defining their - /// previous value. Then introduce FirstOrderRecurrenceSplice VPInstructions - /// to combine the value from the recurrence phis and previous values. The - /// current implementation assumes all users can be sunk after the previous - /// value, which is enforced by earlier legality checks. - /// \returns true if all users of fixed-order recurrences could be re-arranged - /// as needed or false if it is not possible. In the latter case, \p Plan is - /// not valid. - static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder); - - /// Clear NSW/NUW flags from reduction instructions if necessary. - static void clearReductionWrapFlags(VPlan &Plan); - - /// Optimize \p Plan based on \p BestVF and \p BestUF. This may restrict the - /// resulting plan to \p BestVF and \p BestUF. - static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, - unsigned BestUF, - PredicatedScalarEvolution &PSE); }; } // namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -778,3 +778,16 @@ } } } + +void VPlanTransforms::optimize(VPlan &Plan, ScalarEvolution &SE) { + removeRedundantCanonicalIVs(Plan); + removeRedundantInductionCasts(Plan); + + optimizeInductions(Plan, SE); + removeDeadRecipes(Plan); + + createAndOptimizeReplicateRegions(Plan); + + removeRedundantExpandSCEVRecipes(Plan); + mergeBlocksIntoPredecessors(Plan); +}