diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8412,15 +8412,8 @@ assert(CM.foldTailByMasking() && "must fold the tail"); VPBasicBlock *HeaderVPBB = Plan->getEntry()->getEntryBasicBlock(); auto NewInsertionPoint = HeaderVPBB->getFirstNonPhi(); - - VPValue *IV = nullptr; - if (Legal->getPrimaryInduction()) - IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction()); - else { - auto *IVRecipe = new VPWidenCanonicalIVRecipe(); - HeaderVPBB->insert(IVRecipe, NewInsertionPoint); - IV = IVRecipe; - } + auto *IV = new VPWidenCanonicalIVRecipe(); + HeaderVPBB->insert(IV, HeaderVPBB->getFirstNonPhi()); VPBuilder::InsertPointGuard Guard(Builder); Builder.setInsertPoint(HeaderVPBB, NewInsertionPoint); @@ -9195,6 +9188,9 @@ } } + // Check if \p I has only scalar uses. + VPlanTransforms::removeRedundantVPWidenCanonicalIVRecipe( + *Plan, Legal->getPrimaryInduction()); VPlanTransforms::removeRedundantInductionCasts(*Plan); // Now that sink-after is done, move induction recipes for optimized truncates diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -45,6 +45,12 @@ /// in the vectorized loop. There is no need to vectorize the cast - the same /// value can be used for both the phi and casts in the vector loop. static void removeRedundantInductionCasts(VPlan &Plan); + + /// Try to replace VPWidenCanonicalIVRecipes with the primary IV recipe, if it + /// exists. + static void + removeRedundantVPWidenCanonicalIVRecipe(VPlan &Plan, + Value *OriginalCanonicalIV); }; } // namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -324,3 +324,32 @@ E.first->eraseFromParent(); } } + +void VPlanTransforms::removeRedundantVPWidenCanonicalIVRecipe( + VPlan &Plan, Value *OriginalCanonicalIV) { + if (!OriginalCanonicalIV) + return; + + VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock(); + VPWidenCanonicalIVRecipe *WidenNewIV = nullptr; + VPWidenIntOrFpInductionRecipe *WidenOriginalIV = nullptr; + for (VPRecipeBase &Phi : *HeaderVPBB) { + if (auto *IV = dyn_cast(&Phi)) { + // If the induction recipe is for the primary induction use it directly. + if (IV->getUnderlyingValue() == OriginalCanonicalIV) + WidenOriginalIV = IV; + continue; + } + + if (auto *W = dyn_cast(&Phi)) + WidenNewIV = W; + } + + if (!WidenNewIV) + return; + + if (WidenOriginalIV) { + WidenNewIV->replaceAllUsesWith(WidenOriginalIV); + WidenNewIV->eraseFromParent(); + } +}