diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8431,15 +8431,8 @@ assert(CM.foldTailByMasking() && "must fold the tail"); VPBasicBlock *HeaderVPBB = Plan->getEntry()->getEntryBasicBlock(); auto NewInsertionPoint = HeaderVPBB->getFirstNonPhi(); - - VPValue *IV = nullptr; - if (Legal->getPrimaryInduction()) - IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction()); - else { - auto *IVRecipe = new VPWidenCanonicalIVRecipe(Plan->getCanonicalIV()); - HeaderVPBB->insert(IVRecipe, NewInsertionPoint); - IV = IVRecipe; - } + auto *IV = new VPWidenCanonicalIVRecipe(Plan->getCanonicalIV()); + HeaderVPBB->insert(IV, HeaderVPBB->getFirstNonPhi()); VPBuilder::InsertPointGuard Guard(Builder); Builder.setInsertPoint(HeaderVPBB, NewInsertionPoint); @@ -9214,6 +9207,7 @@ } } + VPlanTransforms::removeRedundantVPWidenCanonicalIVRecipe(*Plan); VPlanTransforms::removeRedundantInductionCasts(*Plan); // Now that sink-after is done, move induction recipes for optimized truncates diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1049,6 +1049,7 @@ /// Returns the start value of the induction. VPValue *getStartValue() { return getOperand(0); } + const VPValue *getStartValue() const { return getOperand(0); } /// Returns the first defined value as TruncInst, if it is one or nullptr /// otherwise. @@ -1061,6 +1062,13 @@ /// Returns the induction descriptor for the recipe. const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } + + /// Returns true if the induction is canonical, i.e. starting at 0 and + /// incremented by UF * VF (= the original IV is incremented by 1). + bool isCanonical() const; + + /// Returns the type of the original IV. + const Type *getOriginalType() const { return IV->getType(); } }; /// A pure virtual base class for all recipes modeling header phis, including @@ -1660,6 +1668,11 @@ void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; #endif + + /// Returns the type of the original IV. + const Type *getOriginalType() const { + return getOperand(0)->getLiveInIRValue()->getType(); + } }; /// A Recipe for widening the canonical induction variable of the vector loop. @@ -1676,6 +1689,16 @@ return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC; } + /// Extra classof implementations to allow directly casting from VPUser -> + /// VPWidenCanonicalIVRecipe. + static inline bool classof(const VPUser *U) { + auto *R = dyn_cast(U); + return R && R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC; + } + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC; + } + /// Generate a canonical vector induction variable of the vector loop, with /// start = { for 0 <= Part < UF}, and /// step = . diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1253,6 +1253,12 @@ O << " " << VPlanIngredient(IV); } +bool VPWidenIntOrFpInductionRecipe::isCanonical() const { + auto *StartC = dyn_cast(getStartValue()->getLiveInIRValue()); + auto *StepC = dyn_cast(getInductionDescriptor().getStep()); + return StartC && StartC->isZero() && StepC && StepC->isOne(); +} + void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN-GEP "; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -45,6 +45,10 @@ /// in the vectorized loop. There is no need to vectorize the cast - the same /// value can be used for both the phi and casts in the vector loop. static void removeRedundantInductionCasts(VPlan &Plan); + + /// Try to replace VPWidenCanonicalIVRecipes with the primary IV recipe, if it + /// exists. + static void removeRedundantVPWidenCanonicalIVRecipe(VPlan &Plan); }; } // namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -324,3 +324,35 @@ E.first->eraseFromParent(); } } + +void VPlanTransforms::removeRedundantVPWidenCanonicalIVRecipe(VPlan &Plan) { + VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV(); + VPWidenCanonicalIVRecipe *WidenNewIV = nullptr; + for (VPUser *U : CanonicalIV->users()) { + WidenNewIV = dyn_cast(U); + if (WidenNewIV) + break; + } + + if (!WidenNewIV) + return; + + VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock(); + VPWidenIntOrFpInductionRecipe *WidenOriginalIV = nullptr; + for (VPRecipeBase &Phi : *HeaderVPBB) { + WidenOriginalIV = dyn_cast(&Phi); + if (!WidenOriginalIV) + continue; + + // If the induction recipe is for the primary induction use it directly. + if (WidenOriginalIV->isCanonical() && + WidenOriginalIV->getOriginalType() == CanonicalIV->getOriginalType()) + break; + WidenOriginalIV = nullptr; + } + + if (WidenOriginalIV) { + WidenNewIV->replaceAllUsesWith(WidenOriginalIV); + WidenNewIV->eraseFromParent(); + } +} diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -178,6 +178,7 @@ void replaceAllUsesWith(VPValue *New); VPDef *getDef() { return Def; } + const VPDef *getDef() const { return Def; } /// Returns the underlying IR value, if this VPValue is defined outside the /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef @@ -187,6 +188,11 @@ "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); return getUnderlyingValue(); } + const Value *getLiveInIRValue() const { + assert(!getDef() && + "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); + return getUnderlyingValue(); + } }; typedef DenseMap Value2VPValueTy;