Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7169,11 +7169,12 @@ bool LoopVectorizationPlanner::getDecisionAndClampRange( const std::function &Predicate, VFRange &Range) { - assert(Range.End > Range.Start && "Trying to test an empty VF range."); - bool PredicateAtRangeStart = Predicate(ElementCount::getFixed(Range.Start)); + assert(!Range.isEmpty() && "Trying to test an empty VF range."); + bool PredicateAtRangeStart = Predicate(Range.Start); - for (unsigned TmpVF = Range.Start * 2; TmpVF < Range.End; TmpVF *= 2) - if (Predicate(ElementCount::getFixed(TmpVF)) != PredicateAtRangeStart) { + for (ElementCount TmpVF = Range.Start * 2; + ElementCount::isKnownLT(TmpVF, Range.End); TmpVF *= 2) + if (Predicate(TmpVF) != PredicateAtRangeStart) { Range.End = TmpVF; break; } @@ -7187,8 +7188,10 @@ /// vectorization decision can potentially shorten this sub-range during /// buildVPlan(). void LoopVectorizationPlanner::buildVPlans(unsigned MinVF, unsigned MaxVF) { - for (unsigned VF = MinVF; VF < MaxVF + 1;) { - VFRange SubRange = {VF, MaxVF + 1}; + auto MaxVFPlusOne = ElementCount::getFixed(MaxVF).getWithIncrement(1); + for (ElementCount VF = ElementCount::getFixed(MinVF); + ElementCount::isKnownLT(VF, MaxVFPlusOne);) { + VFRange SubRange = {VF, MaxVFPlusOne}; VPlans.push_back(buildVPlan(SubRange)); VF = SubRange.End; } @@ -7645,8 +7648,10 @@ for (Instruction *I : DeadInstructions) SinkAfter.erase(I); - for (unsigned VF = MinVF; VF < MaxVF + 1;) { - VFRange SubRange = {VF, MaxVF + 1}; + auto MaxVFPlusOne = ElementCount::getFixed(MaxVF).getWithIncrement(1); + for (ElementCount VF = ElementCount::getFixed(MinVF); + ElementCount::isKnownLT(VF, MaxVFPlusOne);) { + VFRange SubRange = {VF, MaxVFPlusOne}; VPlans.push_back(buildVPlanWithVPRecipes(SubRange, NeedDef, DeadInstructions, SinkAfter)); VF = SubRange.End; @@ -7808,7 +7813,7 @@ } // Adjust the recipes for any inloop reductions. - if (Range.Start > 1) + if (Range.Start.isVector()) adjustRecipesForInLoopReductions(Plan, RecipeBuilder); // Finally, if tail is folded by masking, introduce selects between the phi @@ -7827,10 +7832,10 @@ std::string PlanName; raw_string_ostream RSO(PlanName); - ElementCount VF = ElementCount::getFixed(Range.Start); + ElementCount VF = Range.Start; Plan->addVF(VF); RSO << "Initial VPlan for VF={" << VF; - for (VF *= 2; VF.getKnownMinValue() < Range.End; VF *= 2) { + for (VF *= 2; ElementCount::isKnownLT(VF, Range.End); VF *= 2) { Plan->addVF(VF); RSO << "," << VF; } @@ -7856,8 +7861,9 @@ VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan); HCFGBuilder.buildHierarchicalCFG(); - for (unsigned VF = Range.Start; VF < Range.End; VF *= 2) - Plan->addVF(ElementCount::getFixed(VF)); + for (ElementCount VF = Range.Start; ElementCount::isKnownLT(VF, Range.End); + VF *= 2) + Plan->addVF(VF); if (EnableVPlanPredication) { VPlanPredicator VPP(*Plan); Index: llvm/lib/Transforms/Vectorize/VPlan.h =================================================================== --- llvm/lib/Transforms/Vectorize/VPlan.h +++ llvm/lib/Transforms/Vectorize/VPlan.h @@ -65,10 +65,22 @@ /// [1, 9) = {1, 2, 4, 8} struct VFRange { // A power of 2. - const unsigned Start; + const ElementCount Start; // Need not be a power of 2. If End <= Start range is empty. - unsigned End; + ElementCount End; + + bool isEmpty() const { + return End.getKnownMinValue() <= Start.getKnownMinValue(); + } + + VFRange(const ElementCount &Start, const ElementCount &End) + : Start(Start), End(End) { + assert(Start.isScalable() == End.isScalable() && + "Both Start and End should have the same scalable flag"); + assert(isPowerOf2_32(Start.getKnownMinValue()) && + "Expected Start to be a power of 2"); + } }; using VPlanPtr = std::unique_ptr;