diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2333,15 +2333,13 @@ /// variable on which to base the steps, \p Step is the size of the step. static void buildScalarSteps(Value *ScalarIV, Value *Step, const InductionDescriptor &ID, VPValue *Def, - Type *TruncToTy, VPTransformState &State) { + VPTransformState &State) { IRBuilderBase &Builder = State.Builder; Type *ScalarIVTy = ScalarIV->getType()->getScalarType(); - if (TruncToTy) { + if (ScalarIVTy != Step->getType()) { assert(Step->getType()->isIntegerTy() && "Truncation requires an integer step"); - ScalarIV = State.Builder.CreateTrunc(ScalarIV, TruncToTy); - Step = State.Builder.CreateTrunc(Step, TruncToTy); - ScalarIVTy = ScalarIV->getType()->getScalarType(); + Step = State.Builder.CreateTrunc(Step, ScalarIVTy); } // We shouldn't have to build scalar steps if we aren't vectorizing. @@ -9533,6 +9531,32 @@ } } +void VPDerivedIVRecipe::execute(VPTransformState &State) { + assert(!State.Instance && "VPDerivedIVRecipe being replicated."); + + // Fast-math-flags propagate from the original induction instruction. + IRBuilder<>::FastMathFlagGuard FMFG(State.Builder); + if (IndDesc.getInductionBinOp() && + isa(IndDesc.getInductionBinOp())) + State.Builder.setFastMathFlags( + IndDesc.getInductionBinOp()->getFastMathFlags()); + + Value *Step = State.get(getStepValue(), VPIteration(0, 0)); + Value *CanonicalIV = State.get(getCanonicalIV(), VPIteration(0, 0)); + Value *DerivedIV = + emitTransformedIndex(State.Builder, CanonicalIV, + getStartValue()->getLiveInIRValue(), Step, IndDesc); + DerivedIV->setName("offset.idx"); + if (TruncToTy && TruncToTy != DerivedIV->getType()) { + assert(Step->getType()->isIntegerTy() && + "Truncation requires an integer step"); + DerivedIV = State.Builder.CreateTrunc(DerivedIV, TruncToTy); + } + assert(DerivedIV != CanonicalIV && "IV didn't need transforming?"); + + State.set(this, DerivedIV, VPIteration(0, 0)); +} + void VPScalarIVStepsRecipe::execute(VPTransformState &State) { assert(!State.Instance && "VPScalarIVStepsRecipe being replicated."); @@ -9543,21 +9567,10 @@ State.Builder.setFastMathFlags( IndDesc.getInductionBinOp()->getFastMathFlags()); + Value *BaseIV = State.get(getOperand(0), VPIteration(0, 0)); Value *Step = State.get(getStepValue(), VPIteration(0, 0)); - auto CreateScalarIV = [&](Value *&Step) -> Value * { - Value *ScalarIV = State.get(getCanonicalIV(), VPIteration(0, 0)); - auto *CanonicalIV = State.get(getParent()->getPlan()->getCanonicalIV(), 0); - if (!isCanonical() || CanonicalIV->getType() != Step->getType()) { - ScalarIV = emitTransformedIndex(State.Builder, ScalarIV, - getStartValue()->getLiveInIRValue(), Step, - IndDesc); - ScalarIV->setName("offset.idx"); - } - return ScalarIV; - }; - Value *ScalarIV = CreateScalarIV(Step); - buildScalarSteps(ScalarIV, Step, IndDesc, this, TruncToTy, State); + buildScalarSteps(BaseIV, Step, IndDesc, this, State); } void VPInterleaveRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1179,9 +1179,7 @@ } /// Returns the incoming value from the loop backedge. - VPValue *getBackedgeValue() { - return getOperand(1); - } + VPValue *getBackedgeValue() { return getOperand(1); } /// Returns the backedge value as a recipe. The backedge value is guaranteed /// to be a recipe. @@ -1879,6 +1877,10 @@ "Op must be an operand of the recipe"); return true; } + + /// Check if the induction described by \p ID is canonical, i.e. has a step of + /// 1 and the same type and start as the canonical IV. + bool isCanonical(const InductionDescriptor &ID, Type *Ty) const; }; /// A recipe for generating the active lane mask for the vector loop that is @@ -1959,19 +1961,74 @@ } }; +/// A recipe for converting the canonical IV value to the corresponding value of +/// an IV with different start and step values, using Start + CanonicalIV * +/// Step. +class VPDerivedIVRecipe : public VPRecipeBase, public VPValue { + /// If not nullptr, truncate the generated values to TruncToTy. + Type *TruncToTy; + + /// Induction descriptor for the induction the canonical IV is transformed to. + const InductionDescriptor &IndDesc; + +public: + VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start, + VPValue *CanonicalIV, VPValue *Step, Type *TruncToTy) + : VPRecipeBase(VPDerivedIVSC, {Start, CanonicalIV, Step}), + VPValue(VPVDerivedIVSC, nullptr, this), TruncToTy(TruncToTy), + IndDesc(IndDesc) {} + + ~VPDerivedIVRecipe() override = default; + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPDef *D) { + return D->getVPDefID() == VPRecipeBase::VPDerivedIVSC; + } + /// Extra classof implementations to allow directly casting from VPUser -> + /// VPDerivedIVRecipe. + static inline bool classof(const VPUser *U) { + auto *R = dyn_cast(U); + return R && R->getVPDefID() == VPRecipeBase::VPDerivedIVSC; + } + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPRecipeBase::VPDerivedIVSC; + } + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPValue::VPVDerivedIVSC; + } + + /// Generate the transformed value of the induction at offset StartValue (1. + /// operand) + IV (2. operand) * StepValue (3, operand). + void execute(VPTransformState &State) override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif + + VPValue *getStartValue() const { return getOperand(0); } + VPValue *getCanonicalIV() const { return getOperand(1); } + VPValue *getStepValue() const { return getOperand(2); } + + /// Returns true if the recipe only uses the first lane of operand \p Op. + bool onlyFirstLaneUsed(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + return true; + } +}; + /// A recipe for handling phi nodes of integer and floating-point inductions, /// producing their scalar values. class VPScalarIVStepsRecipe : public VPRecipeBase, public VPValue { - /// If not nullptr, truncate the generated values to TruncToTy. - Type *TruncToTy; const InductionDescriptor &IndDesc; public: - VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, - VPValue *CanonicalIV, VPValue *Start, VPValue *Step, - Type *TruncToTy) - : VPRecipeBase(VPScalarIVStepsSC, {CanonicalIV, Start, Step}), - VPValue(nullptr, this), TruncToTy(TruncToTy), IndDesc(IndDesc) {} + VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, + VPValue *Step) + : VPRecipeBase(VPScalarIVStepsSC, {IV, Step}), VPValue(nullptr, this), + IndDesc(IndDesc) {} ~VPScalarIVStepsRecipe() override = default; @@ -1998,13 +2055,7 @@ VPSlotTracker &SlotTracker) const override; #endif - /// Returns true if the induction is canonical, i.e. starting at 0 and - /// incremented by UF * VF (= the original IV is incremented by 1). - bool isCanonical() const; - - VPCanonicalIVPHIRecipe *getCanonicalIV() const; - VPValue *getStartValue() const { return getOperand(1); } - VPValue *getStepValue() const { return getOperand(2); } + VPValue *getStepValue() const { return getOperand(1); } /// Returns true if the recipe only uses the first lane of operand \p Op. bool onlyFirstLaneUsed(const VPValue *Op) const override { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -648,12 +648,14 @@ // When vectorizing the epilogue loop, the canonical induction start value // needs to be changed from zero to the value after the main vector loop. + // FIXME: Improve modeling for canonical IV start values in the epilogue loop. if (CanonicalIVStartValue) { VPValue *VPV = getOrAddExternalDef(CanonicalIVStartValue); auto *IV = getCanonicalIV(); assert(all_of(IV->users(), [](const VPUser *U) { - if (isa(U)) + if (isa(U) || + isa(U)) return true; auto *VPI = cast(U); return VPI->getOpcode() == diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -103,6 +103,7 @@ bool VPRecipeBase::mayHaveSideEffects() const { switch (getVPDefID()) { + case VPDerivedIVSC: case VPPredInstPHISC: return false; case VPWidenIntOrFpInductionSC: @@ -712,22 +713,22 @@ return StartC && StartC->isZero() && StepC && StepC->isOne(); } -VPCanonicalIVPHIRecipe *VPScalarIVStepsRecipe::getCanonicalIV() const { - return cast(getOperand(0)); -} +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPDerivedIVRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent; + printAsOperand(O, SlotTracker); + O << Indent << "= DERIVED-IV "; + getStartValue()->printAsOperand(O, SlotTracker); + O << " + "; + getCanonicalIV()->printAsOperand(O, SlotTracker); + O << " * "; + getStepValue()->printAsOperand(O, SlotTracker); -bool VPScalarIVStepsRecipe::isCanonical() const { - auto *CanIV = getCanonicalIV(); - // The start value of the steps-recipe must match the start value of the - // canonical induction and it must step by 1. - if (CanIV->getStartValue() != getStartValue()) - return false; - auto *StepVPV = getStepValue(); - if (StepVPV->hasDefiningRecipe()) - return false; - auto *StepC = dyn_cast_or_null(StepVPV->getLiveInIRValue()); - return StepC && StepC->isOne(); + if (IndDesc.getStep()->getType() != TruncToTy) + O << " (truncated to " << *TruncToTy << ")"; } +#endif #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPScalarIVStepsRecipe::print(raw_ostream &O, const Twine &Indent, @@ -1050,6 +1051,19 @@ } #endif +bool VPCanonicalIVPHIRecipe::isCanonical(const InductionDescriptor &ID, + Type *Ty) const { + if (Ty != getScalarType() || ID.getInductionOpcode() != Instruction::Add) + return false; + // The start value of the steps-recipe must match the start value of the + // canonical induction and it must step by 1. + if (getStartValue()->getLiveInIRValue() != ID.getStartValue()) + return false; + + ConstantInt *Step = ID.getConstIntStepValue(); + return Step && Step->isOne(); +} + bool VPWidenPointerInductionRecipe::onlyScalarsGenerated(ElementCount VF) { return IsScalarAfterVectorization && (!VF.isScalable() || vputils::onlyFirstLaneUsed(this)); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -382,30 +382,40 @@ VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock(); bool HasOnlyVectorVFs = !Plan.hasVF(ElementCount::getFixed(1)); for (VPRecipeBase &Phi : HeaderVPBB->phis()) { - auto *IV = dyn_cast(&Phi); - if (!IV) + auto *WideIV = dyn_cast(&Phi); + if (!WideIV) continue; - if (HasOnlyVectorVFs && - none_of(IV->users(), [IV](VPUser *U) { return U->usesScalars(IV); })) + if (HasOnlyVectorVFs && none_of(WideIV->users(), [WideIV](VPUser *U) { + return U->usesScalars(WideIV); + })) continue; - const InductionDescriptor &ID = IV->getInductionDescriptor(); + auto IP = HeaderVPBB->getFirstNonPhi(); + VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV(); + Type *ResultTy = WideIV->getPHINode()->getType(); + if (Instruction *TruncI = WideIV->getTruncInst()) + ResultTy = TruncI->getType(); + const InductionDescriptor &ID = WideIV->getInductionDescriptor(); VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, ID.getStep(), SE); - Instruction *TruncI = IV->getTruncInst(); - VPScalarIVStepsRecipe *Steps = new VPScalarIVStepsRecipe( - ID, Plan.getCanonicalIV(), IV->getStartValue(), Step, - TruncI ? TruncI->getType() : nullptr); - HeaderVPBB->insert(Steps, HeaderVPBB->getFirstNonPhi()); + VPValue *BaseIV = CanonicalIV; + if (!CanonicalIV->isCanonical(ID, ResultTy)) { + BaseIV = new VPDerivedIVRecipe(ID, WideIV->getStartValue(), CanonicalIV, + Step, ResultTy); + HeaderVPBB->insert(BaseIV->getDefiningRecipe(), IP); + } + + VPScalarIVStepsRecipe *Steps = new VPScalarIVStepsRecipe(ID, BaseIV, Step); + HeaderVPBB->insert(Steps, IP); // Update scalar users of IV to use Step instead. Use SetVector to ensure // the list of users doesn't contain duplicates. - SetVector Users(IV->user_begin(), IV->user_end()); + SetVector Users(WideIV->user_begin(), WideIV->user_end()); for (VPUser *U : Users) { - if (HasOnlyVectorVFs && !U->usesScalars(IV)) + if (HasOnlyVectorVFs && !U->usesScalars(WideIV)) continue; for (unsigned I = 0, E = U->getNumOperands(); I != E; I++) { - if (U->getOperand(I) != IV) + if (U->getOperand(I) != WideIV) continue; U->setOperand(I, Steps); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -90,6 +90,7 @@ /// type identification. enum { VPValueSC, + VPVDerivedIVSC, VPVInstructionSC, VPVMemoryInstructionSC, VPVReductionSC, @@ -355,6 +356,7 @@ /// type identification. using VPRecipeTy = enum { VPBranchOnMaskSC, + VPDerivedIVSC, VPExpandSCEVSC, VPInstructionSC, VPInterleaveSC, diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll @@ -19,7 +19,7 @@ ; VPLANS-NEXT: vector.body: ; VPLANS-NEXT: EMIT vp<%4> = CANONICAL-INDUCTION ; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<%5> = phi vp<%3>, vp<%10> -; VPLANS-NEXT: vp<%6> = SCALAR-STEPS vp<%4>, ir<0>, ir<1> +; VPLANS-NEXT: vp<%6> = SCALAR-STEPS vp<%4>, ir<1> ; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<%6> ; VPLANS-NEXT: WIDEN store ir<%gep>, ir<%val>, vp<%5> ; VPLANS-NEXT: EMIT vp<%8> = VF * UF + vp<%4> diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -15,7 +15,7 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<0>, ir<1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> ; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<%3> ; CHECK-NEXT: WIDEN ir<%l> = load ir<%gep.src> ; CHECK-NEXT: WIDEN ir<%conv> = fpext ir<%l> @@ -41,7 +41,7 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<0>, ir<1> +; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> ; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, vp<%3> ; CHECK-NEXT: WIDEN ir<%l> = load ir<%gep.src> ; CHECK-NEXT: WIDEN ir<%conv> = fpext ir<%l> diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -56,17 +56,18 @@ ; CHECK-NEXT: Successor(s): vector loop ; CHECK: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<%4> = SCALAR-STEPS vp<%3>, ir<%n>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add vp<%4>, ir<-1> +; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<[[TRANS_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> +; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[TRANS_IV]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%i.0> = add vp<[[SCALAR_STEPS]]>, ir<-1> ; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom> ; CHECK-NEXT: WIDEN ir<%1> = load ir<%arrayidx> ; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom> ; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%add9> -; CHECK-NEXT: EMIT vp<%11> = VF * UF +(nuw) vp<%3> -; CHECK-NEXT: EMIT branch-on-count vp<%11> vp<%2> +; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]> vp<%2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block @@ -187,17 +188,18 @@ ; CHECK-NEXT: Successor(s): vector loop ; CHECK: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<%4> = SCALAR-STEPS vp<%3>, ir<%n>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add vp<%4>, ir<-1> +; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION +; CHECK-NEXT: vp<[[TRANS_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> +; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[TRANS_IV]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%i.0> = add vp<[[SCALAR_STEPS]]>, ir<-1> ; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%B>, ir<%idxprom> ; CHECK-NEXT: WIDEN ir<%1> = load ir<%arrayidx> ; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00> ; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr ir<%A>, ir<%idxprom> ; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%conv1> -; CHECK-NEXT: EMIT vp<%11> = VF * UF +(nuw) vp<%3> -; CHECK-NEXT: EMIT branch-on-count vp<%11> vp<%2> +; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]> vp<%2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -15,7 +15,7 @@ ; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<%8> -; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<0>, ir<1> +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%2>, ir<1> ; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr ir<%ptr>, vp<%5> ; CHECK-NEXT: WIDEN ir<%for.1.next> = load ir<%gep.ptr> ; CHECK-NEXT: EMIT vp<%8> = first-order splice ir<%for.1> ir<%for.1.next> @@ -65,7 +65,7 @@ ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<%9> ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<%10> -; CHECK-NEXT: vp<%6> = SCALAR-STEPS vp<%2>, ir<0>, ir<1> +; CHECK-NEXT: vp<%6> = SCALAR-STEPS vp<%2>, ir<1> ; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr ir<%ptr>, vp<%6> ; CHECK-NEXT: WIDEN ir<%for.1.next> = load ir<%gep.ptr> ; CHECK-NEXT: EMIT vp<%9> = first-order splice ir<%for.1> ir<%for.1.next> diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -21,7 +21,7 @@ ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: @@ -117,7 +117,7 @@ ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: @@ -269,7 +269,7 @@ ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): loop.0 @@ -376,7 +376,7 @@ ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: @@ -467,7 +467,8 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]> -; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<2>, ir<1> +; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1> +; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): loop.0 @@ -487,7 +488,7 @@ ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<%5> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -48,7 +48,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13> ; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20> diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -13,7 +13,8 @@ ; DBG-NEXT: vector loop: { ; DBG-NEXT: vector.body: ; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<%start>, ir<1> +; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%start> + vp<[[CAN_IV]]> * ir<1> +; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> ; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) ; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%dst>, vp<[[IV_STEPS]]> ; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> @@ -69,8 +70,9 @@ ; DBG-NEXT: vector loop: { ; DBG-NEXT: vector.body: ; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; DBG-NEXT: vp<[[STEPS1:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir, ir -; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir + vp<[[CAN_IV]]> * ir +; DBG-NEXT: vp<[[STEPS1:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir +; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; DBG-NEXT: Successor(s): cond.false ; DBG-EMPTY: ; DBG-NEXT: cond.false: @@ -181,13 +183,14 @@ ; DBG-EMPTY: ; DBG-NEXT: vector loop: { ; DBG-NEXT: vector.body: -; DBG-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION -; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<%4> -; DBG-NEXT: vp<%4> = SCALAR-STEPS vp<%2>, ir<0>, ir<1> -; DBG-NEXT: EMIT vp<%5> = first-order splice ir<%for> vp<%4> -; DBG-NEXT: CLONE store vp<%5>, ir<%dst> -; DBG-NEXT: EMIT vp<%7> = VF * UF +(nuw) vp<%2> -; DBG-NEXT: EMIT branch-on-count vp<%7> vp<%1> +; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION +; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]> +; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<1> (truncated to i32) +; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> +; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for> vp<[[SCALAR_STEPS]]> +; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst> +; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF +(nuw) vp<[[CAN_IV]]> +; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]> vp<%1> ; DBG-NEXT: No successors ; DBG-NEXT: } ; DBG-NEXT: Successor(s): middle.block diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -23,7 +23,7 @@ ; CHECK-NEXT: N1 [label = ; CHECK-NEXT: "vector.body:\l" + ; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION\l" + -; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<0\>, ir\<1\>\l" + +; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" + ; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, vp\<[[STEPS]]\>\l" + ; CHECK-NEXT: " WIDEN ir\<%lv\> = load ir\<%arrayidx\>\l" + ; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -17,7 +17,7 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) @@ -64,7 +64,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z> @@ -115,7 +115,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) @@ -160,7 +160,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> -; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, vp<[[IV]]> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop) @@ -204,7 +204,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5> ; CHECK-NEXT: Successor(s): if.then ; CHECK-EMPTY: @@ -282,7 +282,8 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<4> +; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4> ; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr ir<@AB>, ir<0>, vp<[[STEPS]]> ; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0> ; CHECK-NEXT: ir<%AB.0> = load from index 0 @@ -349,7 +350,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%l.a> = load ir<%arrayidx> ; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr ir<%b>, vp<[[STEPS]]> @@ -398,7 +399,7 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%isd> = getelementptr ir<%asd>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%lsd> = load ir<%isd> ; CHECK-NEXT: WIDEN ir<%psd> = add ir<%lsd>, ir<23> @@ -496,7 +497,8 @@ ; CHECK-NEXT: WIDEN-INDUCTION\l" + ; CHECK-NEXT: " %iv = phi %iv.next, 0\l" + ; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, vp<[[EXP_SCEV]]> +; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]> ; CHECK-NEXT: WIDEN ir<%v3> = add ir<%v2>, ir<1> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> @@ -542,7 +544,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off> ; CHECK-NEXT: WIDEN store ir<%gep>, ir<0> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -16,7 +16,7 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2> ; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -22,7 +22,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): loop.0 @@ -89,7 +89,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load @@ -168,7 +168,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load @@ -249,7 +249,8 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 21, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<21>, ir<1> +; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]> vp<[[BTC]]> ; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr ir<%A>, ir<0> @@ -324,7 +325,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j> ; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> @@ -425,7 +426,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> ; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> @@ -541,7 +542,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> ; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> @@ -660,7 +661,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): loop.0 @@ -775,7 +776,7 @@ ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): loop.0 @@ -844,7 +845,7 @@ ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]> ; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): pred.load @@ -996,7 +997,7 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<0>, ir<1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): loop.body ; CHECK-EMPTY: @@ -1074,7 +1075,8 @@ ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<%n>, ir<-1> +; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> +; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> ; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]> ; CHECK-NEXT: Successor(s): loop.0