diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -510,8 +510,7 @@ /// is provided, the integer induction variable will first be truncated to /// the corresponding type. \p CanonicalIV is the scalar value generated for /// the canonical induction variable. - void widenIntOrFpInduction(PHINode *IV, const InductionDescriptor &ID, - Value *Start, TruncInst *Trunc, VPValue *Def, + void widenIntOrFpInduction(PHINode *IV, VPWidenIntOrFpInductionRecipe *Def, VPTransformState &State, Value *CanonicalIV); /// Construct the vector value of a scalarized value \p V one lane at a time. @@ -2478,17 +2477,12 @@ return llvm::any_of(IV->users(), isScalarInst); } -/// Returns true if \p ID starts at 0 and has a step of 1. -static bool isCanonicalID(const InductionDescriptor &ID) { - if (!ID.getConstIntStepValue() || !ID.getConstIntStepValue()->isOne()) - return false; - auto *StartC = dyn_cast(ID.getStartValue()); - return StartC && StartC->isZero(); -} - void InnerLoopVectorizer::widenIntOrFpInduction( - PHINode *IV, const InductionDescriptor &ID, Value *Start, TruncInst *Trunc, - VPValue *Def, VPTransformState &State, Value *CanonicalIV) { + PHINode *IV, VPWidenIntOrFpInductionRecipe *Def, VPTransformState &State, + Value *CanonicalIV) { + Value *Start = Def->getStartValue()->getLiveInIRValue(); + const InductionDescriptor &ID = Def->getInductionDescriptor(); + TruncInst *Trunc = Def->getTruncInst(); IRBuilder<> &Builder = State.Builder; assert(IV->getType() == ID.getStartValue()->getType() && "Types must match"); assert(!State.VF.isZero() && "VF must be non-zero"); @@ -2519,7 +2513,7 @@ auto CreateScalarIV = [&](Value *&Step) -> Value * { Value *ScalarIV = CanonicalIV; Type *NeededType = IV->getType(); - if (!isCanonicalID(ID) || ScalarIV->getType() != NeededType) { + if (!Def->isCanonical() || ScalarIV->getType() != NeededType) { ScalarIV = NeededType->isIntegerTy() ? Builder.CreateSExtOrTrunc(ScalarIV, NeededType) @@ -9702,9 +9696,7 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Int or FP induction being replicated."); auto *CanonicalIV = State.get(getParent()->getPlan()->getCanonicalIV(), 0); - State.ILV->widenIntOrFpInduction(IV, getInductionDescriptor(), - getStartValue()->getLiveInIRValue(), - getTruncInst(), this, State, CanonicalIV); + State.ILV->widenIntOrFpInduction(IV, this, State, CanonicalIV); } void VPWidenPHIRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1059,6 +1059,7 @@ /// Returns the start value of the induction. VPValue *getStartValue() { return getOperand(0); } + const VPValue *getStartValue() const { return getOperand(0); } /// Returns the first defined value as TruncInst, if it is one or nullptr /// otherwise. @@ -1071,6 +1072,10 @@ /// Returns the induction descriptor for the recipe. const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } + + /// Returns true if the induction is canonical, i.e. starting at 0 and + /// incremented by UF * VF (= the original IV is incremented by 1). + bool isCanonical() const; }; /// A pure virtual base class for all recipes modeling header phis, including diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1263,6 +1263,12 @@ O << " " << VPlanIngredient(IV); } +bool VPWidenIntOrFpInductionRecipe::isCanonical() const { + auto *StartC = dyn_cast(getStartValue()->getLiveInIRValue()); + auto *StepC = dyn_cast(getInductionDescriptor().getStep()); + return StartC && StartC->isZero() && StepC && StepC->isOne(); +} + void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN-GEP "; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -178,6 +178,7 @@ void replaceAllUsesWith(VPValue *New); VPDef *getDef() { return Def; } + const VPDef *getDef() const { return Def; } /// Returns the underlying IR value, if this VPValue is defined outside the /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef @@ -187,6 +188,11 @@ "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); return getUnderlyingValue(); } + const Value *getLiveInIRValue() const { + assert(!getDef() && + "VPValue is not a live-in; it is defined by a VPDef inside a VPlan"); + return getUnderlyingValue(); + } }; typedef DenseMap Value2VPValueTy;