Index: ../include/llvm/Transforms/Utils/LoopUtils.h =================================================================== --- ../include/llvm/Transforms/Utils/LoopUtils.h +++ ../include/llvm/Transforms/Utils/LoopUtils.h @@ -286,21 +286,21 @@ Value *getStartValue() const { return StartValue; } InductionKind getKind() const { return IK; } - ConstantInt *getStepValue() const { return StepValue; } + Value *getStepValue() const { return StepValue; } static bool isInductionPHI(PHINode *Phi, ScalarEvolution *SE, InductionDescriptor &D); private: /// Private constructor - used by \c isInductionPHI. - InductionDescriptor(Value *Start, InductionKind K, ConstantInt *Step); + InductionDescriptor(Value *Start, InductionKind K, Value *Step); /// Start value. TrackingVH StartValue; /// Induction kind. InductionKind IK; /// Step value. - ConstantInt *StepValue; + Value *StepValue; }; BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, Index: ../lib/Transforms/Utils/LoopUtils.cpp =================================================================== --- ../lib/Transforms/Utils/LoopUtils.cpp +++ ../lib/Transforms/Utils/LoopUtils.cpp @@ -653,11 +653,14 @@ } InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, - ConstantInt *Step) + Value *Step) : StartValue(Start), IK(K), StepValue(Step) { assert(IK != IK_NoInduction && "Not an induction"); assert(StartValue && "StartValue is null"); - assert(StepValue && !StepValue->isZero() && "StepValue is zero"); + assert((!isa(StepValue) || + !cast(StepValue)->isZero()) && "StepValue is zero"); + assert((IK != IK_PtrInduction || isa(StepValue)) && + "Step value should be constant for pointer induction"); assert((IK != IK_PtrInduction || StartValue->getType()->isPointerTy()) && "StartValue is not a pointer for pointer induction"); assert((IK != IK_IntInduction || StartValue->getType()->isIntegerTy()) && @@ -667,8 +670,9 @@ } int InductionDescriptor::getConsecutiveDirection() const { - if (StepValue && (StepValue->isOne() || StepValue->isMinusOne())) - return StepValue->getSExtValue(); + ConstantInt *ConstStep = dyn_cast(StepValue); + if (ConstStep && (ConstStep->isOne() || ConstStep->isMinusOne())) + return ConstStep->getSExtValue(); return 0; } @@ -677,18 +681,21 @@ case IK_IntInduction: assert(Index->getType() == StartValue->getType() && "Index type does not match StartValue type"); - if (StepValue->isMinusOne()) + if (isa(StepValue) && + cast(StepValue)->isMinusOne()) return B.CreateSub(StartValue, Index); - if (!StepValue->isOne()) + if (!isa(StepValue) || !cast(StepValue)->isOne()) Index = B.CreateMul(Index, StepValue); return B.CreateAdd(StartValue, Index); case IK_PtrInduction: assert(Index->getType() == StepValue->getType() && "Index type does not match StepValue type"); - if (StepValue->isMinusOne()) + assert(isa(StepValue) && + "Expected constant step for pointer induction"); + if (cast(StepValue)->isMinusOne()) Index = B.CreateNeg(Index); - else if (!StepValue->isOne()) + else if (!cast(StepValue)->isOne()) Index = B.CreateMul(Index, StepValue); return B.CreateGEP(nullptr, StartValue, Index); @@ -719,17 +726,34 @@ Phi->getIncomingValueForBlock(AR->getLoop()->getLoopPreheader()); const SCEV *Step = AR->getStepRecurrence(*SE); // Calculate the pointer stride and check if it is consecutive. - const SCEVConstant *C = dyn_cast(Step); - if (!C) + // The stride may be a constant or a loop invariant integer value. + const SCEVConstant *ConstStep = dyn_cast(Step); + if (!ConstStep && !SE->isLoopInvariant(Step, AR->getLoop())) return false; - ConstantInt *CV = C->getValue(); + Value *StepVal; + if (ConstStep) + StepVal = ConstStep->getValue(); + else { + SetVector *Set = SE->getSCEVValues(Step); + if (!Set || Set->size() != 1) + return false; + + StepVal = Set->back(); + if (!StepVal->getType()->isIntegerTy()) + return false; + } if (PhiTy->isIntegerTy()) { - D = InductionDescriptor(StartValue, IK_IntInduction, CV); + D = InductionDescriptor(StartValue, IK_IntInduction, StepVal); return true; } assert(PhiTy->isPointerTy() && "The PHI must be a pointer"); + // Pointer induction should be a constant. + if (!ConstStep) + return false; + + ConstantInt *CV = ConstStep->getValue(); Type *PointerElementType = PhiTy->getPointerElementType(); // The pointer stride cannot be determined if the pointer element type is not // sized. Index: ../lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- ../lib/Transforms/Vectorize/LoopVectorize.cpp +++ ../lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4217,8 +4217,10 @@ Value *Broadcasted = getBroadcastInstrs(ScalarCast); InductionDescriptor II = Legal->getInductionVars()->lookup(OldInduction); - Constant *Step = ConstantInt::getSigned( - CI->getType(), II.getStepValue()->getSExtValue()); + Value *Step = II.getStepValue(); + if (isa(Step)) + Step = ConstantInt::getSigned(CI->getType(), + cast(Step)->getSExtValue()); for (unsigned Part = 0; Part < UF; ++Part) Entry[Part] = getStepVector(Broadcasted, VF * Part, Step); addMetadata(Entry, &*it); @@ -4633,7 +4635,8 @@ // Int inductions are special because we only allow one IV. if (ID.getKind() == InductionDescriptor::IK_IntInduction && - ID.getStepValue()->isOne() && + isa(ID.getStepValue()) && + cast(ID.getStepValue())->isOne() && isa(ID.getStartValue()) && cast(ID.getStartValue())->isNullValue()) { // Use the phi node with the widest type as induction. Use the last Index: ../test/Transforms/LoopVectorize/induction-step.ll =================================================================== --- ../test/Transforms/LoopVectorize/induction-step.ll +++ ../test/Transforms/LoopVectorize/induction-step.ll @@ -0,0 +1,54 @@ +; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -S | FileCheck %s + +; int int_inc; +; +;int @induction_with_global(int init, int *restrict A, int N) { +; int x = init; +; for (int i=0;i undef, i32 %[[INT_INC]], i32 0 +; CHECK: %[[VAR2:.*]] = shufflevector <8 x i32> %[[VAR1]], <8 x i32> undef, <8 x i32> zeroinitializer +; CHECK: %8 = mul <8 x i32> , %[[VAR2]] + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" + + +@int_inc = common global i32 0, align 4 + +define i32 @induction_with_global(i32 %init, i32* noalias nocapture %A, i32 %N) { +entry: + %cmp4 = icmp sgt i32 %N, 0 + br i1 %cmp4, label %for.body.lr.ph, label %for.end + +for.body.lr.ph: ; preds = %entry + %0 = load i32, i32* @int_inc, align 4 + %1 = mul i32 %0, %N + br label %for.body + +for.body: ; preds = %for.body, %for.body.lr.ph + %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ] + %x.05 = phi i32 [ %init, %for.body.lr.ph ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv + store i32 %x.05, i32* %arrayidx, align 4 + %add = add nsw i32 %0, %x.05 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %N + br i1 %exitcond, label %for.end.loopexit, label %for.body + +for.end.loopexit: ; preds = %for.body + %2 = add i32 %1, %init + br label %for.end + +for.end: ; preds = %for.end.loopexit, %entry + %x.0.lcssa = phi i32 [ %init, %entry ], [ %2, %for.end.loopexit ] + ret i32 %x.0.lcssa +}