Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -423,8 +423,8 @@ virtual Value *getStepVector(Value *Val, int StartIdx, const SCEV *Step); /// Create a vector induction variable based on an existing scalar one. - /// Currently only works for integer primary induction variables with - /// a constant step. + /// Currently only works for integer induction variables with a constant + /// step. /// If TruncType is provided, instead of widening the original IV, we /// widen a version of the IV truncated to TruncType. void widenInductionVariable(const InductionDescriptor &II, VectorParts &Entry, @@ -2126,7 +2126,8 @@ Builder.restoreIP(CurrIP); Value *SplatVF = - ConstantVector::getSplat(VF, ConstantInt::get(Start->getType(), VF)); + ConstantVector::getSplat(VF, ConstantInt::getSigned(Start->getType(), + VF * Step->getSExtValue())); // We may need to add the step a number of times, depending on the unroll // factor. The last of those goes into the PHI. PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind", @@ -4098,7 +4099,8 @@ llvm_unreachable("Unknown induction"); case InductionDescriptor::IK_IntInduction: { assert(P->getType() == II.getStartValue()->getType() && "Types must match"); - if (P != OldInduction || VF == 1) { + if (VF == 1 || P->getType() != Induction->getType() || + !II.getConstIntStepValue()) { Value *V = Induction; // Handle other induction variables that are now based on the // canonical one. Index: test/Transforms/LoopVectorize/induction.ll =================================================================== --- test/Transforms/LoopVectorize/induction.ll +++ test/Transforms/LoopVectorize/induction.ll @@ -174,8 +174,11 @@ ; CHECK-LABEL: wrappingindvars1 ; CHECK-LABEL: vector.scevcheck +; CHECK-LABEL: vector.ph +; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, ; CHECK-LABEL: vector.body -; CHECK: add <2 x i32> {{%[^ ]*}}, +; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ] +; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], define void @wrappingindvars1(i8 %t, i32 %len, i32 *%A) { entry: %st = zext i8 %t to i16 @@ -209,8 +212,11 @@ ; The expression gets converted to ({4 * (zext %t to i32),+,4}). ; CHECK-LABEL: wrappingindvars2 ; CHECK-LABEL: vector.scevcheck +; CHECK-LABEL: vector.ph +; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, ; CHECK-LABEL: vector.body -; CHECK: add <2 x i32> {{%[^ ]*}}, +; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ] +; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], define void @wrappingindvars2(i8 %t, i32 %len, i32 *%A) { entry: @@ -302,3 +308,44 @@ exit: ret void } + +; IND-LABEL: nonprimary +; IND-LABEL: vector.ph +; IND: %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0 +; IND: %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer +; IND: %[[START:.*]] = add <2 x i32> %[[SPLAT]], +; IND-LABEL: vector.body: +; IND: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; IND: %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %step.add, %vector.body ] +; IND: %step.add = add <2 x i32> %vec.ind, +; IND: %index.next = add i32 %index, 2 +; IND: %[[CMP:.*]] = icmp eq i32 %index.next +; IND: br i1 %[[CMP]] +; UNROLL-LABEL: nonprimary +; UNROLL-LABEL: vector.ph +; UNROLL: %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0 +; UNROLL: %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer +; UNROLL: %[[START:.*]] = add <2 x i32> %[[SPLAT]], +; UNROLL-LABEL: vector.body: +; UNROLL: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; UNROLL: %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %step.add1, %vector.body ] +; UNROLL: %step.add = add <2 x i32> %vec.ind, +; UNROLL: %step.add1 = add <2 x i32> %vec.ind, +; UNROLL: %index.next = add i32 %index, 4 +; UNROLL: %[[CMP:.*]] = icmp eq i32 %index.next +; UNROLL: br i1 %[[CMP]] +define void @nonprimary(i32* nocapture %a, i32 %start, i32 %i, i32 %k) { +for.body.preheader: + br label %for.body + +for.body: + %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ %i, %for.body.preheader ] + %arrayidx = getelementptr inbounds i32, i32* %a, i32 %indvars.iv + store i32 %indvars.iv, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i32 %indvars.iv, 42 + %exitcond = icmp eq i32 %indvars.iv.next, %k + br i1 %exitcond, label %exit, label %for.body + +exit: + ret void +} Index: test/Transforms/LoopVectorize/reverse_induction.ll =================================================================== --- test/Transforms/LoopVectorize/reverse_induction.ll +++ test/Transforms/LoopVectorize/reverse_induction.ll @@ -6,8 +6,8 @@ ; PR15882 ; CHECK-LABEL: @reverse_induction_i64( -; CHECK: add <4 x i64> %[[SPLAT:.*]], -; CHECK: add <4 x i64> %[[SPLAT]], +; CHECK: %step.add = add <4 x i64> %vec.ind, +; CHECK: %step.add2 = add <4 x i64> %step.add, define i32 @reverse_induction_i64(i64 %startval, i32 * %ptr) { entry: @@ -30,8 +30,8 @@ } ; CHECK-LABEL: @reverse_induction_i128( -; CHECK: add <4 x i128> %[[SPLAT:.*]], -; CHECK: add <4 x i128> %[[SPLAT]], +; CHECK: %step.add = add <4 x i128> %vec.ind, +; CHECK: %step.add2 = add <4 x i128> %step.add, define i32 @reverse_induction_i128(i128 %startval, i32 * %ptr) { entry: br label %for.body @@ -96,7 +96,8 @@ ; CHECK-LABEL: @reverse_forward_induction_i64_i8( ; CHECK: vector.body ; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] -; CHECK: %offset.idx = sub i64 1023, %index +; CHECK: %vec.ind = phi <4 x i64> [ , %vector.ph ] +; CHECK: %step.add = add <4 x i64> %vec.ind, ; CHECK: trunc i64 %index to i8 define void @reverse_forward_induction_i64_i8() { @@ -122,7 +123,8 @@ ; CHECK-LABEL: @reverse_forward_induction_i64_i8_signed( ; CHECK: vector.body: ; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] -; CHECK: %offset.idx = sub i64 1023, %index +; CHECK: %vec.ind = phi <4 x i64> [ , %vector.ph ] +; CHECK: %step.add = add <4 x i64> %vec.ind, define void @reverse_forward_induction_i64_i8_signed() { entry: