Index: lib/Transforms/Vectorize/SLPVectorizer.cpp
===================================================================
--- lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4895,14 +4895,23 @@
// Look for the next elements with the same type.
SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
- while (SameTypeIt != E &&
- (*SameTypeIt)->getType() == (*IncIt)->getType()) {
- VisitedInstrs.insert(*SameTypeIt);
- ++SameTypeIt;
+ Type *Ty = (*IncIt)->getType();
+ assert(DL->getTypeSizeInBits(Ty) > 0 && "Zero sized type?");
+ unsigned MaxNumElts = R.getMaxVecRegSize() / DL->getTypeSizeInBits(Ty);
+ if (MaxNumElts < 2) {
+ // The elements of this type are not vectorizable, skip them.
+ while (IncIt != E && Ty == (*IncIt)->getType())
+ VisitedInstrs.insert(*IncIt++);
+ continue;
}
+ while (SameTypeIt != E &&
+ Ty == (*SameTypeIt)->getType() &&
+ SameTypeIt - IncIt < MaxNumElts)
+ VisitedInstrs.insert(*SameTypeIt++);
// Try to vectorize them.
unsigned NumElts = (SameTypeIt - IncIt);
+ assert(NumElts <= MaxNumElts && "Desired vector size is too big!");
DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n");
// The order in which the phi nodes appear in the program does not matter.
// So allow tryToVectorizeList to reorder them if it is beneficial. This
Index: test/Transforms/SLPVectorizer/X86/phi_limited.ll
===================================================================
--- test/Transforms/SLPVectorizer/X86/phi_limited.ll
+++ test/Transforms/SLPVectorizer/X86/phi_limited.ll
@@ -0,0 +1,251 @@
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx -slp-max-reg-size=64 | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.9.0"
+
+; Same as phi.ll, but the maximum vector register size is limited to 64 bits.
+
+;int foo(double *A, int k) {
+; double A0;
+; double A1;
+; if (k) {
+; A0 = 3;
+; A1 = 5;
+; } else {
+; A0 = A[10];
+; A1 = A[11];
+; }
+; A[0] = A0;
+; A[1] = A1;
+;}
+
+;CHECK: i32 @foo
+;CHECK-NOT: <2 x double>
+;CHECK: load double
+;CHECK: phi double
+;CHECK: store double
+;CHECK: ret i32 undef
+define i32 @foo(double* nocapture %A, i32 %k) {
+entry:
+ %tobool = icmp eq i32 %k, 0
+ br i1 %tobool, label %if.else, label %if.end
+
+if.else: ; preds = %entry
+ %arrayidx = getelementptr inbounds double, double* %A, i64 10
+ %0 = load double, double* %arrayidx, align 8
+ %arrayidx1 = getelementptr inbounds double, double* %A, i64 11
+ %1 = load double, double* %arrayidx1, align 8
+ br label %if.end
+
+if.end: ; preds = %entry, %if.else
+ %A0.0 = phi double [ %0, %if.else ], [ 3.000000e+00, %entry ]
+ %A1.0 = phi double [ %1, %if.else ], [ 5.000000e+00, %entry ]
+ store double %A0.0, double* %A, align 8
+ %arrayidx3 = getelementptr inbounds double, double* %A, i64 1
+ store double %A1.0, double* %arrayidx3, align 8
+ ret i32 undef
+}
+
+
+;int foo(double * restrict B, double * restrict A, int n, int m) {
+; double R=A[1];
+; double G=A[0];
+; for (int i=0; i < 100; i++) {
+; R += 10;
+; G += 10;
+; R *= 4;
+; G *= 4;
+; R += 4;
+; G += 4;
+; }
+; B[0] = G;
+; B[1] = R;
+; return 0;
+;}
+
+;CHECK: foo2
+;CHECK-NOT: <2 x double>
+;CHECK: load double
+;CHECK: phi double
+;CHECK: fmul double
+;CHECK: store double
+;CHECK: ret
+define i32 @foo2(double* noalias nocapture %B, double* noalias nocapture %A, i32 %n, i32 %m) #0 {
+entry:
+ %arrayidx = getelementptr inbounds double, double* %A, i64 1
+ %0 = load double, double* %arrayidx, align 8
+ %1 = load double, double* %A, align 8
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %i.019 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %G.018 = phi double [ %1, %entry ], [ %add5, %for.body ]
+ %R.017 = phi double [ %0, %entry ], [ %add4, %for.body ]
+ %add = fadd double %R.017, 1.000000e+01
+ %add2 = fadd double %G.018, 1.000000e+01
+ %mul = fmul double %add, 4.000000e+00
+ %mul3 = fmul double %add2, 4.000000e+00
+ %add4 = fadd double %mul, 4.000000e+00
+ %add5 = fadd double %mul3, 4.000000e+00
+ %inc = add nsw i32 %i.019, 1
+ %exitcond = icmp eq i32 %inc, 100
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body
+ store double %add5, double* %B, align 8
+ %arrayidx7 = getelementptr inbounds double, double* %B, i64 1
+ store double %add4, double* %arrayidx7, align 8
+ ret i32 0
+}
+
+; float foo3(float *A) {
+;
+; float R = A[0];
+; float G = A[1];
+; float B = A[2];
+; float Y = A[3];
+; float P = A[4];
+; for (int i=0; i < 121; i+=3) {
+; R+=A[i+0]*7;
+; G+=A[i+1]*8;
+; B+=A[i+2]*9;
+; Y+=A[i+3]*10;
+; P+=A[i+4]*11;
+; }
+;
+; return R+G+B+Y+P;
+; }
+
+;CHECK: foo3
+;CHECK-NOT: <4 x float>
+;CHECK-NOT: <5 x float>
+;CHECK: phi <2 x float>
+;CHECK: fmul <2 x float>
+;CHECK: fadd <2 x float>
+
+define float @foo3(float* nocapture readonly %A) #0 {
+entry:
+ %0 = load float, float* %A, align 4
+ %arrayidx1 = getelementptr inbounds float, float* %A, i64 1
+ %1 = load float, float* %arrayidx1, align 4
+ %arrayidx2 = getelementptr inbounds float, float* %A, i64 2
+ %2 = load float, float* %arrayidx2, align 4
+ %arrayidx3 = getelementptr inbounds float, float* %A, i64 3
+ %3 = load float, float* %arrayidx3, align 4
+ %arrayidx4 = getelementptr inbounds float, float* %A, i64 4
+ %4 = load float, float* %arrayidx4, align 4
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %P.056 = phi float [ %4, %entry ], [ %add26, %for.body ]
+ %Y.055 = phi float [ %3, %entry ], [ %add21, %for.body ]
+ %B.054 = phi float [ %2, %entry ], [ %add16, %for.body ]
+ %G.053 = phi float [ %1, %entry ], [ %add11, %for.body ]
+ %R.052 = phi float [ %0, %entry ], [ %add6, %for.body ]
+ %5 = phi float [ %1, %entry ], [ %11, %for.body ]
+ %6 = phi float [ %0, %entry ], [ %9, %for.body ]
+ %mul = fmul float %6, 7.000000e+00
+ %add6 = fadd float %R.052, %mul
+ %mul10 = fmul float %5, 8.000000e+00
+ %add11 = fadd float %G.053, %mul10
+ %7 = add nsw i64 %indvars.iv, 2
+ %arrayidx14 = getelementptr inbounds float, float* %A, i64 %7
+ %8 = load float, float* %arrayidx14, align 4
+ %mul15 = fmul float %8, 9.000000e+00
+ %add16 = fadd float %B.054, %mul15
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 3
+ %arrayidx19 = getelementptr inbounds float, float* %A, i64 %indvars.iv.next
+ %9 = load float, float* %arrayidx19, align 4
+ %mul20 = fmul float %9, 1.000000e+01
+ %add21 = fadd float %Y.055, %mul20
+ %10 = add nsw i64 %indvars.iv, 4
+ %arrayidx24 = getelementptr inbounds float, float* %A, i64 %10
+ %11 = load float, float* %arrayidx24, align 4
+ %mul25 = fmul float %11, 1.100000e+01
+ %add26 = fadd float %P.056, %mul25
+ %12 = trunc i64 %indvars.iv.next to i32
+ %cmp = icmp slt i32 %12, 121
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end: ; preds = %for.body
+ %add28 = fadd float %add6, %add11
+ %add29 = fadd float %add28, %add16
+ %add30 = fadd float %add29, %add21
+ %add31 = fadd float %add30, %add26
+ ret float %add31
+}
+
+; Make sure the order of phi nodes of different types does not prevent
+; vectorization of same typed phi nodes.
+; CHECK-LABEL: sort_phi_type
+; CHECK-NOT: <4 x float>
+; CHECK: phi <2 x float>
+; CHECK: fmul <2 x float>
+
+define float @sort_phi_type(float* nocapture readonly %A) {
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %Y = phi float [ 1.000000e+01, %entry ], [ %mul10, %for.body ]
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %B = phi float [ 1.000000e+01, %entry ], [ %mul15, %for.body ]
+ %G = phi float [ 1.000000e+01, %entry ], [ %mul20, %for.body ]
+ %R = phi float [ 1.000000e+01, %entry ], [ %mul25, %for.body ]
+ %mul10 = fmul float %Y, 8.000000e+00
+ %mul15 = fmul float %B, 9.000000e+00
+ %mul20 = fmul float %R, 10.000000e+01
+ %mul25 = fmul float %G, 11.100000e+01
+ %indvars.iv.next = add nsw i64 %indvars.iv, 4
+ %cmp = icmp slt i64 %indvars.iv.next, 128
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end: ; preds = %for.body
+ %add28 = fadd float 1.000000e+01, %mul10
+ %add29 = fadd float %mul10, %mul15
+ %add30 = fadd float %add29, %mul20
+ %add31 = fadd float %add30, %mul25
+ ret float %add31
+}
+
+define void @test(x86_fp80* %i1, x86_fp80* %i2, x86_fp80* %o) {
+; CHECK-LABEL: @test(
+;
+; Test that we correctly recognize the discontiguous memory in arrays where the
+; size is less than the alignment, and through various different GEP formations.
+;
+; We disable the vectorization of x86_fp80 for now.
+
+entry:
+ %i1.0 = load x86_fp80, x86_fp80* %i1, align 16
+ %i1.gep1 = getelementptr x86_fp80, x86_fp80* %i1, i64 1
+ %i1.1 = load x86_fp80, x86_fp80* %i1.gep1, align 16
+; CHECK: load x86_fp80, x86_fp80*
+; CHECK: load x86_fp80, x86_fp80*
+; CHECK-NOT: insertelement <2 x x86_fp80>
+; CHECK-NOT: insertelement <2 x x86_fp80>
+ br i1 undef, label %then, label %end
+
+then:
+ %i2.gep0 = getelementptr inbounds x86_fp80, x86_fp80* %i2, i64 0
+ %i2.0 = load x86_fp80, x86_fp80* %i2.gep0, align 16
+ %i2.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %i2, i64 1
+ %i2.1 = load x86_fp80, x86_fp80* %i2.gep1, align 16
+; CHECK: load x86_fp80, x86_fp80*
+; CHECK: load x86_fp80, x86_fp80*
+; CHECK-NOT: insertelement <2 x x86_fp80>
+; CHECK-NOT: insertelement <2 x x86_fp80>
+ br label %end
+
+end:
+ %phi0 = phi x86_fp80 [ %i1.0, %entry ], [ %i2.0, %then ]
+ %phi1 = phi x86_fp80 [ %i1.1, %entry ], [ %i2.1, %then ]
+; CHECK-NOT: phi <2 x x86_fp80>
+; CHECK-NOT: extractelement <2 x x86_fp80>
+; CHECK-NOT: extractelement <2 x x86_fp80>
+ store x86_fp80 %phi0, x86_fp80* %o, align 16
+ %o.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %o, i64 1
+ store x86_fp80 %phi1, x86_fp80* %o.gep1, align 16
+ ret void
+}