Index: lib/Target/X86/X86TargetTransformInfo.cpp =================================================================== --- lib/Target/X86/X86TargetTransformInfo.cpp +++ lib/Target/X86/X86TargetTransformInfo.cpp @@ -2639,6 +2639,9 @@ // The cost of the loads/stores is accounted for separately. // static const CostTblEntry AVX2InterleavedLoadTbl[] = { + { 2, MVT::v4i64, 6 }, //(load 8i64 and) deinterleave into 2 x 4i64 + { 2, MVT::v4f64, 6 }, //(load 8f64 and) deinterleave into 2 x 4f64 + { 3, MVT::v2i8, 10 }, //(load 6i8 and) deinterleave into 3 x 2i8 { 3, MVT::v4i8, 4 }, //(load 12i8 and) deinterleave into 3 x 4i8 { 3, MVT::v8i8, 9 }, //(load 24i8 and) deinterleave into 3 x 8i8 @@ -2656,6 +2659,9 @@ }; static const CostTblEntry AVX2InterleavedStoreTbl[] = { + { 2, MVT::v4i64, 6 }, //interleave into 2 x 4i64 into 8i64 (and store) + { 2, MVT::v4f64, 6 }, //interleave into 2 x 4f64 into 8f64 (and store) + { 3, MVT::v2i8, 7 }, //interleave 3 x 2i8 into 6i8 (and store) { 3, MVT::v4i8, 8 }, //interleave 3 x 4i8 into 12i8 (and store) { 3, MVT::v8i8, 11 }, //interleave 3 x 8i8 into 24i8 (and store) Index: test/Analysis/CostModel/X86/interleaved-load-double.ll =================================================================== --- /dev/null +++ test/Analysis/CostModel/X86/interleaved-load-double.ll @@ -0,0 +1,45 @@ +; REQUIRES: asserts +; RUN: opt -S -loop-vectorize -debug-only=loop-vectorize -mcpu=skylake %s 2>&1 | FileCheck %s +target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" +target triple = "i386-unknown-linux-gnu" + +@doublesrc = common local_unnamed_addr global [120 x double] zeroinitializer, align 4 +@doubledst = common local_unnamed_addr global [120 x double] zeroinitializer, align 4 + +; Function Attrs: norecurse nounwind +define void @stride2double(double %k, i32 %width_) { +entry: + +; CHECK: Found an estimated cost of 8 for VF 4 For instruction: %0 = load double + + %cmp27 = icmp sgt i32 %width_, 0 + br i1 %cmp27, label %for.body.lr.ph, label %for.cond.cleanup + +for.body.lr.ph: ; preds = %entry + br label %for.body + +for.cond.cleanup: ; preds = %for.body, %entry + ret void + +for.body: ; preds = %for.body.lr.ph, %for.body + %i.028 = phi i32 [ 0, %for.body.lr.ph ], [ %add16, %for.body ] + %arrayidx = getelementptr inbounds [120 x double], [120 x double]* @doublesrc, i32 0, i32 %i.028 + %0 = load double, double* %arrayidx, align 4 + %mul = fmul fast double %0, %k + %arrayidx2 = getelementptr inbounds [120 x double], [120 x double]* @doubledst, i32 0, i32 %i.028 + %1 = load double, double* %arrayidx2, align 4 + %add3 = fadd fast double %1, %mul + store double %add3, double* %arrayidx2, align 4 + %add4 = add nuw nsw i32 %i.028, 1 + %arrayidx5 = getelementptr inbounds [120 x double], [120 x double]* @doublesrc, i32 0, i32 %add4 + %2 = load double, double* %arrayidx5, align 4 + %mul6 = fmul fast double %2, %k + %arrayidx8 = getelementptr inbounds [120 x double], [120 x double]* @doubledst, i32 0, i32 %add4 + %3 = load double, double* %arrayidx8, align 4 + %add9 = fadd fast double %3, %mul6 + store double %add9, double* %arrayidx8, align 4 + %add16 = add nuw nsw i32 %i.028, 2 + %cmp = icmp slt i32 %add16, %width_ + br i1 %cmp, label %for.body, label %for.cond.cleanup +} + Index: test/Analysis/CostModel/X86/interleaved-load-i64.ll =================================================================== --- /dev/null +++ test/Analysis/CostModel/X86/interleaved-load-i64.ll @@ -0,0 +1,45 @@ +; REQUIRES: asserts +; RUN: opt -S -loop-vectorize -debug-only=loop-vectorize -mcpu=skylake %s 2>&1 | FileCheck %s +target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" +target triple = "i386-unknown-linux-gnu" + +@i64src = common local_unnamed_addr global [120 x i64] zeroinitializer, align 4 +@i64dst = common local_unnamed_addr global [120 x i64] zeroinitializer, align 4 + +; Function Attrs: norecurse nounwind +define void @stride2i64(i64 %k, i32 %width_) { +entry: + +; CHECK: Found an estimated cost of 8 for VF 4 For instruction: %0 = load i64 + + %cmp27 = icmp sgt i32 %width_, 0 + br i1 %cmp27, label %for.body.lr.ph, label %for.cond.cleanup + +for.body.lr.ph: ; preds = %entry + br label %for.body + +for.cond.cleanup: ; preds = %for.body, %entry + ret void + +for.body: + %i.028 = phi i32 [ 0, %for.body.lr.ph ], [ %add16, %for.body ] + %arrayidx = getelementptr inbounds [120 x i64], [120 x i64]* @i64src, i32 0, i32 %i.028 + %0 = load i64, i64* %arrayidx, align 4 + %mul = mul i64 %0, %k + %arrayidx2 = getelementptr inbounds [120 x i64], [120 x i64]* @i64dst, i32 0, i32 %i.028 + %1 = load i64, i64* %arrayidx2, align 4 + %add3 = add i64 %1, %mul + store i64 %add3, i64* %arrayidx2, align 4 + %add4 = add nuw nsw i32 %i.028, 1 + %arrayidx5 = getelementptr inbounds [120 x i64], [120 x i64]* @i64src, i32 0, i32 %add4 + %2 = load i64, i64* %arrayidx5, align 4 + %mul6 = mul i64 %2, %k + %arrayidx8 = getelementptr inbounds [120 x i64], [120 x i64]* @i64dst, i32 0, i32 %add4 + %3 = load i64, i64* %arrayidx8, align 4 + %add9 = add i64 %3, %mul6 + store i64 %add9, i64* %arrayidx8, align 4 + %add16 = add nuw nsw i32 %i.028, 2 + %cmp = icmp slt i32 %add16, %width_ + br i1 %cmp, label %for.body, label %for.cond.cleanup +} + Index: test/Analysis/CostModel/X86/interleaved-store-double.ll =================================================================== --- /dev/null +++ test/Analysis/CostModel/X86/interleaved-store-double.ll @@ -0,0 +1,45 @@ +; REQUIRES: asserts +; RUN: opt -S -loop-vectorize -debug-only=loop-vectorize -mcpu=skylake %s 2>&1 | FileCheck %s +target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" +target triple = "i386-unknown-linux-gnu" + +@doublesrc = common local_unnamed_addr global [120 x double] zeroinitializer, align 4 +@doubledst = common local_unnamed_addr global [120 x double] zeroinitializer, align 4 + +; Function Attrs: norecurse nounwind +define void @stride2double(double %k, i32 %width_) { +entry: + +; CHECK: Found an estimated cost of 8 for VF 4 For instruction: store double + + %cmp27 = icmp sgt i32 %width_, 0 + br i1 %cmp27, label %for.body.lr.ph, label %for.cond.cleanup + +for.body.lr.ph: ; preds = %entry + br label %for.body + +for.cond.cleanup: ; preds = %for.body, %entry + ret void + +for.body: ; preds = %for.body.lr.ph, %for.body + %i.028 = phi i32 [ 0, %for.body.lr.ph ], [ %add16, %for.body ] + %arrayidx = getelementptr inbounds [120 x double], [120 x double]* @doublesrc, i32 0, i32 %i.028 + %0 = load double, double* %arrayidx, align 4 + %mul = fmul fast double %0, %k + %arrayidx2 = getelementptr inbounds [120 x double], [120 x double]* @doubledst, i32 0, i32 %i.028 + %1 = load double, double* %arrayidx2, align 4 + %add3 = fadd fast double %1, %mul + store double %add3, double* %arrayidx2, align 4 + %add4 = add nuw nsw i32 %i.028, 1 + %arrayidx5 = getelementptr inbounds [120 x double], [120 x double]* @doublesrc, i32 0, i32 %add4 + %2 = load double, double* %arrayidx5, align 4 + %mul6 = fmul fast double %2, %k + %arrayidx8 = getelementptr inbounds [120 x double], [120 x double]* @doubledst, i32 0, i32 %add4 + %3 = load double, double* %arrayidx8, align 4 + %add9 = fadd fast double %3, %mul6 + store double %add9, double* %arrayidx8, align 4 + %add16 = add nuw nsw i32 %i.028, 2 + %cmp = icmp slt i32 %add16, %width_ + br i1 %cmp, label %for.body, label %for.cond.cleanup +} + Index: test/Analysis/CostModel/X86/interleaved-store-i64.ll =================================================================== --- /dev/null +++ test/Analysis/CostModel/X86/interleaved-store-i64.ll @@ -0,0 +1,45 @@ +; REQUIRES: asserts +; RUN: opt -S -loop-vectorize -debug-only=loop-vectorize -mcpu=skylake %s 2>&1 | FileCheck %s +target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" +target triple = "i386-unknown-linux-gnu" + +@i64src = common local_unnamed_addr global [120 x i64] zeroinitializer, align 4 +@i64dst = common local_unnamed_addr global [120 x i64] zeroinitializer, align 4 + +; Function Attrs: norecurse nounwind +define void @stride2i64(i64 %k, i32 %width_) { +entry: + +; CHECK: Found an estimated cost of 8 for VF 4 For instruction: store i64 + + %cmp27 = icmp sgt i32 %width_, 0 + br i1 %cmp27, label %for.body.lr.ph, label %for.cond.cleanup + +for.body.lr.ph: ; preds = %entry + br label %for.body + +for.cond.cleanup: ; preds = %for.body, %entry + ret void + +for.body: + %i.028 = phi i32 [ 0, %for.body.lr.ph ], [ %add16, %for.body ] + %arrayidx = getelementptr inbounds [120 x i64], [120 x i64]* @i64src, i32 0, i32 %i.028 + %0 = load i64, i64* %arrayidx, align 4 + %mul = mul i64 %0, %k + %arrayidx2 = getelementptr inbounds [120 x i64], [120 x i64]* @i64dst, i32 0, i32 %i.028 + %1 = load i64, i64* %arrayidx2, align 4 + %add3 = add i64 %1, %mul + store i64 %add3, i64* %arrayidx2, align 4 + %add4 = add nuw nsw i32 %i.028, 1 + %arrayidx5 = getelementptr inbounds [120 x i64], [120 x i64]* @i64src, i32 0, i32 %add4 + %2 = load i64, i64* %arrayidx5, align 4 + %mul6 = mul i64 %2, %k + %arrayidx8 = getelementptr inbounds [120 x i64], [120 x i64]* @i64dst, i32 0, i32 %add4 + %3 = load i64, i64* %arrayidx8, align 4 + %add9 = add i64 %3, %mul6 + store i64 %add9, i64* %arrayidx8, align 4 + %add16 = add nuw nsw i32 %i.028, 2 + %cmp = icmp slt i32 %add16, %width_ + br i1 %cmp, label %for.body, label %for.cond.cleanup +} +