diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3703,6 +3703,10 @@ PromEltTyBits = 16; // promote to i16, AVX512BW. break; } + if (ST->hasDQI()) { + PromEltTyBits = 32; // promote to i32, AVX512F. + break; + } return bailout(); default: return bailout(); diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll b/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll --- a/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll @@ -8,6 +8,7 @@ ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512F ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512F +; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512BW ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512VBMI @@ -91,6 +92,17 @@ ; AVX512F-NEXT: Cost Model: Found an estimated cost of 384 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <256 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; AVX512DQ-LABEL: 'replication_i1_stride2' +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <2 x i32> zeroinitializer +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <4 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <8 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <16 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <32 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <64 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <128 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <256 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; AVX512BW-LABEL: 'replication_i1_stride2' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <2 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <4 x i32> @@ -202,6 +214,17 @@ ; AVX512F-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <384 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; AVX512DQ-LABEL: 'replication_i1_stride3' +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <3 x i32> zeroinitializer +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <6 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <12 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <24 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <48 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <96 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <192 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <384 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; AVX512BW-LABEL: 'replication_i1_stride3' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <3 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <6 x i32> @@ -313,6 +336,17 @@ ; AVX512F-NEXT: Cost Model: Found an estimated cost of 640 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <512 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; AVX512DQ-LABEL: 'replication_i1_stride4' +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <4 x i32> zeroinitializer +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <8 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <16 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <32 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <64 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <128 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <256 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <512 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; AVX512BW-LABEL: 'replication_i1_stride4' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <4 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <8 x i32> @@ -424,6 +458,17 @@ ; AVX512F-NEXT: Cost Model: Found an estimated cost of 768 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <640 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; AVX512DQ-LABEL: 'replication_i1_stride5' +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <5 x i32> zeroinitializer +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <10 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <20 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <40 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <80 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <160 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <320 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <640 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; AVX512BW-LABEL: 'replication_i1_stride5' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <5 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <10 x i32> @@ -535,6 +580,17 @@ ; AVX512F-NEXT: Cost Model: Found an estimated cost of 896 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <768 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; AVX512DQ-LABEL: 'replication_i1_stride6' +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <6 x i32> zeroinitializer +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <12 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <24 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <48 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <96 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <192 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <384 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 120 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <768 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; AVX512BW-LABEL: 'replication_i1_stride6' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <6 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <12 x i32> @@ -646,6 +702,17 @@ ; AVX512F-NEXT: Cost Model: Found an estimated cost of 1024 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <896 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; AVX512DQ-LABEL: 'replication_i1_stride7' +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <7 x i32> zeroinitializer +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <14 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <28 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <56 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <112 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <224 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <448 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <896 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; AVX512BW-LABEL: 'replication_i1_stride7' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <7 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <14 x i32> @@ -757,6 +824,17 @@ ; AVX512F-NEXT: Cost Model: Found an estimated cost of 1152 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <1024 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; +; AVX512DQ-LABEL: 'replication_i1_stride8' +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <8 x i32> zeroinitializer +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <16 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <32 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <64 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <128 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <256 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 68 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <512 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 136 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <1024 x i32> +; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; ; AVX512BW-LABEL: 'replication_i1_stride8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <8 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <16 x i32>