diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -5313,26 +5313,17 @@ }; static const CostTblEntry AVX2InterleavedStoreTbl[] = { - {2, MVT::v2i8, 1}, // interleave 2 x 2i8 into 4i8 (and store) - {2, MVT::v4i8, 1}, // interleave 2 x 4i8 into 8i8 (and store) - {2, MVT::v8i8, 1}, // interleave 2 x 8i8 into 16i8 (and store) {2, MVT::v16i8, 3}, // interleave 2 x 16i8 into 32i8 (and store) {2, MVT::v32i8, 4}, // interleave 2 x 32i8 into 64i8 (and store) - {2, MVT::v2i16, 1}, // interleave 2 x 2i16 into 4i16 (and store) - {2, MVT::v4i16, 1}, // interleave 2 x 4i16 into 8i16 (and store) {2, MVT::v8i16, 3}, // interleave 2 x 8i16 into 16i16 (and store) {2, MVT::v16i16, 4}, // interleave 2 x 16i16 into 32i16 (and store) {2, MVT::v32i16, 8}, // interleave 2 x 32i16 into 64i16 (and store) - {2, MVT::v2i32, 1}, // interleave 2 x 2i32 into 4i32 (and store) - {2, MVT::v4i32, 2}, // interleave 2 x 4i32 into 8i32 (and store) {2, MVT::v8i32, 4}, // interleave 2 x 8i32 into 16i32 (and store) {2, MVT::v16i32, 8}, // interleave 2 x 16i32 into 32i32 (and store) {2, MVT::v32i32, 16}, // interleave 2 x 32i32 into 64i32 (and store) - {2, MVT::v2i64, 2}, // interleave 2 x 2i64 into 4i64 (and store) - {2, MVT::v4i64, 4}, // interleave 2 x 4i64 into 8i64 (and store) {2, MVT::v8i64, 8}, // interleave 2 x 8i64 into 16i64 (and store) {2, MVT::v16i64, 16}, // interleave 2 x 16i64 into 32i64 (and store) @@ -5400,6 +5391,21 @@ {6, MVT::v8i64, 30}, // interleave 6 x 8i64 into 48i64 (and store) }; + static const CostTblEntry SSE2InterleavedStoreTbl[] = { + {2, MVT::v2i8, 1}, // interleave 2 x 2i8 into 4i8 (and store) + {2, MVT::v4i8, 1}, // interleave 2 x 4i8 into 8i8 (and store) + {2, MVT::v8i8, 1}, // interleave 2 x 8i8 into 16i8 (and store) + + {2, MVT::v2i16, 1}, // interleave 2 x 2i16 into 4i16 (and store) + {2, MVT::v4i16, 1}, // interleave 2 x 4i16 into 8i16 (and store) + + {2, MVT::v2i32, 1}, // interleave 2 x 2i32 into 4i32 (and store) + {2, MVT::v4i32, 2}, // interleave 2 x 4i32 into 8i32 (and store) + + {2, MVT::v2i64, 2}, // interleave 2 x 2i64 into 4i64 (and store) + {2, MVT::v4i64, 4}, // interleave 2 x 4i64 into 8i64 (and store) + }; + if (Opcode == Instruction::Load) { // FIXME: if we have a partially-interleaved groups, with gaps, // should we discount the not-demanded indicies? @@ -5426,6 +5432,11 @@ if (const auto *Entry = CostTableLookup(AVX2InterleavedStoreTbl, Factor, ETy.getSimpleVT())) return MemOpCosts + Entry->Cost; + + if (ST->hasSSE2()) + if (const auto *Entry = CostTableLookup(SSE2InterleavedStoreTbl, Factor, + ETy.getSimpleVT())) + return MemOpCosts + Entry->Cost; } return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-2.ll --- a/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-2.ll +++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-f32-stride-2.ll @@ -13,14 +13,14 @@ ; CHECK: LV: Checking a loop in "test" ; ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, float* %out1, align 4 -; SSE2: LV: Found an estimated cost of 6 for VF 2 For instruction: store float %v1, float* %out1, align 4 -; SSE2: LV: Found an estimated cost of 14 for VF 4 For instruction: store float %v1, float* %out1, align 4 +; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store float %v1, float* %out1, align 4 +; SSE2: LV: Found an estimated cost of 4 for VF 4 For instruction: store float %v1, float* %out1, align 4 ; SSE2: LV: Found an estimated cost of 28 for VF 8 For instruction: store float %v1, float* %out1, align 4 ; SSE2: LV: Found an estimated cost of 56 for VF 16 For instruction: store float %v1, float* %out1, align 4 ; ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store float %v1, float* %out1, align 4 -; AVX1: LV: Found an estimated cost of 6 for VF 2 For instruction: store float %v1, float* %out1, align 4 -; AVX1: LV: Found an estimated cost of 15 for VF 4 For instruction: store float %v1, float* %out1, align 4 +; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: store float %v1, float* %out1, align 4 +; AVX1: LV: Found an estimated cost of 3 for VF 4 For instruction: store float %v1, float* %out1, align 4 ; AVX1: LV: Found an estimated cost of 38 for VF 8 For instruction: store float %v1, float* %out1, align 4 ; AVX1: LV: Found an estimated cost of 76 for VF 16 For instruction: store float %v1, float* %out1, align 4 ; AVX1: LV: Found an estimated cost of 152 for VF 32 For instruction: store float %v1, float* %out1, align 4 diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll --- a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll +++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll @@ -13,14 +13,14 @@ ; CHECK: LV: Checking a loop in "test" ; ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, double* %out1, align 8 -; SSE2: LV: Found an estimated cost of 6 for VF 2 For instruction: store double %v1, double* %out1, align 8 -; SSE2: LV: Found an estimated cost of 12 for VF 4 For instruction: store double %v1, double* %out1, align 8 +; SSE2: LV: Found an estimated cost of 4 for VF 2 For instruction: store double %v1, double* %out1, align 8 +; SSE2: LV: Found an estimated cost of 8 for VF 4 For instruction: store double %v1, double* %out1, align 8 ; SSE2: LV: Found an estimated cost of 24 for VF 8 For instruction: store double %v1, double* %out1, align 8 ; SSE2: LV: Found an estimated cost of 48 for VF 16 For instruction: store double %v1, double* %out1, align 8 ; ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store double %v1, double* %out1, align 8 -; AVX1: LV: Found an estimated cost of 6 for VF 2 For instruction: store double %v1, double* %out1, align 8 -; AVX1: LV: Found an estimated cost of 16 for VF 4 For instruction: store double %v1, double* %out1, align 8 +; AVX1: LV: Found an estimated cost of 3 for VF 2 For instruction: store double %v1, double* %out1, align 8 +; AVX1: LV: Found an estimated cost of 6 for VF 4 For instruction: store double %v1, double* %out1, align 8 ; AVX1: LV: Found an estimated cost of 32 for VF 8 For instruction: store double %v1, double* %out1, align 8 ; AVX1: LV: Found an estimated cost of 64 for VF 16 For instruction: store double %v1, double* %out1, align 8 ; AVX1: LV: Found an estimated cost of 128 for VF 32 For instruction: store double %v1, double* %out1, align 8 diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-2.ll --- a/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-2.ll +++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-2.ll @@ -13,14 +13,14 @@ ; CHECK: LV: Checking a loop in "test" ; ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, i16* %out1, align 2 -; SSE2: LV: Found an estimated cost of 9 for VF 2 For instruction: store i16 %v1, i16* %out1, align 2 -; SSE2: LV: Found an estimated cost of 17 for VF 4 For instruction: store i16 %v1, i16* %out1, align 2 +; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %v1, i16* %out1, align 2 +; SSE2: LV: Found an estimated cost of 2 for VF 4 For instruction: store i16 %v1, i16* %out1, align 2 ; SSE2: LV: Found an estimated cost of 34 for VF 8 For instruction: store i16 %v1, i16* %out1, align 2 ; SSE2: LV: Found an estimated cost of 68 for VF 16 For instruction: store i16 %v1, i16* %out1, align 2 ; ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v1, i16* %out1, align 2 -; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction: store i16 %v1, i16* %out1, align 2 -; AVX1: LV: Found an estimated cost of 17 for VF 4 For instruction: store i16 %v1, i16* %out1, align 2 +; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %v1, i16* %out1, align 2 +; AVX1: LV: Found an estimated cost of 2 for VF 4 For instruction: store i16 %v1, i16* %out1, align 2 ; AVX1: LV: Found an estimated cost of 35 for VF 8 For instruction: store i16 %v1, i16* %out1, align 2 ; AVX1: LV: Found an estimated cost of 86 for VF 16 For instruction: store i16 %v1, i16* %out1, align 2 ; AVX1: LV: Found an estimated cost of 172 for VF 32 For instruction: store i16 %v1, i16* %out1, align 2 diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-2.ll --- a/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-2.ll +++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-i32-stride-2.ll @@ -13,14 +13,14 @@ ; CHECK: LV: Checking a loop in "test" ; ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, i32* %out1, align 4 -; SSE2: LV: Found an estimated cost of 14 for VF 2 For instruction: store i32 %v1, i32* %out1, align 4 -; SSE2: LV: Found an estimated cost of 30 for VF 4 For instruction: store i32 %v1, i32* %out1, align 4 +; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %v1, i32* %out1, align 4 +; SSE2: LV: Found an estimated cost of 4 for VF 4 For instruction: store i32 %v1, i32* %out1, align 4 ; SSE2: LV: Found an estimated cost of 60 for VF 8 For instruction: store i32 %v1, i32* %out1, align 4 ; SSE2: LV: Found an estimated cost of 120 for VF 16 For instruction: store i32 %v1, i32* %out1, align 4 ; ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %v1, i32* %out1, align 4 -; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction: store i32 %v1, i32* %out1, align 4 -; AVX1: LV: Found an estimated cost of 19 for VF 4 For instruction: store i32 %v1, i32* %out1, align 4 +; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %v1, i32* %out1, align 4 +; AVX1: LV: Found an estimated cost of 3 for VF 4 For instruction: store i32 %v1, i32* %out1, align 4 ; AVX1: LV: Found an estimated cost of 46 for VF 8 For instruction: store i32 %v1, i32* %out1, align 4 ; AVX1: LV: Found an estimated cost of 92 for VF 16 For instruction: store i32 %v1, i32* %out1, align 4 ; AVX1: LV: Found an estimated cost of 184 for VF 32 For instruction: store i32 %v1, i32* %out1, align 4 diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll --- a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll +++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll @@ -13,14 +13,14 @@ ; CHECK: LV: Checking a loop in "test" ; ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, i64* %out1, align 8 -; SSE2: LV: Found an estimated cost of 14 for VF 2 For instruction: store i64 %v1, i64* %out1, align 8 -; SSE2: LV: Found an estimated cost of 28 for VF 4 For instruction: store i64 %v1, i64* %out1, align 8 +; SSE2: LV: Found an estimated cost of 4 for VF 2 For instruction: store i64 %v1, i64* %out1, align 8 +; SSE2: LV: Found an estimated cost of 8 for VF 4 For instruction: store i64 %v1, i64* %out1, align 8 ; SSE2: LV: Found an estimated cost of 56 for VF 8 For instruction: store i64 %v1, i64* %out1, align 8 ; SSE2: LV: Found an estimated cost of 112 for VF 16 For instruction: store i64 %v1, i64* %out1, align 8 ; ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %v1, i64* %out1, align 8 -; AVX1: LV: Found an estimated cost of 11 for VF 2 For instruction: store i64 %v1, i64* %out1, align 8 -; AVX1: LV: Found an estimated cost of 26 for VF 4 For instruction: store i64 %v1, i64* %out1, align 8 +; AVX1: LV: Found an estimated cost of 3 for VF 2 For instruction: store i64 %v1, i64* %out1, align 8 +; AVX1: LV: Found an estimated cost of 6 for VF 4 For instruction: store i64 %v1, i64* %out1, align 8 ; AVX1: LV: Found an estimated cost of 52 for VF 8 For instruction: store i64 %v1, i64* %out1, align 8 ; AVX1: LV: Found an estimated cost of 104 for VF 16 For instruction: store i64 %v1, i64* %out1, align 8 ; AVX1: LV: Found an estimated cost of 208 for VF 32 For instruction: store i64 %v1, i64* %out1, align 8 diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-2.ll --- a/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-2.ll +++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-2.ll @@ -13,15 +13,15 @@ ; CHECK: LV: Checking a loop in "test" ; ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, i8* %out1, align 1 -; SSE2: LV: Found an estimated cost of 14 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1 -; SSE2: LV: Found an estimated cost of 30 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1 -; SSE2: LV: Found an estimated cost of 62 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1 +; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1 +; SSE2: LV: Found an estimated cost of 2 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1 +; SSE2: LV: Found an estimated cost of 2 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1 ; SSE2: LV: Found an estimated cost of 126 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1 ; ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, i8* %out1, align 1 -; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1 -; AVX1: LV: Found an estimated cost of 17 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1 -; AVX1: LV: Found an estimated cost of 33 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1 +; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1 +; AVX1: LV: Found an estimated cost of 2 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1 +; AVX1: LV: Found an estimated cost of 2 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1 ; AVX1: LV: Found an estimated cost of 67 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1 ; AVX1: LV: Found an estimated cost of 166 for VF 32 For instruction: store i8 %v1, i8* %out1, align 1