Index: lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -13985,10 +13985,16 @@ EVT InVT1 = VecIn1.getValueType(); EVT InVT2 = VecIn2.getNode() ? VecIn2.getValueType() : InVT1; - unsigned Vec2Offset = InVT1.getVectorNumElements(); + unsigned Vec2Offset = 0; unsigned NumElems = VT.getVectorNumElements(); unsigned ShuffleNumElems = NumElems; + if (!(VecIn1.getNode() && VecIn2.getNode() && + (VecIn1.getOpcode() == ISD::EXTRACT_SUBVECTOR) && + (VecIn2.getOpcode() == ISD::EXTRACT_SUBVECTOR) && + (VecIn1.getOperand(0) == VecIn2.getOperand(0)))) + Vec2Offset = InVT1.getVectorNumElements(); + // We can't generate a shuffle node with mismatched input and output types. // Try to make the types match the type of the output. if (InVT1 != VT || InVT2 != VT) { @@ -14111,13 +14117,12 @@ // VectorMask maps each element to its vector number, and VecIn maps vector // numbers to their initial SDValues. + unsigned ValidElems = 0; SmallVector VectorMask(NumElems, -1); SmallVector VecIn; VecIn.push_back(SDValue()); - for (unsigned i = 0; i != NumElems; ++i) { SDValue Op = N->getOperand(i); - if (Op.isUndef()) continue; @@ -14129,13 +14134,11 @@ VectorMask[i] = 0; continue; } - // Not an undef or zero. If the input is something other than an // EXTRACT_VECTOR_ELT with a constant index, bail out. if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || !isa(Op.getOperand(1))) return SDValue(); - SDValue ExtractedFromVec = Op.getOperand(0); // All inputs must have the same element type as the output. @@ -14150,13 +14153,49 @@ VecIn.begin(), std::find(VecIn.begin(), VecIn.end(), ExtractedFromVec)); if (Idx == VecIn.size()) VecIn.push_back(ExtractedFromVec); - VectorMask[i] = Idx; + ValidElems++; } // If we didn't find at least one input vector, bail out. if (VecIn.size() < 2) return SDValue(); + + // If all the Operands of BUILD_VECTOR extract from same + // vector, then split the vector efficiently based on the maximum + // vector access index and adjust the VectorMask and + // VecIn accordingly. + if (VecIn.size() == 2 && NumElems == ValidElems) { + unsigned MaxIndex = 0; + unsigned NearestPow2 = 0; + SDValue Vec = VecIn.back(); + EVT InVT = Vec.getValueType(); + MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); + SmallVector IndexVec(NumElems, 0); + + for(unsigned i = 0 ; i < NumElems ; i++) { + unsigned Index = N->getOperand(i).getConstantOperandVal(1); + IndexVec[i] = Index; + MaxIndex = (MaxIndex < Index) ? Index : MaxIndex; + } + if ((NearestPow2 = PowerOf2Ceil(MaxIndex)) && ((NumElems*2) < NearestPow2)) { + unsigned SplitSize = NearestPow2 / 2; + if ( 0 == SplitSize % 2) { + EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), + InVT.getVectorElementType(), SplitSize); + SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, + DAG.getConstant(SplitSize, DL, IdxTy)); + SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, + DAG.getConstant(0, DL, IdxTy)); + VecIn.pop_back(); + VecIn.push_back(VecIn1); + VecIn.push_back(VecIn2); + + for(unsigned i = 0 ; i < NumElems ; i++) + VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2; + } + } + } // TODO: We want to sort the vectors by descending length, so that adjacent // pairs have similar length, and the longer vector is always first in the @@ -14246,7 +14285,6 @@ DAG.getVectorShuffle(VT, DL, Shuffles[Left], Shuffles[Right], Mask); } } - return Shuffles[0]; } Index: test/CodeGen/X86/oddshuffles.ll =================================================================== --- test/CodeGen/X86/oddshuffles.ll +++ test/CodeGen/X86/oddshuffles.ll @@ -940,17 +940,17 @@ ; ; AVX2-LABEL: interleave_24i16_out: ; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %ymm0 -; AVX2-NEXT: vmovdqu 32(%rdi), %xmm1 -; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15] +; AVX2-NEXT: vmovdqu 32(%rdi), %xmm0 +; AVX2-NEXT: vmovdqu (%rdi), %ymm1 +; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15] ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6],xmm3[7] ; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11] -; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15] +; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15] ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7] ; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13] -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15] +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15] Index: test/CodeGen/X86/pr33784-vector-shuffle.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/pr33784-vector-shuffle.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py + +; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=knl | FileCheck --check-prefixes=ALL,KNL %s +; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=skx | FileCheck --check-prefixes=ALL,SKX %s + +define <16 x i8> @foo(<64 x i8> %x) { +; KNL-LABEL: foo: +; KNL: # BB#0: +; KNL-NEXT: vextracti128 $1, %ymm1, %xmm2 +; KNL-NEXT: vmovdqa {{.*#+}} xmm3 = +; KNL-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; KNL-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; KNL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm2 +; KNL-NEXT: vmovdqa {{.*#+}} xmm3 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u> +; KNL-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; KNL-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; KNL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; KNL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] +; KNL-NEXT: retq +; +; SKX-LABEL: foo: +; SKX: # BB#0: +; SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1 +; SKX-NEXT: vextracti128 $1, %ymm1, %xmm2 +; SKX-NEXT: vmovdqu {{.*#+}} xmm3 = +; SKX-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; SKX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; SKX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm2 +; SKX-NEXT: vmovdqu {{.*#+}} xmm3 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u> +; SKX-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; SKX-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; SKX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SKX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] +; SKX-NEXT: vzeroupper +; SKX-NEXT: retq + %res = shufflevector <64 x i8> %x, <64 x i8> undef, <16 x i32> + ret <16 x i8> %res +} Index: test/CodeGen/X86/vector-shuffle-512-v16.ll =================================================================== --- test/CodeGen/X86/vector-shuffle-512-v16.ll +++ test/CodeGen/X86/vector-shuffle-512-v16.ll @@ -295,13 +295,10 @@ define <4 x i32> @test_v16i32_0_1_2_12 (<16 x i32> %v) { ; ALL-LABEL: test_v16i32_0_1_2_12: ; ALL: # BB#0: -; ALL-NEXT: vpextrd $1, %xmm0, %eax -; ALL-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1 -; ALL-NEXT: vpextrd $2, %xmm0, %eax -; ALL-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 -; ALL-NEXT: vextracti32x4 $3, %zmm0, %xmm0 -; ALL-NEXT: vmovd %xmm0, %eax -; ALL-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 +; ALL-NEXT: vextracti32x8 $1, %zmm0, %ymm1 +; ALL-NEXT: vextracti128 $1, %ymm1, %xmm1 +; ALL-NEXT: vpbroadcastd %xmm1, %xmm1 +; ALL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] ; ALL-NEXT: vzeroupper ; ALL-NEXT: retq %res = shufflevector <16 x i32> %v, <16 x i32> undef, <4 x i32> Index: test/CodeGen/X86/vector-shuffle-512-v32.ll =================================================================== --- test/CodeGen/X86/vector-shuffle-512-v32.ll +++ test/CodeGen/X86/vector-shuffle-512-v32.ll @@ -351,24 +351,10 @@ ; ; SKX-LABEL: pr32967: ; SKX: ## BB#0: -; SKX-NEXT: vpextrw $5, %xmm0, %eax -; SKX-NEXT: vpextrw $1, %xmm0, %ecx -; SKX-NEXT: vmovd %ecx, %xmm1 -; SKX-NEXT: vpinsrw $1, %eax, %xmm1, %xmm1 -; SKX-NEXT: vextracti32x4 $1, %zmm0, %xmm2 -; SKX-NEXT: vpextrw $1, %xmm2, %eax -; SKX-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1 -; SKX-NEXT: vpextrw $5, %xmm2, %eax -; SKX-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1 -; SKX-NEXT: vextracti32x4 $2, %zmm0, %xmm2 -; SKX-NEXT: vpextrw $1, %xmm2, %eax -; SKX-NEXT: vpinsrw $4, %eax, %xmm1, %xmm1 -; SKX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm2[5],xmm1[6,7] -; SKX-NEXT: vextracti32x4 $3, %zmm0, %xmm0 -; SKX-NEXT: vpextrw $1, %xmm0, %eax -; SKX-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1 -; SKX-NEXT: vpextrw $5, %xmm0, %eax -; SKX-NEXT: vpinsrw $7, %eax, %xmm1, %xmm0 +; SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm2 +; SKX-NEXT: vmovdqu {{.*#+}} ymm1 = <1,5,9,13,17,21,25,29,u,u,u,u,u,u,u,u> +; SKX-NEXT: vpermi2w %ymm2, %ymm0, %ymm1 +; SKX-NEXT: vmovdqa %xmm1, %xmm0 ; SKX-NEXT: vzeroupper ; SKX-NEXT: retq %shuffle = shufflevector <32 x i16> %v, <32 x i16> undef, <8 x i32> Index: test/CodeGen/X86/vector-shuffle-512-v8.ll =================================================================== --- test/CodeGen/X86/vector-shuffle-512-v8.ll +++ test/CodeGen/X86/vector-shuffle-512-v8.ll @@ -2726,22 +2726,17 @@ define <2 x i64> @test_v8i64_2_5 (<8 x i64> %v) { ; AVX512F-LABEL: test_v8i64_2_5: ; AVX512F: # BB#0: -; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1 -; AVX512F-NEXT: vextracti32x4 $1, %zmm0, %xmm0 +; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1 +; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0 ; AVX512F-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; ; AVX512F-32-LABEL: test_v8i64_2_5: ; AVX512F-32: # BB#0: -; AVX512F-32-NEXT: vextracti32x4 $1, %zmm0, %xmm1 -; AVX512F-32-NEXT: vpextrd $1, %xmm1, %eax -; AVX512F-32-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 -; AVX512F-32-NEXT: vextracti32x4 $2, %zmm0, %xmm0 -; AVX512F-32-NEXT: vpextrd $2, %xmm0, %eax -; AVX512F-32-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 -; AVX512F-32-NEXT: vpextrd $3, %xmm0, %eax -; AVX512F-32-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 +; AVX512F-32-NEXT: vextracti64x4 $1, %zmm0, %ymm1 +; AVX512F-32-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX512F-32-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] ; AVX512F-32-NEXT: vzeroupper ; AVX512F-32-NEXT: retl %res = shufflevector <8 x i64> %v, <8 x i64> undef, <2 x i32>