Index: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -415,6 +415,7 @@ SDValue CombineConsecutiveLoads(SDNode *N, EVT VT); SDValue CombineExtLoad(SDNode *N); SDValue combineRepeatedFPDivisors(SDNode *N); + SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex); SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT); SDValue BuildSDIV(SDNode *N); SDValue BuildSDIVPow2(SDNode *N); @@ -13747,6 +13748,60 @@ return St1; } +/// Convert a disguised subvector insertion into a shuffle: +/// insert_vector_elt V, (bitcast X from vector type), IdxC --> +/// bitcast(shuffle (bitcast V), (extended X), Mask) +/// Note: We do not use an insert_subvector node because that requires a legal +/// subvector type. +SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) { + SDValue InsertVal = N->getOperand(1); + if (InsertVal.getOpcode() != ISD::BITCAST || !InsertVal.hasOneUse() || + !InsertVal.getOperand(0).getValueType().isVector()) + return SDValue(); + + SDValue SubVec = InsertVal.getOperand(0); + SDValue DestVec = N->getOperand(0); + EVT SubVecVT = SubVec.getValueType(); + EVT VT = DestVec.getValueType(); + unsigned NumSrcElts = SubVecVT.getVectorNumElements(); + unsigned ExtendRatio = VT.getSizeInBits() / SubVecVT.getSizeInBits(); + unsigned NumMaskVals = ExtendRatio * NumSrcElts; + + // Step 1: Create a shuffle mask that implements this insert operation. The + // vector that we are inserting into will be operand 0 of the shuffle, so + // those elements are just 'i'. The inserted subvector is in the first + // positions of operand 1 of the shuffle. Example: + // insert v4i32 V, (v2i16 X), 2 --> shuffle v8i16 V', X', {0,1,2,3,8,9,6,7} + SmallVector Mask(NumMaskVals); + for (unsigned i = 0; i != NumMaskVals; ++i) { + if (i / NumSrcElts == InsIndex) + Mask[i] = (i % NumSrcElts) + NumMaskVals; + else + Mask[i] = i; + } + + // Bail out if the target can not handle the shuffle we want to create. + EVT SubVecEltVT = SubVecVT.getVectorElementType(); + EVT ShufVT = EVT::getVectorVT(*DAG.getContext(), SubVecEltVT, NumMaskVals); + if (!TLI.isShuffleMaskLegal(Mask, ShufVT)) + return SDValue(); + + // Step 2: Create a wide vector from the inserted source vector by appending + // undefined elements. This is the same size as our destination vector. + SDLoc DL(N); + SmallVector ConcatOps(ExtendRatio, DAG.getUNDEF(SubVecVT)); + ConcatOps[0] = SubVec; + SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps); + + // Step 3: Shuffle in the padded subvector. + SDValue DestVecBC = DAG.getBitcast(ShufVT, DestVec); + SDValue Shuf = DAG.getVectorShuffle(ShufVT, DL, DestVecBC, PaddedSubV, Mask); + AddToWorklist(PaddedSubV.getNode()); + AddToWorklist(DestVecBC.getNode()); + AddToWorklist(Shuf.getNode()); + return DAG.getBitcast(VT, Shuf); +} + SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { SDValue InVec = N->getOperand(0); SDValue InVal = N->getOperand(1); @@ -13765,10 +13820,14 @@ InVec == InVal.getOperand(0) && EltNo == InVal.getOperand(1)) return InVec; - // Check that we know which element is being inserted - if (!isa(EltNo)) + // We must know which element is being inserted for folds below here. + auto *IndexC = dyn_cast(EltNo); + if (!IndexC) return SDValue(); - unsigned Elt = cast(EltNo)->getZExtValue(); + unsigned Elt = IndexC->getZExtValue(); + + if (SDValue Shuf = combineInsertEltToShuffle(N, Elt)) + return Shuf; // Canonicalize insert_vector_elt dag nodes. // Example: Index: llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll =================================================================== --- llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll +++ llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll @@ -140,7 +140,7 @@ define <2 x double> @ins1f2(<1 x double> %tmp1, <2 x double> %tmp2) { ; CHECK-LABEL: ins1f2: -; CHECK: ins {{v[0-9]+}}.d[1], {{v[0-9]+}}.d[0] +; CHECK: zip1 {{v[0-9]+}}.2d, {{v[0-9]+}}.2d %tmp3 = extractelement <1 x double> %tmp1, i32 0 %tmp4 = insertelement <2 x double> %tmp2, double %tmp3, i32 1 ret <2 x double> %tmp4 Index: llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll +++ llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll @@ -7,42 +7,34 @@ define <8 x float> @insert_subvector_256(i16 %x0, i16 %x1, <8 x float> %v) nounwind { ; X32_AVX256-LABEL: insert_subvector_256: ; X32_AVX256: # BB#0: -; X32_AVX256-NEXT: pushl %eax ; X32_AVX256-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero ; X32_AVX256-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1 -; X32_AVX256-NEXT: vmovd %xmm1, (%esp) -; X32_AVX256-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3] -; X32_AVX256-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; X32_AVX256-NEXT: popl %eax +; X32_AVX256-NEXT: vpbroadcastd %xmm1, %xmm1 +; X32_AVX256-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7] ; X32_AVX256-NEXT: retl ; ; X64_AVX256-LABEL: insert_subvector_256: ; X64_AVX256: # BB#0: ; X64_AVX256-NEXT: vmovd %edi, %xmm1 ; X64_AVX256-NEXT: vpinsrw $1, %esi, %xmm1, %xmm1 -; X64_AVX256-NEXT: vmovd %xmm1, -{{[0-9]+}}(%rsp) -; X64_AVX256-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3] -; X64_AVX256-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; X64_AVX256-NEXT: vpbroadcastd %xmm1, %xmm1 +; X64_AVX256-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7] ; X64_AVX256-NEXT: retq ; ; X32_AVX512-LABEL: insert_subvector_256: ; X32_AVX512: # BB#0: -; X32_AVX512-NEXT: pushl %eax ; X32_AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero ; X32_AVX512-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1 -; X32_AVX512-NEXT: vmovd %xmm1, (%esp) -; X32_AVX512-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3] -; X32_AVX512-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; X32_AVX512-NEXT: popl %eax +; X32_AVX512-NEXT: vpbroadcastd %xmm1, %xmm1 +; X32_AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7] ; X32_AVX512-NEXT: retl ; ; X64_AVX512-LABEL: insert_subvector_256: ; X64_AVX512: # BB#0: ; X64_AVX512-NEXT: vmovd %edi, %xmm1 ; X64_AVX512-NEXT: vpinsrw $1, %esi, %xmm1, %xmm1 -; X64_AVX512-NEXT: vmovd %xmm1, -{{[0-9]+}}(%rsp) -; X64_AVX512-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3] -; X64_AVX512-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; X64_AVX512-NEXT: vpbroadcastd %xmm1, %xmm1 +; X64_AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7] ; X64_AVX512-NEXT: retq %ins1 = insertelement <2 x i16> undef, i16 %x0, i32 0 %ins2 = insertelement <2 x i16> %ins1, i16 %x1, i32 1 @@ -80,28 +72,17 @@ ; ; X32_AVX512-LABEL: insert_subvector_512: ; X32_AVX512: # BB#0: -; X32_AVX512-NEXT: pushl %ebp -; X32_AVX512-NEXT: movl %esp, %ebp -; X32_AVX512-NEXT: andl $-8, %esp -; X32_AVX512-NEXT: subl $8, %esp -; X32_AVX512-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; X32_AVX512-NEXT: vmovlps %xmm1, (%esp) -; X32_AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 -; X32_AVX512-NEXT: vpinsrd $0, (%esp), %xmm1, %xmm1 -; X32_AVX512-NEXT: vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1 -; X32_AVX512-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0 -; X32_AVX512-NEXT: movl %ebp, %esp -; X32_AVX512-NEXT: popl %ebp +; X32_AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; X32_AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,1,0,8,0,3,0,4,0,5,0,6,0,7,0] +; X32_AVX512-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 ; X32_AVX512-NEXT: retl ; ; X64_AVX512-LABEL: insert_subvector_512: ; X64_AVX512: # BB#0: ; X64_AVX512-NEXT: vmovd %edi, %xmm1 ; X64_AVX512-NEXT: vpinsrd $1, %esi, %xmm1, %xmm1 -; X64_AVX512-NEXT: vmovq %xmm1, %rax -; X64_AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 -; X64_AVX512-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1 -; X64_AVX512-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0 +; X64_AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,1,8,3,4,5,6,7] +; X64_AVX512-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 ; X64_AVX512-NEXT: retq %ins1 = insertelement <2 x i32> undef, i32 %x0, i32 0 %ins2 = insertelement <2 x i32> %ins1, i32 %x1, i32 1 @@ -144,22 +125,8 @@ ; ; X32_AVX512-LABEL: insert_subvector_into_undef: ; X32_AVX512: # BB#0: -; X32_AVX512-NEXT: pushl %ebp -; X32_AVX512-NEXT: movl %esp, %ebp -; X32_AVX512-NEXT: andl $-8, %esp -; X32_AVX512-NEXT: subl $8, %esp ; X32_AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; X32_AVX512-NEXT: vmovlps %xmm0, (%esp) -; X32_AVX512-NEXT: movl (%esp), %eax -; X32_AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32_AVX512-NEXT: vmovd %eax, %xmm0 -; X32_AVX512-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 -; X32_AVX512-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0 -; X32_AVX512-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0 -; X32_AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; X32_AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 -; X32_AVX512-NEXT: movl %ebp, %esp -; X32_AVX512-NEXT: popl %ebp +; X32_AVX512-NEXT: vbroadcastsd %xmm0, %zmm0 ; X32_AVX512-NEXT: retl ; ; X64_AVX512-LABEL: insert_subvector_into_undef: