Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -9232,11 +9232,88 @@ return SDValue(); } +static bool IsVUZPShuffleNode(SDNode *N) { + // VUZP shuffle node. + if (N->getOpcode() == ARMISD::VUZP) + return true; + + // "VUZP" on i32 is an alias for VTRN. + if (N->getOpcode() == ARMISD::VTRN && N->getValueType(0) == MVT::v2i32) + return true; + + return false; +} + +// AddCombineToVPADD - For pair-wise add on neon, use the vpadd instruction. +static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, + TargetLowering::DAGCombinerInfo &DCI, + const ARMSubtarget *Subtarget) { + if (IsVUZPShuffleNode(N0.getNode()) && N0.getNode() == N1.getNode() && + N0 != N1 && N->getValueType(0).is64BitVector()) { + // An add where the inputs both come from the same vuzp forms a vpadd. + SelectionDAG &DAG = DCI.DAG; + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDLoc dl(N); + SDNode *Unzip = N0.getNode(); + EVT VT = N->getValueType(0); + + SmallVector Ops; + Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpadd, dl, + TLI.getPointerTy(DAG.getDataLayout()))); + Ops.push_back(Unzip->getOperand(0)); + Ops.push_back(Unzip->getOperand(1)); + + return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); + } + + return SDValue(); +} + // AddCombineToVPADDL- For pair-wise add on neon, use the vpaddl instruction // (only after legalization). static SDValue AddCombineToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget) { + if ((N0.getOpcode() == ISD::SIGN_EXTEND && + N1.getOpcode() == ISD::SIGN_EXTEND) || + (N0.getOpcode() == ISD::ZERO_EXTEND && + N1.getOpcode() == ISD::ZERO_EXTEND)) { + SDValue N00 = N0.getOperand(0); + SDValue N10 = N1.getOperand(0); + if (IsVUZPShuffleNode(N00.getNode()) && N00.getNode() == N10.getNode() && + N00 != N10 && N00.getValueType().is64BitVector() && + N0.getValueType().is128BitVector()) { + // An extended add where the inputs both come from the same vuzp forms + // a vpaddl. + SelectionDAG &DAG = DCI.DAG; + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDLoc dl(N); + EVT VT = N->getValueType(0); + + SmallVector Ops; + // Form vpaddl.sN or vpaddl.uN depending on the kind of extension. + unsigned Opcode; + if (N0.getOpcode() == ISD::SIGN_EXTEND) + Opcode = Intrinsic::arm_neon_vpaddls; + else + Opcode = Intrinsic::arm_neon_vpaddlu; + Ops.push_back(DAG.getConstant(Opcode, dl, + TLI.getPointerTy(DAG.getDataLayout()))); + EVT ElemTy = N00.getValueType().getVectorElementType(); + unsigned NumElts = VT.getVectorNumElements(); + EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), ElemTy, NumElts * 2); + SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT, + N00, N10); + Ops.push_back(Concat); + + return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); + + } + } + + // FIXME: We shouldn't produce a DAG which matches the following pattern. + // There are more efficient ways to lower this sort of shuffle in the + // general case. // Only perform optimization if after legalize, and if NEON is available. We // also expected both operands to be BUILD_VECTORs. @@ -9564,6 +9641,9 @@ static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget){ + // Attempt to create vpadd for this add. + if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget)) + return Result; // Attempt to create vpaddl for this add. if (SDValue Result = AddCombineToVPADDL(N, N0, N1, DCI, Subtarget)) Index: test/CodeGen/ARM/vpadd.ll =================================================================== --- test/CodeGen/ARM/vpadd.ll +++ test/CodeGen/ARM/vpadd.ll @@ -214,14 +214,11 @@ } ; Combine vuzp+vadd->vpadd. -; FIXME: Implement this optimization -define void @addCombineToVPADD(<16 x i8> *%cbcr, <8 x i8> *%X) nounwind ssp { -; CHECK-LABEL: addCombineToVPADD: +define void @addCombineToVPADD_i8(<16 x i8> *%cbcr, <8 x i8> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADD_i8: ; CHECK: @ BB#0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r0] -; CHECK-NEXT: vorr d18, d17, d17 -; CHECK-NEXT: vuzp.8 d16, d18 -; CHECK-NEXT: vadd.i8 d16, d18, d16 +; CHECK-NEXT: vpadd.i8 d16, d16, d17 ; CHECK-NEXT: vstr d16, [r1] ; CHECK-NEXT: mov pc, lr %tmp = load <16 x i8>, <16 x i8>* %cbcr @@ -233,15 +230,46 @@ ret void } -; Combine vuzp+vaddl->vpaddl -; FIXME: Implement this optimization. -define void @addCombineToVPADDL_sext(<16 x i8> *%cbcr, <8 x i16> *%X) nounwind ssp { -; CHECK-LABEL: addCombineToVPADDL_sext: +; Combine vuzp+vadd->vpadd. +define void @addCombineToVPADD_i16(<8 x i16> *%cbcr, <4 x i16> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADD_i16: ; CHECK: @ BB#0: ; CHECK-NEXT: vld1.64 {d16, d17}, [r0] -; CHECK-NEXT: vorr d18, d17, d17 -; CHECK-NEXT: vuzp.8 d16, d18 -; CHECK-NEXT: vaddl.s8 q8, d18, d16 +; CHECK-NEXT: vpadd.i16 d16, d16, d17 +; CHECK-NEXT: vstr d16, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <8 x i16>, <8 x i16>* %cbcr + %tmp1 = shufflevector <8 x i16> %tmp, <8 x i16> undef, <4 x i32> + %tmp3 = shufflevector <8 x i16> %tmp, <8 x i16> undef, <4 x i32> + %add = add <4 x i16> %tmp3, %tmp1 + store <4 x i16> %add, <4 x i16>* %X, align 8 + ret void +} + +; Combine vtrn+vadd->vpadd. +define void @addCombineToVPADD_i32(<4 x i32> *%cbcr, <2 x i32> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADD_i32: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r0] +; CHECK-NEXT: vpadd.i32 d16, d16, d17 +; CHECK-NEXT: vstr d16, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <4 x i32>, <4 x i32>* %cbcr + %tmp1 = shufflevector <4 x i32> %tmp, <4 x i32> undef, <2 x i32> + %tmp3 = shufflevector <4 x i32> %tmp, <4 x i32> undef, <2 x i32> + %add = add <2 x i32> %tmp3, %tmp1 + store <2 x i32> %add, <2 x i32>* %X, align 8 + ret void +} + +; Combine vuzp+vaddl->vpaddl +define void @addCombineToVPADDLq_s8(<16 x i8> *%cbcr, <8 x i16> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDLq_s8: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d1, d2}, [r0] +; CHECK-NEXT: vorr d0, d2, d2 +; CHECK-NEXT: vuzp.8 d1, d0 +; CHECK-NEXT: vpaddl.s8 q8, q0 ; CHECK-NEXT: vst1.64 {d16, d17}, [r1] ; CHECK-NEXT: mov pc, lr %tmp = load <16 x i8>, <16 x i8>* %cbcr @@ -254,10 +282,192 @@ ret void } -; Legalization produces a EXTRACT_VECTOR_ELT DAG node which performs an extend from -; i16 to i32. In this case the input for the formed VPADDL needs to be a vector of i16s. -define <2 x i16> @fromExtendingExtractVectorElt(<4 x i16> %in) { -; CHECK-LABEL: fromExtendingExtractVectorElt: +; Combine vuzp+vaddl->vpaddl +; FIXME: Legalization butchers the shuffles. +define void @addCombineToVPADDL_s8(<16 x i8> *%cbcr, <4 x i16> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDL_s8: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r0] +; CHECK-NEXT: vmov.i16 d18, #0x8 +; CHECK-NEXT: vneg.s16 d18, d18 +; CHECK-NEXT: vext.8 d19, d16, d16, #1 +; CHECK-NEXT: vshl.i16 d16, d16, #8 +; CHECK-NEXT: vshl.i16 d17, d19, #8 +; CHECK-NEXT: vshl.s16 d16, d16, d18 +; CHECK-NEXT: vshl.s16 d17, d17, d18 +; CHECK-NEXT: vadd.i16 d16, d17, d16 +; CHECK-NEXT: vstr d16, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <16 x i8>, <16 x i8>* %cbcr + %tmp1 = shufflevector <16 x i8> %tmp, <16 x i8> undef, <4 x i32> + %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> undef, <4 x i32> + %tmp4 = sext <4 x i8> %tmp3 to <4 x i16> + %tmp5 = sext <4 x i8> %tmp1 to <4 x i16> + %add = add <4 x i16> %tmp4, %tmp5 + store <4 x i16> %add, <4 x i16>* %X, align 8 + ret void +} + +; Combine vuzp+vaddl->vpaddl +define void @addCombineToVPADDLq_u8(<16 x i8> *%cbcr, <8 x i16> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDLq_u8: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d1, d2}, [r0] +; CHECK-NEXT: vorr d0, d2, d2 +; CHECK-NEXT: vuzp.8 d1, d0 +; CHECK-NEXT: vpaddl.u8 q8, q0 +; CHECK-NEXT: vst1.64 {d16, d17}, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <16 x i8>, <16 x i8>* %cbcr + %tmp1 = shufflevector <16 x i8> %tmp, <16 x i8> undef, <8 x i32> + %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> undef, <8 x i32> + %tmp4 = zext <8 x i8> %tmp3 to <8 x i16> + %tmp5 = zext <8 x i8> %tmp1 to <8 x i16> + %add = add <8 x i16> %tmp4, %tmp5 + store <8 x i16> %add, <8 x i16>* %X, align 8 + ret void +} + +; In theory, it's possible to match this to vpaddl, but rearranging the +; shuffle is awkward, so this doesn't match at the moment. +define void @addCombineToVPADDLq_u8_early_zext(<16 x i8> *%cbcr, <8 x i16> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDLq_u8_early_zext: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r0] +; CHECK-NEXT: vmovl.u8 q9, d17 +; CHECK-NEXT: vmovl.u8 q8, d16 +; CHECK-NEXT: vuzp.16 q8, q9 +; CHECK-NEXT: vadd.i16 q8, q8, q9 +; CHECK-NEXT: vst1.64 {d16, d17}, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <16 x i8>, <16 x i8>* %cbcr + %tmp1 = zext <16 x i8> %tmp to <16 x i16> + %tmp2 = shufflevector <16 x i16> %tmp1, <16 x i16> undef, <8 x i32> + %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> undef, <8 x i32> + %add = add <8 x i16> %tmp2, %tmp3 + store <8 x i16> %add, <8 x i16>* %X, align 8 + ret void +} + +; Combine vuzp+vaddl->vpaddl +; FIXME: Legalization butchers the shuffle. +define void @addCombineToVPADDL_u8(<16 x i8> *%cbcr, <4 x i16> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDL_u8: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d16, d17}, [r0] +; CHECK-NEXT: vext.8 d18, d16, d16, #1 +; CHECK-NEXT: vbic.i16 d16, #0xff00 +; CHECK-NEXT: vbic.i16 d18, #0xff00 +; CHECK-NEXT: vadd.i16 d16, d18, d16 +; CHECK-NEXT: vstr d16, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <16 x i8>, <16 x i8>* %cbcr + %tmp1 = shufflevector <16 x i8> %tmp, <16 x i8> undef, <4 x i32> + %tmp3 = shufflevector <16 x i8> %tmp, <16 x i8> undef, <4 x i32> + %tmp4 = zext <4 x i8> %tmp3 to <4 x i16> + %tmp5 = zext <4 x i8> %tmp1 to <4 x i16> + %add = add <4 x i16> %tmp4, %tmp5 + store <4 x i16> %add, <4 x i16>* %X, align 8 + ret void +} + +; Combine vuzp+vaddl->vpaddl +define void @addCombineToVPADDLq_s16(<8 x i16> *%cbcr, <4 x i32> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDLq_s16: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d1, d2}, [r0] +; CHECK-NEXT: vorr d0, d2, d2 +; CHECK-NEXT: vuzp.16 d1, d0 +; CHECK-NEXT: vpaddl.s16 q8, q0 +; CHECK-NEXT: vst1.64 {d16, d17}, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <8 x i16>, <8 x i16>* %cbcr + %tmp1 = shufflevector <8 x i16> %tmp, <8 x i16> undef, <4 x i32> + %tmp3 = shufflevector <8 x i16> %tmp, <8 x i16> undef, <4 x i32> + %tmp4 = sext <4 x i16> %tmp3 to <4 x i32> + %tmp5 = sext <4 x i16> %tmp1 to <4 x i32> + %add = add <4 x i32> %tmp4, %tmp5 + store <4 x i32> %add, <4 x i32>* %X, align 8 + ret void +} + +; Combine vuzp+vaddl->vpaddl +define void @addCombineToVPADDLq_u16(<8 x i16> *%cbcr, <4 x i32> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDLq_u16: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d1, d2}, [r0] +; CHECK-NEXT: vorr d0, d2, d2 +; CHECK-NEXT: vuzp.16 d1, d0 +; CHECK-NEXT: vpaddl.u16 q8, q0 +; CHECK-NEXT: vst1.64 {d16, d17}, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <8 x i16>, <8 x i16>* %cbcr + %tmp1 = shufflevector <8 x i16> %tmp, <8 x i16> undef, <4 x i32> + %tmp3 = shufflevector <8 x i16> %tmp, <8 x i16> undef, <4 x i32> + %tmp4 = zext <4 x i16> %tmp3 to <4 x i32> + %tmp5 = zext <4 x i16> %tmp1 to <4 x i32> + %add = add <4 x i32> %tmp4, %tmp5 + store <4 x i32> %add, <4 x i32>* %X, align 8 + ret void +} + +; Combine vtrn+vaddl->vpaddl +define void @addCombineToVPADDLq_s32(<4 x i32> *%cbcr, <2 x i64> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDLq_s32: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d1, d2}, [r0] +; CHECK-NEXT: vorr d0, d2, d2 +; CHECK-NEXT: vtrn.32 d1, d0 +; CHECK-NEXT: vpaddl.s32 q8, q0 +; CHECK-NEXT: vst1.64 {d16, d17}, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <4 x i32>, <4 x i32>* %cbcr + %tmp1 = shufflevector <4 x i32> %tmp, <4 x i32> undef, <2 x i32> + %tmp3 = shufflevector <4 x i32> %tmp, <4 x i32> undef, <2 x i32> + %tmp4 = sext <2 x i32> %tmp3 to <2 x i64> + %tmp5 = sext <2 x i32> %tmp1 to <2 x i64> + %add = add <2 x i64> %tmp4, %tmp5 + store <2 x i64> %add, <2 x i64>* %X, align 8 + ret void +} + +; Combine vtrn+vaddl->vpaddl +define void @addCombineToVPADDLq_u32(<4 x i32> *%cbcr, <2 x i64> *%X) nounwind ssp { +; CHECK-LABEL: addCombineToVPADDLq_u32: +; CHECK: @ BB#0: +; CHECK-NEXT: vld1.64 {d1, d2}, [r0] +; CHECK-NEXT: vorr d0, d2, d2 +; CHECK-NEXT: vtrn.32 d1, d0 +; CHECK-NEXT: vpaddl.u32 q8, q0 +; CHECK-NEXT: vst1.64 {d16, d17}, [r1] +; CHECK-NEXT: mov pc, lr + %tmp = load <4 x i32>, <4 x i32>* %cbcr + %tmp1 = shufflevector <4 x i32> %tmp, <4 x i32> undef, <2 x i32> + %tmp3 = shufflevector <4 x i32> %tmp, <4 x i32> undef, <2 x i32> + %tmp4 = zext <2 x i32> %tmp3 to <2 x i64> + %tmp5 = zext <2 x i32> %tmp1 to <2 x i64> + %add = add <2 x i64> %tmp4, %tmp5 + store <2 x i64> %add, <2 x i64>* %X, align 8 + ret void +} + +; Legalization promotes the <4 x i8> to <4 x i16>. +define <4 x i8> @fromExtendingExtractVectorElt_i8(<8 x i8> %in) { +; CHECK-LABEL: fromExtendingExtractVectorElt_i8: +; CHECK: @ BB#0: +; CHECK-NEXT: vmov d16, r0, r1 +; CHECK-NEXT: vpaddl.s8 d16, d16 +; CHECK-NEXT: vmov r0, r1, d16 +; CHECK-NEXT: mov pc, lr + %tmp1 = shufflevector <8 x i8> %in, <8 x i8> undef, <4 x i32> + %tmp2 = shufflevector <8 x i8> %in, <8 x i8> undef, <4 x i32> + %x = add <4 x i8> %tmp2, %tmp1 + ret <4 x i8> %x +} + +; Legalization promotes the <2 x i16> to <2 x i32>. +define <2 x i16> @fromExtendingExtractVectorElt_i16(<4 x i16> %in) { +; CHECK-LABEL: fromExtendingExtractVectorElt_i16: ; CHECK: @ BB#0: ; CHECK-NEXT: vmov d16, r0, r1 ; CHECK-NEXT: vpaddl.s16 d16, d16 Index: test/CodeGen/ARM/vtrn.ll =================================================================== --- test/CodeGen/ARM/vtrn.ll +++ test/CodeGen/ARM/vtrn.ll @@ -70,14 +70,14 @@ ; CHECK-NEXT: vldr d16, [r1] ; CHECK-NEXT: vldr d17, [r0] ; CHECK-NEXT: vtrn.32 d17, d16 -; CHECK-NEXT: vadd.i32 d16, d17, d16 +; CHECK-NEXT: vmul.i32 d16, d17, d16 ; CHECK-NEXT: vmov r0, r1, d16 ; CHECK-NEXT: mov pc, lr %tmp1 = load <2 x i32>, <2 x i32>* %A %tmp2 = load <2 x i32>, <2 x i32>* %B %tmp3 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp2, <2 x i32> %tmp4 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp2, <2 x i32> - %tmp5 = add <2 x i32> %tmp3, %tmp4 + %tmp5 = mul <2 x i32> %tmp3, %tmp4 ret <2 x i32> %tmp5 }