Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -6690,6 +6690,40 @@ return false; } +// check if the current node of build vector is a zero extended vedtor. +// if so, return the value extended. +static SDValue isZeroExtendedVector(BuildVectorSDNode *Op, unsigned *NumElt, + MVT *EltType) { + SDValue ExtValue = Op->getOperand(0); + unsigned Delta = Op->getNumOperands(); + + for (unsigned i = 1; i < Op->getNumOperands(); i++) { + if (Op->getOperand(i) == ExtValue) { + Delta = i; + break; + } + if (!(Op->getOperand(i).isUndef() || isNullConstant(Op->getOperand(i)))) + return SDValue(); + } + if (!isPowerOf2_32(Delta) || Delta == 1) + return SDValue(); + + for (unsigned i = Delta; i < Op->getNumOperands(); i++) { + if (i % Delta == 0) { + if (Op->getOperand(i) != ExtValue) + return SDValue(); + } else if (!(isNullConstant(Op->getOperand(i)) || + Op->getOperand(i).isUndef())) + return SDValue(); + } + unsigned EltSize = + Op->getSimpleValueType(0).getVectorElementType().getSizeInBits(); + unsigned ExtVTSize = EltSize * Delta; + *EltType = MVT::getIntegerVT(ExtVTSize); + *NumElt = Op->getNumOperands() / Delta; + return ExtValue; +} + /// Attempt to use the vbroadcast instruction to generate a splat value /// from a splat BUILD_VECTOR which uses: /// a. A single scalar load, or a constant. @@ -6712,6 +6746,25 @@ assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && "Unsupported vector type for broadcast."); + // checking for broadcastm pattern + if (Subtarget.hasCDI() && (VT.getSizeInBits() == 512 || Subtarget.hasVLX())) { + MVT EltType; + unsigned NumElts = 1; + SDValue ZeroExtended = isZeroExtendedVector(BVOp, &NumElts, &EltType); + if (ZeroExtended && ZeroExtended.getOpcode() == ISD::BITCAST) { + SDValue BOperand = ZeroExtended.getOperand(0); + if (BOperand.getSimpleValueType().getVectorElementType() == MVT::i1) { + if ((EltType == MVT::i64 && VT.getVectorElementType() == MVT::i8) || //for broadcastmb2q + (EltType == MVT::i32 && VT.getVectorElementType() == MVT::i16)) { //for broadcastmw2d + SDValue Brdcst = + DAG.getNode(X86ISD::VBROADCASTM, dl, + MVT::getVectorVT(EltType, NumElts), BOperand); + return DAG.getBitcast(VT, Brdcst); + } + } + } + } + BitVector UndefElements; SDValue Ld = BVOp->getSplatValue(&UndefElements); Index: test/CodeGen/X86/broadcastm-lowering.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/broadcastm-lowering.ll @@ -0,0 +1,164 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefix=ALL --check-prefix=SKX +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefix=ALL --check-prefix=KNL + +define <2 x i64> @test_mm_epi64(<8 x i16> %a, <8 x i16> %b) { +; SKX-LABEL: test_mm_epi64: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpcmpeqw %xmm1, %xmm0, %k0 +; SKX-NEXT: vpbroadcastmb2q %k0, %xmm0 +; SKX-NEXT: retq +; +; KNL-LABEL: test_mm_epi64: +; KNL: # BB#0: # %entry +; KNL-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0 +; KNL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrb $8, %eax, %xmm0, %xmm0 +; KNL-NEXT: retq +entry: + %0 = icmp eq <8 x i16> %a, %b + %1 = bitcast <8 x i1> %0 to i8 + %conv.i = zext i8 %1 to i64 + %vecinit.i.i = insertelement <2 x i64> undef, i64 %conv.i, i32 0 + %vecinit1.i.i = shufflevector <2 x i64> %vecinit.i.i, <2 x i64> undef, <2 x i32> zeroinitializer + ret <2 x i64> %vecinit1.i.i +} + +define <4 x i32> @test_mm_epi32(<16 x i8> %a, <16 x i8> %b) { +; SKX-LABEL: test_mm_epi32: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpcmpeqb %xmm1, %xmm0, %k0 +; SKX-NEXT: vpbroadcastmw2d %k0, %xmm0 +; SKX-NEXT: retq +; +; KNL-LABEL: test_mm_epi32: +; KNL: # BB#0: # %entry +; KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $4, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0 +; KNL-NEXT: retq +entry: + %0 = icmp eq <16 x i8> %a, %b + %1 = bitcast <16 x i1> %0 to i16 + %conv.i = zext i16 %1 to i32 + %vecinit.i.i = insertelement <4 x i32> undef, i32 %conv.i, i32 0 + %vecinit3.i.i = shufflevector <4 x i32> %vecinit.i.i, <4 x i32> undef, <4 x i32> zeroinitializer + ret <4 x i32> %vecinit3.i.i +} + +define <16 x i32> @test_mm512_epi32(<16 x i32> %a, <16 x i32> %b) { +; SKX-LABEL: test_mm512_epi32: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 +; SKX-NEXT: vpbroadcastmw2d %k0, %zmm0 +; SKX-NEXT: retq +; +; KNL-LABEL: test_mm512_epi32: +; KNL: # BB#0: # %entry +; KNL-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: vpbroadcastd %eax, %zmm0 +; KNL-NEXT: retq +entry: + %0 = icmp eq <16 x i32> %a, %b + %1 = bitcast <16 x i1> %0 to i16 + %conv.i = zext i16 %1 to i32 + %vecinit.i.i = insertelement <16 x i32> undef, i32 %conv.i, i32 0 + %vecinit15.i.i = shufflevector <16 x i32> %vecinit.i.i, <16 x i32> undef, <16 x i32> zeroinitializer + ret <16 x i32> %vecinit15.i.i +} + +define <8 x i64> @test_mm512_epi64(<8 x i32> %a, <8 x i32> %b) { +; SKX-LABEL: test_mm512_epi64: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpcmpeqd %ymm1, %ymm0, %k0 +; SKX-NEXT: vpbroadcastmb2q %k0, %zmm0 +; SKX-NEXT: retq +; +; KNL-LABEL: test_mm512_epi64: +; KNL: # BB#0: # %entry +; KNL-NEXT: # kill: %YMM1 %YMM1 %ZMM1 +; KNL-NEXT: # kill: %YMM0 %YMM0 %ZMM0 +; KNL-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpbroadcastq %rax, %zmm0 +; KNL-NEXT: retq +entry: + %0 = icmp eq <8 x i32> %a, %b + %1 = bitcast <8 x i1> %0 to i8 + %conv.i = zext i8 %1 to i64 + %vecinit.i.i = insertelement <8 x i64> undef, i64 %conv.i, i32 0 + %vecinit7.i.i = shufflevector <8 x i64> %vecinit.i.i, <8 x i64> undef, <8 x i32> zeroinitializer + ret <8 x i64> %vecinit7.i.i +} + +define <4 x i64> @test_mm256_epi64(<8 x i32> %a, <8 x i32> %b) { +; SKX-LABEL: test_mm256_epi64: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpcmpeqd %ymm1, %ymm0, %k0 +; SKX-NEXT: vpbroadcastmb2q %k0, %ymm0 +; SKX-NEXT: retq +; +; KNL-LABEL: test_mm256_epi64: +; KNL: # BB#0: # %entry +; KNL-NEXT: # kill: %YMM1 %YMM1 %ZMM1 +; KNL-NEXT: # kill: %YMM0 %YMM0 %ZMM0 +; KNL-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0 +; KNL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrb $8, %eax, %xmm0, %xmm0 +; KNL-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; KNL-NEXT: retq +entry: + %0 = icmp eq <8 x i32> %a, %b + %1 = bitcast <8 x i1> %0 to i8 + %conv.i = zext i8 %1 to i64 + %vecinit.i.i = insertelement <4 x i64> undef, i64 %conv.i, i32 0 + %vecinit3.i.i = shufflevector <4 x i64> %vecinit.i.i, <4 x i64> undef, <4 x i32> zeroinitializer + ret <4 x i64> %vecinit3.i.i +} + +define <8 x i32> @test_mm256_epi32(<16 x i16> %a, <16 x i16> %b) { +; SKX-LABEL: test_mm256_epi32: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpcmpeqw %ymm1, %ymm0, %k0 +; SKX-NEXT: vpbroadcastmw2d %k0, %ymm0 +; SKX-NEXT: retq +; +; KNL-LABEL: test_mm256_epi32: +; KNL: # BB#0: # %entry +; KNL-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0 +; KNL-NEXT: vpmovsxwd %ymm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $4, %eax, %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0 +; KNL-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; KNL-NEXT: retq +entry: + %0 = icmp eq <16 x i16> %a, %b + %1 = bitcast <16 x i1> %0 to i16 + %conv.i = zext i16 %1 to i32 + %vecinit.i.i = insertelement <8 x i32> undef, i32 %conv.i, i32 0 + %vecinit7.i.i = shufflevector <8 x i32> %vecinit.i.i, <8 x i32> undef, <8 x i32> zeroinitializer + ret <8 x i32> %vecinit7.i.i +} +