Index: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp +++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp @@ -1530,6 +1530,13 @@ setOperationAction(ISD::BITCAST, MVT::v32i16, Custom); setOperationAction(ISD::BITCAST, MVT::v64i8, Custom); } + + if (Subtarget.hasVBMI2()) { + for (auto VT : { MVT::v16i32, MVT::v8i64 }) { + setOperationAction(ISD::FSHL, VT, Custom); + setOperationAction(ISD::FSHR, VT, Custom); + } + } }// has AVX-512 // This block controls legalization for operations that don't have @@ -1705,6 +1712,11 @@ for (auto VT : { MVT::v64i8, MVT::v32i16 }) setOperationAction(ISD::CTPOP, VT, Legal); } + + if (Subtarget.hasVBMI2()) { + setOperationAction(ISD::FSHL, MVT::v32i16, Custom); + setOperationAction(ISD::FSHR, MVT::v32i16, Custom); + } } if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) { @@ -1751,6 +1763,15 @@ setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal); setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal); } + + if (Subtarget.hasVBMI2()) { + // TODO: Make these legal even without VLX? + for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64, + MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { + setOperationAction(ISD::FSHL, VT, Custom); + setOperationAction(ISD::FSHR, VT, Custom); + } + } } // We want to custom lower some of our intrinsics. @@ -17101,20 +17122,39 @@ MVT VT = Op.getSimpleValueType(); assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) && "Unexpected funnel shift opcode!"); - assert((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && - "Unexpected funnel shift type!"); SDLoc DL(Op); SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); SDValue Amt = Op.getOperand(2); + bool IsFSHR = Op.getOpcode() == ISD::FSHR; + + if (VT.isVector()) { + assert(Subtarget.hasVBMI2() && "Expected VBMI2"); + + if (IsFSHR) + std::swap(Op0, Op1); + + APInt APIntShiftAmt; + if (isConstantSplat(Amt, APIntShiftAmt)) { + uint64_t ShiftAmt = APIntShiftAmt.getZExtValue(); + return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, + Op0, Op1, DAG.getConstant(ShiftAmt, DL, MVT::i8)); + } + + return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT, + Op0, Op1, Amt); + } + + assert((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && + "Unexpected funnel shift type!"); + // Expand slow SHLD/SHRD cases if we are not optimizing for size. bool OptForSize = DAG.getMachineFunction().getFunction().optForSize(); if (!OptForSize && Subtarget.isSHLDSlow()) return SDValue(); - bool IsFSHR = Op.getOpcode() == ISD::FSHR; if (IsFSHR) std::swap(Op0, Op1); Index: llvm/trunk/test/CodeGen/X86/vector-fshl-128.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/vector-fshl-128.ll +++ llvm/trunk/test/CodeGen/X86/vector-fshl-128.ll @@ -6,7 +6,9 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2 @@ -152,6 +154,23 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [63,63] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsllvq %xmm4, %xmm0, %xmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VBMI2-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm5, %xmm1 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v2i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm3 = [63,63] @@ -166,6 +185,11 @@ ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvq %xmm2, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v2i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -380,6 +404,23 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsllvd %xmm4, %xmm0, %xmm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm5, %xmm1 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v4i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31] @@ -394,6 +435,11 @@ ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvd %xmm2, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v4i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -689,6 +735,24 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm5, %xmm1 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v8i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] @@ -703,6 +767,11 @@ ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvw %xmm2, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: var_funnnel_v8i16: ; XOP: # %bb.0: ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -991,6 +1060,28 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v16i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %xmm4, %xmm2, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm6, %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %xmm5, %xmm6, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm5, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1 +; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VBMI2-NEXT: vptestnmb %zmm4, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v16i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] @@ -1011,6 +1102,26 @@ ; AVX512VLBW-NEXT: vzeroupper ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v16i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VLVBMI2-NEXT: vpsllvw %ymm5, %ymm6, %ymm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1 +; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1 +; AVX512VLVBMI2-NEXT: vptestnmb %xmm3, %xmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: vzeroupper +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: var_funnnel_v16i8: ; XOP: # %bb.0: ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -1210,6 +1321,23 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [63,63] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsllq %xmm4, %xmm0, %xmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VBMI2-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsrlq %xmm4, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm5, %xmm1 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %xmm2 @@ -1225,6 +1353,12 @@ ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vpshldvq %xmm2, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v2i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1] @@ -1413,6 +1547,25 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpslld %xmm5, %xmm0, %xmm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpsrld %xmm4, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm5, %xmm1 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %xmm2 @@ -1430,6 +1583,12 @@ ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vpshldvd %xmm2, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v4i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0] @@ -1615,6 +1774,25 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsllw %xmm5, %xmm0, %xmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsrlw %xmm4, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm5, %xmm1 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v8i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %xmm2 @@ -1632,6 +1810,12 @@ ; AVX512VLBW-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vpshldvw %xmm2, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v8i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7] @@ -1864,6 +2048,28 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %xmm4, %xmm2, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm6, %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %xmm5, %xmm6, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm5, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm3, %ymm1 +; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VBMI2-NEXT: vptestnmb %zmm4, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %xmm2 @@ -1885,6 +2091,27 @@ ; AVX512VLBW-NEXT: vzeroupper ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VLVBMI2-NEXT: vpsllvw %ymm5, %ymm6, %ymm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %ymm4, %ymm1, %ymm1 +; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1 +; AVX512VLVBMI2-NEXT: vptestnmb %xmm3, %xmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: vzeroupper +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v16i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 @@ -2000,12 +2227,45 @@ ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v2i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 -; AVX512-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v2i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v2i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v2i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v2i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvq {{.*}}(%rip), %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v2i64: ; XOPAVX1: # %bb.0: @@ -2097,12 +2357,45 @@ ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v4i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 -; AVX512-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v4i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v4i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v4i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvd {{.*}}(%rip), %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v4i32: ; XOPAVX1: # %bb.0: @@ -2207,6 +2500,19 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = [16,15,14,13,12,11,10,9] +; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7] +; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm0, %zmm2 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm2, %xmm1 +; AVX512VBMI2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v8i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %xmm1, %xmm1 @@ -2215,6 +2521,11 @@ ; AVX512VLBW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvw {{.*}}(%rip), %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: constant_funnnel_v8i16: ; XOP: # %bb.0: ; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm1, %xmm1 @@ -2384,6 +2695,21 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v16i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [8,7,6,5,4,3,2,1,8,1,2,3,4,5,6,7] +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1] +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm3, %zmm2 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm2, %ymm1 +; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255] +; AVX512VBMI2-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v16i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero @@ -2399,6 +2725,21 @@ ; AVX512VLBW-NEXT: vzeroupper ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VLVBMI2-NEXT: vpsrlvw {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %ymm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpor %ymm1, %ymm2, %ymm1 +; AVX512VLVBMI2-NEXT: vpmovwb %ymm1, %xmm1 +; AVX512VLVBMI2-NEXT: movw $257, %ax # imm = 0x101 +; AVX512VLVBMI2-NEXT: kmovd %eax, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: vzeroupper +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: constant_funnnel_v16i8: ; XOP: # %bb.0: ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm1, %xmm1 @@ -2461,12 +2802,45 @@ ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v2i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlq $50, %xmm1, %xmm1 -; AVX512-NEXT: vpsllq $14, %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v2i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlq $50, %xmm1, %xmm1 +; AVX512F-NEXT: vpsllq $14, %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v2i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlq $50, %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllq $14, %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v2i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlq $50, %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllq $14, %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlq $50, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllq $14, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlq $50, %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllq $14, %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldq $14, %xmm1, %xmm0, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v2i64: ; XOP: # %bb.0: @@ -2500,12 +2874,45 @@ ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v4i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrld $28, %xmm1, %xmm1 -; AVX512-NEXT: vpslld $4, %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $28, %xmm1, %xmm1 +; AVX512F-NEXT: vpslld $4, %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v4i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrld $28, %xmm1, %xmm1 +; AVX512VL-NEXT: vpslld $4, %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v4i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrld $28, %xmm1, %xmm1 +; AVX512BW-NEXT: vpslld $4, %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrld $28, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpslld $4, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrld $28, %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpslld $4, %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldd $4, %xmm1, %xmm0, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v4i32: ; XOP: # %bb.0: @@ -2539,12 +2946,45 @@ ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v8i16: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlw $9, %xmm1, %xmm1 -; AVX512-NEXT: vpsllw $7, %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v8i16: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlw $9, %xmm1, %xmm1 +; AVX512F-NEXT: vpsllw $7, %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v8i16: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlw $9, %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllw $7, %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v8i16: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlw $9, %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllw $7, %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlw $9, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllw $7, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlw $9, %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllw $7, %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldw $7, %xmm1, %xmm0, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v8i16: ; XOP: # %bb.0: Index: llvm/trunk/test/CodeGen/X86/vector-fshl-256.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/vector-fshl-256.ll +++ llvm/trunk/test/CodeGen/X86/vector-fshl-256.ll @@ -4,7 +4,9 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2 @@ -113,6 +115,22 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsllvq %ymm4, %ymm0, %ymm5 +; AVX512VBMI2-NEXT: vpbroadcastq {{.*#+}} ymm6 = [64,64,64,64] +; AVX512VBMI2-NEXT: vpsubq %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpsrlvq %ymm4, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v4i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63] @@ -127,6 +145,11 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvq %ymm2, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v4i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 @@ -278,6 +301,22 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsllvd %ymm4, %ymm0, %ymm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} ymm6 = [32,32,32,32,32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpsrlvd %ymm4, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v8i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31] @@ -292,6 +331,11 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvd %ymm2, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v8i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 @@ -485,6 +529,23 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v16i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] @@ -499,6 +560,11 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvw %ymm2, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v16i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 @@ -745,6 +811,27 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v32i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm5, %zmm6, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v32i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] @@ -764,6 +851,25 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v32i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VLVBMI2-NEXT: vpsllvw %zmm5, %zmm6, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VLVBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VLVBMI2-NEXT: vptestnmb %ymm3, %ymm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %ymm0, %ymm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v32i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 @@ -906,6 +1012,22 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsllq %xmm4, %ymm0, %ymm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VBMI2-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsrlq %xmm4, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v4i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %ymm2 @@ -921,6 +1043,12 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpshldvq %ymm2, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v4i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vmovddup {{.*#+}} xmm2 = xmm2[0,0] @@ -1064,6 +1192,24 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpslld %xmm5, %ymm0, %ymm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpsrld %xmm4, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v8i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %ymm2 @@ -1081,6 +1227,12 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpshldvd %ymm2, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v8i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,0,0,0] @@ -1228,6 +1380,24 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsllw %xmm5, %ymm0, %ymm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsrlw %xmm4, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm5, %ymm1 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v16i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %ymm2 @@ -1245,6 +1415,12 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpshldvw %ymm2, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v16i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7] @@ -1430,6 +1606,27 @@ ; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v32i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm5, %zmm6, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v32i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %ymm2 @@ -1450,6 +1647,26 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VLVBMI2-NEXT: vpsllvw %zmm5, %zmm6, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VLVBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VLVBMI2-NEXT: vptestnmb %ymm3, %ymm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %ymm0, %ymm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v32i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpxor %xmm8, %xmm8, %xmm8 @@ -1537,12 +1754,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v4i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 -; AVX512-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v4i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v4i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v4i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v4i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvq {{.*}}(%rip), %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v4i64: ; XOPAVX1: # %bb.0: @@ -1600,12 +1850,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v8i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 -; AVX512-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v8i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v8i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v8i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v8i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvd {{.*}}(%rip), %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v8i32: ; XOPAVX1: # %bb.0: @@ -1693,6 +1976,19 @@ ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1] +; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm0, %zmm2 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm2, %ymm1 +; AVX512VBMI2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15] +; AVX512VBMI2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v16i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %ymm1, %ymm1 @@ -1702,6 +1998,11 @@ ; AVX512VLBW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvw {{.*}}(%rip), %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: constant_funnnel_v16i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm1, %xmm2 @@ -1866,6 +2167,18 @@ ; AVX512BW-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v32i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VBMI2-NEXT: vpsrlvw {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm2, %zmm1 +; AVX512VBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255] +; AVX512VBMI2-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v32i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero @@ -1880,6 +2193,20 @@ ; AVX512VLBW-NEXT: vmovdqa %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v32i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VLVBMI2-NEXT: vpsrlvw {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm2, %zmm1 +; AVX512VLVBMI2-NEXT: vpmovwb %zmm1, %ymm1 +; AVX512VLVBMI2-NEXT: movl $16843009, %eax # imm = 0x1010101 +; AVX512VLVBMI2-NEXT: kmovd %eax, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %ymm0, %ymm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: constant_funnnel_v32i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -1942,12 +2269,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v4i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlq $50, %ymm1, %ymm1 -; AVX512-NEXT: vpsllq $14, %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v4i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlq $50, %ymm1, %ymm1 +; AVX512F-NEXT: vpsllq $14, %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v4i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlq $50, %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllq $14, %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v4i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlq $50, %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllq $14, %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlq $50, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllq $14, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlq $50, %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllq $14, %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldq $14, %ymm1, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v4i64: ; XOPAVX1: # %bb.0: @@ -1993,12 +2353,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v8i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrld $28, %ymm1, %ymm1 -; AVX512-NEXT: vpslld $4, %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v8i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $28, %ymm1, %ymm1 +; AVX512F-NEXT: vpslld $4, %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v8i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrld $28, %ymm1, %ymm1 +; AVX512VL-NEXT: vpslld $4, %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v8i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrld $28, %ymm1, %ymm1 +; AVX512BW-NEXT: vpslld $4, %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrld $28, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpslld $4, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrld $28, %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpslld $4, %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldd $4, %ymm1, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v8i32: ; XOPAVX1: # %bb.0: @@ -2044,12 +2437,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v16i16: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlw $9, %ymm1, %ymm1 -; AVX512-NEXT: vpsllw $7, %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v16i16: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlw $9, %ymm1, %ymm1 +; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v16i16: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlw $9, %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v16i16: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlw $9, %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllw $7, %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlw $9, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllw $7, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlw $9, %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllw $7, %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldw $7, %ymm1, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v16i16: ; XOPAVX1: # %bb.0: Index: llvm/trunk/test/CodeGen/X86/vector-fshl-512.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/vector-fshl-512.ll +++ llvm/trunk/test/CodeGen/X86/vector-fshl-512.ll @@ -2,7 +2,9 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2 declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>) declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>) @@ -14,37 +16,141 @@ ; define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind { -; AVX512-LABEL: var_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] -; AVX512-NEXT: vpandq %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpsllvq %zmm4, %zmm0, %zmm5 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] -; AVX512-NEXT: vpsubq %zmm4, %zmm6, %zmm4 -; AVX512-NEXT: vpsrlvq %zmm4, %zmm1, %zmm1 -; AVX512-NEXT: vporq %zmm1, %zmm5, %zmm1 -; AVX512-NEXT: vptestnmq %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} -; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: var_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512F-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpsllvq %zmm4, %zmm0, %zmm5 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512F-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512F-NEXT: vpsrlvq %zmm4, %zmm1, %zmm1 +; AVX512F-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512F-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: var_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VL-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpsllvq %zmm4, %zmm0, %zmm5 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512VL-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512VL-NEXT: vpsrlvq %zmm4, %zmm1, %zmm1 +; AVX512VL-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VL-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VL-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: var_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpsllvq %zmm4, %zmm0, %zmm5 +; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512BW-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512BW-NEXT: vpsrlvq %zmm4, %zmm1, %zmm1 +; AVX512BW-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512BW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: var_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: var_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpsllvq %zmm4, %zmm0, %zmm5 +; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512VLBW-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512VLBW-NEXT: vpsrlvq %zmm4, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VLBW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) ret <8 x i64> %res } define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind { -; AVX512-LABEL: var_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] -; AVX512-NEXT: vpandd %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpsllvd %zmm4, %zmm0, %zmm5 -; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] -; AVX512-NEXT: vpsubd %zmm4, %zmm6, %zmm4 -; AVX512-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1 -; AVX512-NEXT: vpord %zmm1, %zmm5, %zmm1 -; AVX512-NEXT: vptestnmd %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} -; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: var_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512F-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpsllvd %zmm4, %zmm0, %zmm5 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512F-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512F-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1 +; AVX512F-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512F-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: var_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VL-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpsllvd %zmm4, %zmm0, %zmm5 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512VL-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512VL-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1 +; AVX512VL-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512VL-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VL-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: var_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512BW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpsllvd %zmm4, %zmm0, %zmm5 +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512BW-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512BW-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1 +; AVX512BW-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512BW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: var_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: var_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VLBW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpsllvd %zmm4, %zmm0, %zmm5 +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512VLBW-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512VLBW-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512VLBW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) ret <16 x i32> %res } @@ -126,6 +232,11 @@ ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] @@ -139,6 +250,11 @@ ; AVX512VLBW-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} ; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) ret <32 x i16> %res } @@ -320,6 +436,46 @@ ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %zmm4, %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vpsllw $5, %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vpaddb %zmm5, %zmm5, %zmm6 +; AVX512VBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VBMI2-NEXT: vpmovb2m %zmm5, %k2 +; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm5 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm1 {%k2} +; AVX512VBMI2-NEXT: vpsrlw $2, %zmm1, %zmm5 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vpsrlw $1, %zmm1, %zmm5 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vpaddb %zmm6, %zmm6, %zmm6 +; AVX512VBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vpsllw $5, %zmm4, %zmm4 +; AVX512VBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm5 +; AVX512VBMI2-NEXT: vpmovb2m %zmm5, %k1 +; AVX512VBMI2-NEXT: vpmovb2m %zmm4, %k2 +; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm4 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VBMI2-NEXT: vpblendmb %zmm4, %zmm0, %zmm4 {%k2} +; AVX512VBMI2-NEXT: vpsllw $2, %zmm4, %zmm6 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm6, %zmm6 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm6, %zmm4 {%k1} +; AVX512VBMI2-NEXT: vpaddb %zmm5, %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vpmovb2m %zmm5, %k1 +; AVX512VBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm4 {%k1} +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm4, %zmm1 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] @@ -359,6 +515,46 @@ ; AVX512VLBW-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} ; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %zmm4, %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpaddb %zmm5, %zmm5, %zmm6 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm5, %k2 +; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm5 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm1 {%k2} +; AVX512VLVBMI2-NEXT: vpsrlw $2, %zmm1, %zmm5 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm1 {%k1} +; AVX512VLVBMI2-NEXT: vpsrlw $1, %zmm1, %zmm5 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpaddb %zmm6, %zmm6, %zmm6 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm1 {%k1} +; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm5 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm5, %k1 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm4, %k2 +; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm4 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vpblendmb %zmm4, %zmm0, %zmm4 {%k2} +; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm4, %zmm6 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm6, %zmm6 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm6, %zmm4 {%k1} +; AVX512VLVBMI2-NEXT: vpaddb %zmm5, %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm5, %k1 +; AVX512VLVBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm4 {%k1} +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm4, %zmm1 +; AVX512VLVBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) ret <64 x i8> %res } @@ -368,42 +564,162 @@ ; define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind { -; AVX512-LABEL: splatvar_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastq %xmm2, %zmm2 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] -; AVX512-NEXT: vpandq %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpsllq %xmm4, %zmm0, %zmm5 -; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] -; AVX512-NEXT: vpsubq %xmm4, %xmm6, %xmm4 -; AVX512-NEXT: vpsrlq %xmm4, %zmm1, %zmm1 -; AVX512-NEXT: vporq %zmm1, %zmm5, %zmm1 -; AVX512-NEXT: vptestnmq %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} -; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatvar_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512F-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpsllq %xmm4, %zmm0, %zmm5 +; AVX512F-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512F-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512F-NEXT: vpsrlq %xmm4, %zmm1, %zmm1 +; AVX512F-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512F-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatvar_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VL-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpsllq %xmm4, %zmm0, %zmm5 +; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VL-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VL-NEXT: vpsrlq %xmm4, %zmm1, %zmm1 +; AVX512VL-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VL-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VL-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatvar_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpsllq %xmm4, %zmm0, %zmm5 +; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512BW-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512BW-NEXT: vpsrlq %xmm4, %zmm1, %zmm1 +; AVX512BW-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512BW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatvar_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatvar_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpsllq %xmm4, %zmm0, %zmm5 +; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VLBW-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VLBW-NEXT: vpsrlq %xmm4, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VLBW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1} +; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %splat) ret <8 x i64> %res } define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind { -; AVX512-LABEL: splatvar_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastd %xmm2, %zmm2 -; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] -; AVX512-NEXT: vpandd %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero -; AVX512-NEXT: vpslld %xmm5, %zmm0, %zmm5 -; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] -; AVX512-NEXT: vpsubd %xmm4, %xmm6, %xmm4 -; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero -; AVX512-NEXT: vpsrld %xmm4, %zmm1, %zmm1 -; AVX512-NEXT: vpord %zmm1, %zmm5, %zmm1 -; AVX512-NEXT: vptestnmd %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} -; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatvar_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512F-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512F-NEXT: vpslld %xmm5, %zmm0, %zmm5 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512F-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512F-NEXT: vpsrld %xmm4, %zmm1, %zmm1 +; AVX512F-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512F-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatvar_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VL-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VL-NEXT: vpslld %xmm5, %zmm0, %zmm5 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VL-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VL-NEXT: vpsrld %xmm4, %zmm1, %zmm1 +; AVX512VL-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512VL-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VL-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatvar_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512BW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512BW-NEXT: vpslld %xmm5, %zmm0, %zmm5 +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512BW-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512BW-NEXT: vpsrld %xmm4, %zmm1, %zmm1 +; AVX512BW-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512BW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatvar_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatvar_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VLBW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VLBW-NEXT: vpslld %xmm5, %zmm0, %zmm5 +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VLBW-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VLBW-NEXT: vpsrld %xmm4, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpord %zmm1, %zmm5, %zmm1 +; AVX512VLBW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1} +; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %splat) ret <16 x i32> %res @@ -467,6 +783,12 @@ ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %zmm2 @@ -483,6 +805,12 @@ ; AVX512VLBW-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} ; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %splat) ret <32 x i16> %res @@ -574,6 +902,31 @@ ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VBMI2-NEXT: vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VBMI2-NEXT: vpsllw %xmm5, %zmm0, %zmm6 +; AVX512VBMI2-NEXT: vpternlogd $255, %zmm7, %zmm7, %zmm7 +; AVX512VBMI2-NEXT: vpsllw %xmm5, %zmm7, %zmm5 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm5, %zmm5 +; AVX512VBMI2-NEXT: vpandq %zmm5, %zmm6, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VBMI2-NEXT: vpsrlw %xmm4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpsrlw %xmm4, %zmm7, %zmm4 +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm4, %zmm4 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm4, %zmm4 +; AVX512VBMI2-NEXT: vpandq %zmm4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %zmm2 @@ -598,6 +951,31 @@ ; AVX512VLBW-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} ; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VLVBMI2-NEXT: vpsllw %xmm5, %zmm0, %zmm6 +; AVX512VLVBMI2-NEXT: vpternlogd $255, %zmm7, %zmm7, %zmm7 +; AVX512VLVBMI2-NEXT: vpsllw %xmm5, %zmm7, %zmm5 +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpandq %zmm5, %zmm6, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VLVBMI2-NEXT: vpsrlw %xmm4, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpsrlw %xmm4, %zmm7, %zmm4 +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vpandq %zmm4, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm5, %zmm1 +; AVX512VLVBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %splat) ret <64 x i8> %res @@ -608,23 +986,85 @@ ; define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind { -; AVX512-LABEL: constant_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 -; AVX512-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 -; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldvq {{.*}}(%rip), %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvq {{.*}}(%rip), %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> ) ret <8 x i64> %res } define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind { -; AVX512-LABEL: constant_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 -; AVX512-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 -; AVX512-NEXT: vpord %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldvd {{.*}}(%rip), %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VLBW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvd {{.*}}(%rip), %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> ) ret <16 x i32> %res } @@ -671,6 +1111,11 @@ ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldvw {{.*}}(%rip), %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %zmm1, %zmm1 @@ -681,6 +1126,11 @@ ; AVX512VLBW-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} ; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldvw {{.*}}(%rip), %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> ) ret <32 x i16> %res } @@ -818,6 +1268,37 @@ ; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256] +; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm3 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k1} +; AVX512VBMI2-NEXT: vpsllw $2, %zmm3, %zmm4 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm4, %zmm3 {%k1} +; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm3 {%k1} +; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpackuswb %zmm2, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm3, %zmm1 +; AVX512VBMI2-NEXT: movabsq $72340172838076673, %rax # imm = 0x101010101010101 +; AVX512VBMI2-NEXT: kmovq %rax, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256] @@ -848,6 +1329,37 @@ ; AVX512VLBW-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} ; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256] +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm3 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VLVBMI2-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k1} +; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm3, %zmm4 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm4, %zmm3 {%k1} +; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm3 {%k1} +; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpackuswb %zmm2, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm3, %zmm1 +; AVX512VLVBMI2-NEXT: movabsq $72340172838076673, %rax # imm = 0x101010101010101 +; AVX512VLVBMI2-NEXT: kmovq %rax, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res } @@ -857,23 +1369,85 @@ ; define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind { -; AVX512-LABEL: splatconstant_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlq $50, %zmm1, %zmm1 -; AVX512-NEXT: vpsllq $14, %zmm0, %zmm0 -; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlq $50, %zmm1, %zmm1 +; AVX512F-NEXT: vpsllq $14, %zmm0, %zmm0 +; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlq $50, %zmm1, %zmm1 +; AVX512VL-NEXT: vpsllq $14, %zmm0, %zmm0 +; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlq $50, %zmm1, %zmm1 +; AVX512BW-NEXT: vpsllq $14, %zmm0, %zmm0 +; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldq $14, %zmm1, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlq $50, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpsllq $14, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldq $14, %zmm1, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> ) ret <8 x i64> %res } define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind { -; AVX512-LABEL: splatconstant_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrld $28, %zmm1, %zmm1 -; AVX512-NEXT: vpslld $4, %zmm0, %zmm0 -; AVX512-NEXT: vpord %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $28, %zmm1, %zmm1 +; AVX512F-NEXT: vpslld $4, %zmm0, %zmm0 +; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrld $28, %zmm1, %zmm1 +; AVX512VL-NEXT: vpslld $4, %zmm0, %zmm0 +; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrld $28, %zmm1, %zmm1 +; AVX512BW-NEXT: vpslld $4, %zmm0, %zmm0 +; AVX512BW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldd $4, %zmm1, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrld $28, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpslld $4, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldd $4, %zmm1, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> ) ret <16 x i32> %res } @@ -906,12 +1480,22 @@ ; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshldw $7, %zmm1, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatconstant_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlw $9, %zmm1, %zmm1 ; AVX512VLBW-NEXT: vpsllw $7, %zmm0, %zmm0 ; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshldw $7, %zmm1, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> ) ret <32 x i16> %res } @@ -958,6 +1542,15 @@ ; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlw $4, %zmm1, %zmm1 @@ -966,6 +1559,15 @@ ; AVX512VLBW-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 ; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res } Index: llvm/trunk/test/CodeGen/X86/vector-fshr-128.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/vector-fshr-128.ll +++ llvm/trunk/test/CodeGen/X86/vector-fshr-128.ll @@ -6,7 +6,9 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2 @@ -153,6 +155,23 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [63,63] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsrlvq %xmm4, %xmm1, %xmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VBMI2-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsllvq %xmm4, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm5, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v2i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm3 = [63,63] @@ -166,6 +185,12 @@ ; AVX512VLBW-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvq %xmm2, %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v2i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -383,6 +408,23 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsrlvd %xmm4, %xmm1, %xmm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsllvd %xmm4, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm5, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v4i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31] @@ -396,6 +438,12 @@ ; AVX512VLBW-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvd %xmm2, %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v4i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -695,6 +743,24 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpor %xmm5, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v8i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] @@ -708,6 +774,12 @@ ; AVX512VLBW-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvw %xmm2, %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: var_funnnel_v8i16: ; XOP: # %bb.0: ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -1004,6 +1076,28 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v16i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %xmm4, %xmm2, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm6, %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %xmm5, %xmm6, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm5, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpor %ymm3, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmb %zmm4, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v16i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] @@ -1023,6 +1117,25 @@ ; AVX512VLBW-NEXT: vzeroupper ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v16i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %ymm5, %ymm6, %ymm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VLVBMI2-NEXT: vpsllvw %ymm4, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: vpmovwb %ymm0, %xmm0 +; AVX512VLVBMI2-NEXT: vptestnmb %xmm3, %xmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm1, %xmm0 {%k1} +; AVX512VLVBMI2-NEXT: vzeroupper +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: var_funnnel_v16i8: ; XOP: # %bb.0: ; XOP-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 @@ -1225,6 +1338,23 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [63,63] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpsrlq %xmm4, %xmm1, %xmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VBMI2-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsllq %xmm4, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm5, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %xmm2 @@ -1239,6 +1369,13 @@ ; AVX512VLBW-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vpshrdvq %xmm2, %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v2i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1] @@ -1428,6 +1565,25 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpsrld %xmm5, %xmm1, %xmm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpslld %xmm4, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm5, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %xmm2 @@ -1444,6 +1600,13 @@ ; AVX512VLBW-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vpshrdvd %xmm2, %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v4i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0] @@ -1631,6 +1794,25 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsrlw %xmm5, %xmm1, %xmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsllw %xmm4, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm5, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v8i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %xmm2 @@ -1647,6 +1829,13 @@ ; AVX512VLBW-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vpshrdvw %xmm2, %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v8i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7] @@ -1881,6 +2070,28 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %xmm2 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %xmm4, %xmm2, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm6, %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %xmm5, %xmm6, %xmm5 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm5, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpor %ymm3, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmb %zmm4, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %xmm2 @@ -1901,6 +2112,26 @@ ; AVX512VLBW-NEXT: vzeroupper ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %xmm2 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %ymm5, %ymm6, %ymm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VLVBMI2-NEXT: vpsllvw %ymm4, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: vpmovwb %ymm0, %xmm0 +; AVX512VLVBMI2-NEXT: vptestnmb %xmm3, %xmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm1, %xmm0 {%k1} +; AVX512VLVBMI2-NEXT: vzeroupper +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v16i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 @@ -2021,12 +2252,46 @@ ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v2i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 -; AVX512-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v2i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v2i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v2i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v2i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvq {{.*}}(%rip), %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v2i64: ; XOPAVX1: # %bb.0: @@ -2118,12 +2383,46 @@ ; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v4i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 -; AVX512-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v4i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v4i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v4i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvd {{.*}}(%rip), %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v4i32: ; XOPAVX1: # %bb.0: @@ -2231,6 +2530,19 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7] +; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm3 = [16,15,14,13,12,11,10,9] +; AVX512VBMI2-NEXT: vpsllvw %zmm3, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7] +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v8i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %xmm1, %xmm2 @@ -2239,6 +2551,12 @@ ; AVX512VLBW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7] ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvw {{.*}}(%rip), %xmm0, %xmm1 +; AVX512VLVBMI2-NEXT: vmovdqa %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: constant_funnnel_v8i16: ; XOP: # %bb.0: ; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm1, %xmm2 @@ -2401,6 +2719,21 @@ ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v16i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1] +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm3, %zmm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [8,7,6,5,4,3,2,1,8,1,2,3,4,5,6,7] +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm3, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpor %ymm2, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255] +; AVX512VBMI2-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0 +; AVX512VBMI2-NEXT: vzeroupper +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v16i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero @@ -2415,6 +2748,20 @@ ; AVX512VLBW-NEXT: vzeroupper ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; AVX512VLVBMI2-NEXT: vpsrlvw {{.*}}(%rip), %ymm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: vpor %ymm2, %ymm0, %ymm0 +; AVX512VLVBMI2-NEXT: vpmovwb %ymm0, %xmm0 +; AVX512VLVBMI2-NEXT: movw $257, %ax # imm = 0x101 +; AVX512VLVBMI2-NEXT: kmovd %eax, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %xmm1, %xmm0 {%k1} +; AVX512VLVBMI2-NEXT: vzeroupper +; AVX512VLVBMI2-NEXT: retq +; ; XOP-LABEL: constant_funnnel_v16i8: ; XOP: # %bb.0: ; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm1, %xmm2 @@ -2474,12 +2821,45 @@ ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v2i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlq $14, %xmm1, %xmm1 -; AVX512-NEXT: vpsllq $50, %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v2i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlq $14, %xmm1, %xmm1 +; AVX512F-NEXT: vpsllq $50, %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v2i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlq $14, %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllq $50, %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v2i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlq $14, %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllq $50, %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlq $14, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllq $50, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlq $14, %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllq $50, %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdq $14, %xmm0, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v2i64: ; XOP: # %bb.0: @@ -2513,12 +2893,45 @@ ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v4i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrld $4, %xmm1, %xmm1 -; AVX512-NEXT: vpslld $28, %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $4, %xmm1, %xmm1 +; AVX512F-NEXT: vpslld $28, %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v4i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrld $4, %xmm1, %xmm1 +; AVX512VL-NEXT: vpslld $28, %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v4i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrld $4, %xmm1, %xmm1 +; AVX512BW-NEXT: vpslld $28, %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrld $4, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpslld $28, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrld $4, %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpslld $28, %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdd $4, %xmm0, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v4i32: ; XOP: # %bb.0: @@ -2552,12 +2965,45 @@ ; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v8i16: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlw $7, %xmm1, %xmm1 -; AVX512-NEXT: vpsllw $9, %xmm0, %xmm0 -; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v8i16: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlw $7, %xmm1, %xmm1 +; AVX512F-NEXT: vpsllw $9, %xmm0, %xmm0 +; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v8i16: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlw $7, %xmm1, %xmm1 +; AVX512VL-NEXT: vpsllw $9, %xmm0, %xmm0 +; AVX512VL-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v8i16: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlw $7, %xmm1, %xmm1 +; AVX512BW-NEXT: vpsllw $9, %xmm0, %xmm0 +; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlw $7, %xmm1, %xmm1 +; AVX512VBMI2-NEXT: vpsllw $9, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlw $7, %xmm1, %xmm1 +; AVX512VLBW-NEXT: vpsllw $9, %xmm0, %xmm0 +; AVX512VLBW-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdw $7, %xmm0, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v8i16: ; XOP: # %bb.0: Index: llvm/trunk/test/CodeGen/X86/vector-fshr-256.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/vector-fshr-256.ll +++ llvm/trunk/test/CodeGen/X86/vector-fshr-256.ll @@ -4,7 +4,9 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2 @@ -112,6 +114,22 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsrlvq %ymm4, %ymm1, %ymm5 +; AVX512VBMI2-NEXT: vpbroadcastq {{.*#+}} ymm6 = [64,64,64,64] +; AVX512VBMI2-NEXT: vpsubq %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpsllvq %ymm4, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v4i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63] @@ -125,6 +143,12 @@ ; AVX512VLBW-NEXT: vmovdqa64 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvq %ymm2, %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v4i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm2, %ymm2 @@ -277,6 +301,22 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsrlvd %ymm4, %ymm1, %ymm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} ymm6 = [32,32,32,32,32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpsllvd %ymm4, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v8i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31] @@ -290,6 +330,12 @@ ; AVX512VLBW-NEXT: vmovdqa32 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvd %ymm2, %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v8i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm2, %ymm2 @@ -485,6 +531,23 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm1, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v16i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] @@ -498,6 +561,12 @@ ; AVX512VLBW-NEXT: vmovdqu16 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvw %ymm2, %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v16i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm2, %ymm2 @@ -746,6 +815,27 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v32i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm5, %zmm6, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v32i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] @@ -764,6 +854,24 @@ ; AVX512VLBW-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: var_funnnel_v32i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %zmm5, %zmm6, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VLVBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VLVBMI2-NEXT: vptestnmb %ymm3, %ymm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1} +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: var_funnnel_v32i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm2, %ymm2 @@ -907,6 +1015,22 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpsrlq %xmm4, %ymm1, %ymm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VBMI2-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpsllq %xmm4, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v4i64: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %ymm2 @@ -921,6 +1045,13 @@ ; AVX512VLBW-NEXT: vmovdqa64 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpshrdvq %ymm2, %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v4i64: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vmovddup {{.*#+}} xmm2 = xmm2[0,0] @@ -1063,6 +1194,24 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpsrld %xmm5, %ymm1, %ymm5 +; AVX512VBMI2-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VBMI2-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VBMI2-NEXT: vpslld %xmm4, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v8i32: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %ymm2 @@ -1079,6 +1228,13 @@ ; AVX512VLBW-NEXT: vmovdqa32 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpshrdvd %ymm2, %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v8i32: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,0,0,0] @@ -1226,6 +1382,24 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsrlw %xmm5, %ymm1, %ymm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16] +; AVX512VBMI2-NEXT: vpsubw %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero +; AVX512VBMI2-NEXT: vpsllw %xmm4, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm5, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmw %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v16i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %ymm2 @@ -1242,6 +1416,13 @@ ; AVX512VLBW-NEXT: vmovdqu16 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vpshrdvw %ymm2, %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v16i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7] @@ -1426,6 +1607,27 @@ ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v32i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %ymm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VBMI2-NEXT: vpsrlvw %zmm5, %zmm6, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v32i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %ymm2 @@ -1445,6 +1647,25 @@ ; AVX512VLBW-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %ymm2 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpand %ymm3, %ymm2, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VLVBMI2-NEXT: vpsrlvw %zmm5, %zmm6, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %ymm4, %ymm6, %ymm4 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VLVBMI2-NEXT: vpsllvw %zmm4, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VLVBMI2-NEXT: vptestnmb %ymm3, %ymm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1} +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: splatvar_funnnel_v32i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 @@ -1534,12 +1755,46 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v4i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 -; AVX512-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v4i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v4i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v4i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v4i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvq {{.*}}(%rip), %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v4i64: ; XOPAVX1: # %bb.0: @@ -1597,12 +1852,46 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: constant_funnnel_v8i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 -; AVX512-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v8i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v8i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v8i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v8i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvd {{.*}}(%rip), %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: constant_funnnel_v8i32: ; XOPAVX1: # %bb.0: @@ -1697,6 +1986,19 @@ ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 +; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +; AVX512VBMI2-NEXT: vpsrlvw %zmm2, %zmm1, %zmm2 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm3 = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1] +; AVX512VBMI2-NEXT: vpsllvw %zmm3, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpor %ymm2, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] +; AVX512VBMI2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v16i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %ymm1, %ymm2 @@ -1706,6 +2008,12 @@ ; AVX512VLBW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvw {{.*}}(%rip), %ymm0, %ymm1 +; AVX512VLVBMI2-NEXT: vmovdqa %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: constant_funnnel_v16i16: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm1, %xmm2 @@ -1874,6 +2182,18 @@ ; AVX512BW-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v32i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VBMI2-NEXT: vpsrlvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255] +; AVX512VBMI2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v32i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero @@ -1887,6 +2207,19 @@ ; AVX512VLBW-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1} ; AVX512VLBW-NEXT: retq ; +; AVX512VLVBMI2-LABEL: constant_funnnel_v32i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero +; AVX512VLVBMI2-NEXT: vpsrlvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512VLVBMI2-NEXT: movl $16843009, %eax # imm = 0x1010101 +; AVX512VLVBMI2-NEXT: kmovd %eax, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1} +; AVX512VLVBMI2-NEXT: retq +; ; XOPAVX1-LABEL: constant_funnnel_v32i8: ; XOPAVX1: # %bb.0: ; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -1949,12 +2282,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v4i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlq $14, %ymm1, %ymm1 -; AVX512-NEXT: vpsllq $50, %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v4i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlq $14, %ymm1, %ymm1 +; AVX512F-NEXT: vpsllq $50, %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v4i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlq $14, %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllq $50, %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v4i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlq $14, %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllq $50, %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlq $14, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllq $50, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlq $14, %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllq $50, %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdq $14, %ymm0, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v4i64: ; XOPAVX1: # %bb.0: @@ -2000,12 +2366,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v8i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrld $4, %ymm1, %ymm1 -; AVX512-NEXT: vpslld $28, %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v8i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $4, %ymm1, %ymm1 +; AVX512F-NEXT: vpslld $28, %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v8i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrld $4, %ymm1, %ymm1 +; AVX512VL-NEXT: vpslld $28, %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v8i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrld $4, %ymm1, %ymm1 +; AVX512BW-NEXT: vpslld $28, %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrld $4, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpslld $28, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrld $4, %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpslld $28, %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdd $4, %ymm0, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v8i32: ; XOPAVX1: # %bb.0: @@ -2051,12 +2450,45 @@ ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; AVX512-LABEL: splatconstant_funnnel_v16i16: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlw $7, %ymm1, %ymm1 -; AVX512-NEXT: vpsllw $9, %ymm0, %ymm0 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v16i16: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlw $7, %ymm1, %ymm1 +; AVX512F-NEXT: vpsllw $9, %ymm0, %ymm0 +; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v16i16: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlw $7, %ymm1, %ymm1 +; AVX512VL-NEXT: vpsllw $9, %ymm0, %ymm0 +; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v16i16: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlw $7, %ymm1, %ymm1 +; AVX512BW-NEXT: vpsllw $9, %ymm0, %ymm0 +; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlw $7, %ymm1, %ymm1 +; AVX512VBMI2-NEXT: vpsllw $9, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlw $7, %ymm1, %ymm1 +; AVX512VLBW-NEXT: vpsllw $9, %ymm0, %ymm0 +; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdw $7, %ymm0, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v16i16: ; XOPAVX1: # %bb.0: Index: llvm/trunk/test/CodeGen/X86/vector-fshr-512.ll =================================================================== --- llvm/trunk/test/CodeGen/X86/vector-fshr-512.ll +++ llvm/trunk/test/CodeGen/X86/vector-fshr-512.ll @@ -2,7 +2,9 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2 declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>) declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>) @@ -14,35 +16,137 @@ ; define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind { -; AVX512-LABEL: var_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] -; AVX512-NEXT: vpandq %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpsrlvq %zmm4, %zmm1, %zmm5 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] -; AVX512-NEXT: vpsubq %zmm4, %zmm6, %zmm4 -; AVX512-NEXT: vpsllvq %zmm4, %zmm0, %zmm0 -; AVX512-NEXT: vporq %zmm5, %zmm0, %zmm0 -; AVX512-NEXT: vptestnmq %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} -; AVX512-NEXT: retq +; AVX512F-LABEL: var_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512F-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpsrlvq %zmm4, %zmm1, %zmm5 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512F-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512F-NEXT: vpsllvq %zmm4, %zmm0, %zmm0 +; AVX512F-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512F-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: var_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VL-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpsrlvq %zmm4, %zmm1, %zmm5 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512VL-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512VL-NEXT: vpsllvq %zmm4, %zmm0, %zmm0 +; AVX512VL-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VL-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: var_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpsrlvq %zmm4, %zmm1, %zmm5 +; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512BW-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512BW-NEXT: vpsllvq %zmm4, %zmm0, %zmm0 +; AVX512BW-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512BW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: var_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdvq %zmm2, %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: var_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpsrlvq %zmm4, %zmm1, %zmm5 +; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64] +; AVX512VLBW-NEXT: vpsubq %zmm4, %zmm6, %zmm4 +; AVX512VLBW-NEXT: vpsllvq %zmm4, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvq %zmm2, %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) ret <8 x i64> %res } define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind { -; AVX512-LABEL: var_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] -; AVX512-NEXT: vpandd %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpsrlvd %zmm4, %zmm1, %zmm5 -; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] -; AVX512-NEXT: vpsubd %zmm4, %zmm6, %zmm4 -; AVX512-NEXT: vpsllvd %zmm4, %zmm0, %zmm0 -; AVX512-NEXT: vpord %zmm5, %zmm0, %zmm0 -; AVX512-NEXT: vptestnmd %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} -; AVX512-NEXT: retq +; AVX512F-LABEL: var_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512F-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpsrlvd %zmm4, %zmm1, %zmm5 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512F-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512F-NEXT: vpsllvd %zmm4, %zmm0, %zmm0 +; AVX512F-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512F-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: var_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VL-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpsrlvd %zmm4, %zmm1, %zmm5 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512VL-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512VL-NEXT: vpsllvd %zmm4, %zmm0, %zmm0 +; AVX512VL-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512VL-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: var_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512BW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpsrlvd %zmm4, %zmm1, %zmm5 +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512BW-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512BW-NEXT: vpsllvd %zmm4, %zmm0, %zmm0 +; AVX512BW-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512BW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: var_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdvd %zmm2, %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: var_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VLBW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpsrlvd %zmm4, %zmm1, %zmm5 +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32] +; AVX512VLBW-NEXT: vpsubd %zmm4, %zmm6, %zmm4 +; AVX512VLBW-NEXT: vpsllvd %zmm4, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvd %zmm2, %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) ret <16 x i32> %res } @@ -123,6 +227,12 @@ ; AVX512BW-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdvw %zmm2, %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] @@ -135,6 +245,12 @@ ; AVX512VLBW-NEXT: vptestnmw %zmm3, %zmm2, %k1 ; AVX512VLBW-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvw %zmm2, %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) ret <32 x i16> %res } @@ -315,6 +431,45 @@ ; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: var_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VBMI2-NEXT: vpsllw $5, %zmm4, %zmm5 +; AVX512VBMI2-NEXT: vpaddb %zmm5, %zmm5, %zmm6 +; AVX512VBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VBMI2-NEXT: vpmovb2m %zmm5, %k2 +; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm5 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vpblendmb %zmm5, %zmm1, %zmm5 {%k2} +; AVX512VBMI2-NEXT: vpsrlw $2, %zmm5, %zmm7 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm7, %zmm7 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm7, %zmm5 {%k1} +; AVX512VBMI2-NEXT: vpsrlw $1, %zmm5, %zmm7 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm7, %zmm7 +; AVX512VBMI2-NEXT: vpaddb %zmm6, %zmm6, %zmm6 +; AVX512VBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm7, %zmm5 {%k1} +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %zmm4, %zmm6, %zmm4 +; AVX512VBMI2-NEXT: vpsllw $5, %zmm4, %zmm4 +; AVX512VBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm6 +; AVX512VBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VBMI2-NEXT: vpmovb2m %zmm4, %k2 +; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm4 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm4, %zmm0 {%k2} +; AVX512VBMI2-NEXT: vpsllw $2, %zmm0, %zmm4 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm4, %zmm0 {%k1} +; AVX512VBMI2-NEXT: vpaddb %zmm6, %zmm6, %zmm4 +; AVX512VBMI2-NEXT: vpmovb2m %zmm4, %k1 +; AVX512VBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1} +; AVX512VBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: var_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] @@ -353,6 +508,45 @@ ; AVX512VLBW-NEXT: vptestnmb %zmm3, %zmm2, %k1 ; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: var_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm4, %zmm5 +; AVX512VLVBMI2-NEXT: vpaddb %zmm5, %zmm5, %zmm6 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm5, %k2 +; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm5 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpblendmb %zmm5, %zmm1, %zmm5 {%k2} +; AVX512VLVBMI2-NEXT: vpsrlw $2, %zmm5, %zmm7 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm7, %zmm7 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm7, %zmm5 {%k1} +; AVX512VLVBMI2-NEXT: vpsrlw $1, %zmm5, %zmm7 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm7, %zmm7 +; AVX512VLVBMI2-NEXT: vpaddb %zmm6, %zmm6, %zmm6 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm7, %zmm5 {%k1} +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %zmm4, %zmm6, %zmm4 +; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm6 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm6, %k1 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm4, %k2 +; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm4 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm4, %zmm0 {%k2} +; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm0, %zmm4 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm4, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: vpaddb %zmm6, %zmm6, %zmm4 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm4, %k1 +; AVX512VLVBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) ret <64 x i8> %res } @@ -362,40 +556,158 @@ ; define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind { -; AVX512-LABEL: splatvar_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastq %xmm2, %zmm2 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] -; AVX512-NEXT: vpandq %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpsrlq %xmm4, %zmm1, %zmm5 -; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] -; AVX512-NEXT: vpsubq %xmm4, %xmm6, %xmm4 -; AVX512-NEXT: vpsllq %xmm4, %zmm0, %zmm0 -; AVX512-NEXT: vporq %zmm5, %zmm0, %zmm0 -; AVX512-NEXT: vptestnmq %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} -; AVX512-NEXT: retq +; AVX512F-LABEL: splatvar_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512F-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpsrlq %xmm4, %zmm1, %zmm5 +; AVX512F-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512F-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512F-NEXT: vpsllq %xmm4, %zmm0, %zmm0 +; AVX512F-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512F-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatvar_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VL-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpsrlq %xmm4, %zmm1, %zmm5 +; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VL-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VL-NEXT: vpsllq %xmm4, %zmm0, %zmm0 +; AVX512VL-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VL-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatvar_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpsrlq %xmm4, %zmm1, %zmm5 +; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512BW-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512BW-NEXT: vpsllq %xmm4, %zmm0, %zmm0 +; AVX512BW-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512BW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatvar_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vpshrdvq %zmm2, %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatvar_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63] +; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpsrlq %xmm4, %zmm1, %zmm5 +; AVX512VLBW-NEXT: vmovdqa {{.*#+}} xmm6 = [64,64] +; AVX512VLBW-NEXT: vpsubq %xmm4, %xmm6, %xmm4 +; AVX512VLBW-NEXT: vpsllq %xmm4, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vptestnmq %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1} +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpshrdvq %zmm2, %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %splat) ret <8 x i64> %res } define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind { -; AVX512-LABEL: splatvar_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpbroadcastd %xmm2, %zmm2 -; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] -; AVX512-NEXT: vpandd %zmm3, %zmm2, %zmm4 -; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero -; AVX512-NEXT: vpsrld %xmm5, %zmm1, %zmm5 -; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] -; AVX512-NEXT: vpsubd %xmm4, %xmm6, %xmm4 -; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero -; AVX512-NEXT: vpslld %xmm4, %zmm0, %zmm0 -; AVX512-NEXT: vpord %zmm5, %zmm0, %zmm0 -; AVX512-NEXT: vptestnmd %zmm3, %zmm2, %k1 -; AVX512-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} -; AVX512-NEXT: retq +; AVX512F-LABEL: splatvar_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512F-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512F-NEXT: vpsrld %xmm5, %zmm1, %zmm5 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512F-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512F-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512F-NEXT: vpslld %xmm4, %zmm0, %zmm0 +; AVX512F-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512F-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatvar_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VL-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VL-NEXT: vpsrld %xmm5, %zmm1, %zmm5 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VL-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VL-NEXT: vpslld %xmm4, %zmm0, %zmm0 +; AVX512VL-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512VL-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VL-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatvar_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512BW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512BW-NEXT: vpsrld %xmm5, %zmm1, %zmm5 +; AVX512BW-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512BW-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512BW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512BW-NEXT: vpslld %xmm4, %zmm0, %zmm0 +; AVX512BW-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512BW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512BW-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatvar_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vpshrdvd %zmm2, %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatvar_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512VLBW-NEXT: vpandd %zmm3, %zmm2, %zmm4 +; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero +; AVX512VLBW-NEXT: vpsrld %xmm5, %zmm1, %zmm5 +; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32] +; AVX512VLBW-NEXT: vpsubd %xmm4, %xmm6, %xmm4 +; AVX512VLBW-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero +; AVX512VLBW-NEXT: vpslld %xmm4, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vpord %zmm5, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vptestnmd %zmm3, %zmm2, %k1 +; AVX512VLBW-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1} +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpshrdvd %zmm2, %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %splat) ret <16 x i32> %res @@ -458,6 +770,13 @@ ; AVX512BW-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vpshrdvw %zmm2, %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastw %xmm2, %zmm2 @@ -473,6 +792,13 @@ ; AVX512VLBW-NEXT: vptestnmw %zmm3, %zmm2, %k1 ; AVX512VLBW-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpshrdvw %zmm2, %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %splat) ret <32 x i16> %res @@ -563,6 +889,30 @@ ; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpbroadcastb %xmm2, %zmm2 +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VBMI2-NEXT: vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VBMI2-NEXT: vpsrlw %xmm5, %zmm1, %zmm6 +; AVX512VBMI2-NEXT: vpternlogd $255, %zmm7, %zmm7, %zmm7 +; AVX512VBMI2-NEXT: vpsrlw %xmm5, %zmm7, %zmm5 +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm5, %zmm5 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm5, %zmm5 +; AVX512VBMI2-NEXT: vpandq %zmm5, %zmm6, %zmm5 +; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VBMI2-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VBMI2-NEXT: vpsllw %xmm4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpsllw %xmm4, %zmm7, %zmm4 +; AVX512VBMI2-NEXT: vpbroadcastb %xmm4, %zmm4 +; AVX512VBMI2-NEXT: vpandq %zmm4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpbroadcastb %xmm2, %zmm2 @@ -586,6 +936,30 @@ ; AVX512VLBW-NEXT: vptestnmb %zmm3, %zmm2, %k1 ; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VLVBMI2-NEXT: vpsrlw %xmm5, %zmm1, %zmm6 +; AVX512VLVBMI2-NEXT: vpternlogd $255, %zmm7, %zmm7, %zmm7 +; AVX512VLVBMI2-NEXT: vpsrlw %xmm5, %zmm7, %zmm5 +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm5, %zmm5 +; AVX512VLVBMI2-NEXT: vpandq %zmm5, %zmm6, %zmm5 +; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VLVBMI2-NEXT: vpsubb %xmm4, %xmm6, %xmm4 +; AVX512VLVBMI2-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero +; AVX512VLVBMI2-NEXT: vpsllw %xmm4, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vpsllw %xmm4, %zmm7, %zmm4 +; AVX512VLVBMI2-NEXT: vpbroadcastb %xmm4, %zmm4 +; AVX512VLVBMI2-NEXT: vpandq %zmm4, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vporq %zmm5, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vptestnmb %zmm3, %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: retq %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %splat) ret <64 x i8> %res @@ -596,23 +970,89 @@ ; define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind { -; AVX512-LABEL: constant_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 -; AVX512-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 -; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512F-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VL-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512BW-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdvq {{.*}}(%rip), %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvq {{.*}}(%rip), %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> ) ret <8 x i64> %res } define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind { -; AVX512-LABEL: constant_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 -; AVX512-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 -; AVX512-NEXT: vpord %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: constant_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512F-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: constant_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VL-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: constant_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512BW-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: constant_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdvd {{.*}}(%rip), %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: constant_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlvd {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VLBW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvd {{.*}}(%rip), %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> ) ret <16 x i32> %res } @@ -666,6 +1106,12 @@ ; AVX512BW-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdvw {{.*}}(%rip), %zmm0, %zmm1 +; AVX512VBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlvw {{.*}}(%rip), %zmm1, %zmm2 @@ -675,6 +1121,12 @@ ; AVX512VLBW-NEXT: kmovd %eax, %k1 ; AVX512VLBW-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdvw {{.*}}(%rip), %zmm0, %zmm1 +; AVX512VLVBMI2-NEXT: vmovdqa64 %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> ) ret <32 x i16> %res } @@ -811,6 +1263,36 @@ ; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: constant_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536] +; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm3 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1} +; AVX512VBMI2-NEXT: vpsllw $2, %zmm0, %zmm3 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1} +; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1} +; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3 +; AVX512VBMI2-NEXT: vpackuswb %zmm2, %zmm3, %zmm2 +; AVX512VBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: movabsq $72340172838076673, %rax # imm = 0x101010101010101 +; AVX512VBMI2-NEXT: kmovq %rax, %k1 +; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: constant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536] @@ -840,6 +1322,36 @@ ; AVX512VLBW-NEXT: kmovq %rax, %k1 ; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536] +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm3 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm0, %zmm3 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1 +; AVX512VLVBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2 +; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3 +; AVX512VLVBMI2-NEXT: vpsllvw {{.*}}(%rip), %zmm3, %zmm3 +; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3 +; AVX512VLVBMI2-NEXT: vpackuswb %zmm2, %zmm3, %zmm2 +; AVX512VLVBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: movabsq $72340172838076673, %rax # imm = 0x101010101010101 +; AVX512VLVBMI2-NEXT: kmovq %rax, %k1 +; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1} +; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res } @@ -849,23 +1361,85 @@ ; define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind { -; AVX512-LABEL: splatconstant_funnnel_v8i64: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrlq $14, %zmm1, %zmm1 -; AVX512-NEXT: vpsllq $50, %zmm0, %zmm0 -; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v8i64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrlq $14, %zmm1, %zmm1 +; AVX512F-NEXT: vpsllq $50, %zmm0, %zmm0 +; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v8i64: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrlq $14, %zmm1, %zmm1 +; AVX512VL-NEXT: vpsllq $50, %zmm0, %zmm0 +; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v8i64: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrlq $14, %zmm1, %zmm1 +; AVX512BW-NEXT: vpsllq $50, %zmm0, %zmm0 +; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i64: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdq $14, %zmm0, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v8i64: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrlq $14, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpsllq $50, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i64: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdq $14, %zmm0, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> ) ret <8 x i64> %res } define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind { -; AVX512-LABEL: splatconstant_funnnel_v16i32: -; AVX512: # %bb.0: -; AVX512-NEXT: vpsrld $4, %zmm1, %zmm1 -; AVX512-NEXT: vpslld $28, %zmm0, %zmm0 -; AVX512-NEXT: vpord %zmm1, %zmm0, %zmm0 -; AVX512-NEXT: retq +; AVX512F-LABEL: splatconstant_funnnel_v16i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $4, %zmm1, %zmm1 +; AVX512F-NEXT: vpslld $28, %zmm0, %zmm0 +; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: splatconstant_funnnel_v16i32: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpsrld $4, %zmm1, %zmm1 +; AVX512VL-NEXT: vpslld $28, %zmm0, %zmm0 +; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_funnnel_v16i32: +; AVX512BW: # %bb.0: +; AVX512BW-NEXT: vpsrld $4, %zmm1, %zmm1 +; AVX512BW-NEXT: vpslld $28, %zmm0, %zmm0 +; AVX512BW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i32: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdd $4, %zmm0, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; +; AVX512VLBW-LABEL: splatconstant_funnnel_v16i32: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vpsrld $4, %zmm1, %zmm1 +; AVX512VLBW-NEXT: vpslld $28, %zmm0, %zmm0 +; AVX512VLBW-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i32: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdd $4, %zmm0, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> ) ret <16 x i32> %res } @@ -898,12 +1472,22 @@ ; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpshrdw $7, %zmm0, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatconstant_funnnel_v32i16: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlw $7, %zmm1, %zmm1 ; AVX512VLBW-NEXT: vpsllw $9, %zmm0, %zmm0 ; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpshrdw $7, %zmm0, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> ) ret <32 x i16> %res } @@ -950,6 +1534,15 @@ ; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: retq ; +; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: +; AVX512VBMI2: # %bb.0: +; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VBMI2-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VBMI2-NEXT: retq +; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsrlw $4, %zmm1, %zmm1 @@ -958,6 +1551,15 @@ ; AVX512VLBW-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 ; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: +; AVX512VLVBMI2: # %bb.0: +; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm1, %zmm1 +; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res }