[X86][AVX512] lowering kunpack intrinsic - llvm part

[jina.nahias]

Summary

the clang part : https://reviews.llvm.org/D39719

[jina.nahias]

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

6–389

there are no tests here cause of a bug not related to this patch. will be added after his fixing.

[craig.topper]

I don't see any tests that produce kunpckbw after this change.

lib/IR/AutoUpgrade.cpp

1071

Can this just be

And = (1 << Shift) - 1;

[jina.nahias]

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

6–389

In D39720#918931, @craig.topper wrote:

I don't see any tests that produce kunpckbw after this change.

it should be here. @craig.topper

[craig.topper]

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

6–389

I only see a test case from avx512-intrinsics.ll moved to avx512-intrinsics-upgrade.ll, but its not producing a kunpckbw instruction. I don't see a test case that produces a kunpck from shift and or in this diff. Unless there's one already in trunk?

[jina.nahias]

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

6–389

there are no tests producing a kunpckbw instruction cause of a bug not related to this patch. will be added after his fixing. i wont push this commit before his fixing.
@guyblank @craig.topper

[jina.nahias]

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

6–389

Done @craig.topper

[uweigand]

This has caused build bot failures on SystemZ (and others):

/home/uweigand/sandbox/buildbot/clang-s390x-linux/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll:32:18: error: AVX512BW-NEXT: is not on the line after the previous match
; AVX512BW-NEXT: movq %rsi, %rax
                 ^
<stdin>:20:2: note: 'next' match was here
 movq %rsi, %rax
 ^
<stdin>:18:16: note: previous match ended here
 shlq $32, %rsi
               ^
<stdin>:19:1: note: non-matching line after previous match is here
 orq %rdi, %rsi

include/llvm/IR/IntrinsicsX86.td

def int_x86_avx512_kor_w : GCCBuiltin<"__builtin_ia32_korhi">,
               Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty],
                          [IntrNoMem]>;
   def int_x86_avx512_kxor_w : GCCBuiltin<"__builtin_ia32_kxorhi">,
               Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty],
                          [IntrNoMem]>;
   def int_x86_avx512_kxnor_w : GCCBuiltin<"__builtin_ia32_kxnorhi">,
               Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty],
                          [IntrNoMem]>;
-  def int_x86_avx512_kunpck_bw : GCCBuiltin<"__builtin_ia32_kunpckhi">,
-              Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty],
-                         [IntrNoMem]>;
-  def int_x86_avx512_kunpck_wd : GCCBuiltin<"__builtin_ia32_kunpcksi">,
-              Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
-                         [IntrNoMem]>;
-  def int_x86_avx512_kunpck_dq : GCCBuiltin<"__builtin_ia32_kunpckdi">,
-              Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty],
-                         [IntrNoMem]>;
   def int_x86_avx512_kortestz_w : GCCBuiltin<"__builtin_ia32_kortestzhi">,
               Intrinsic<[llvm_i32_ty], [llvm_i16_ty, llvm_i16_ty],
                         [IntrNoMem]>;
   def int_x86_avx512_kortestc_w : GCCBuiltin<"__builtin_ia32_kortestchi">,
               Intrinsic<[llvm_i32_ty], [llvm_i16_ty, llvm_i16_ty],
                         [IntrNoMem]>;
 }
 

lib/IR/AutoUpgrade.cpp

static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
   // like to use this information to remove upgrade code for some older
   // intrinsics. It is currently undecided how we will determine that future
   // point.
   if (Name=="ssse3.pabs.b.128" || // Added in 6.0
       Name=="ssse3.pabs.w.128" || // Added in 6.0
       Name=="ssse3.pabs.d.128" || // Added in 6.0
       Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
       Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
+      Name.startswith("avx512.kunpck") || //added in 6.0 
       Name.startswith("avx2.pabs.") || // Added in 6.0
       Name.startswith("avx512.mask.pabs.") || // Added in 6.0
       Name.startswith("avx512.broadcastm") || // Added in 6.0
       Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
       Name.startswith("sse2.pcmpeq.") || // Added in 3.1
       Name.startswith("sse2.pcmpgt.") || // Added in 3.1
       Name.startswith("avx2.pcmpeq.") || // Added in 3.1
       Name.startswith("avx2.pcmpgt.") || // Added in 3.1
if (IsX86 && (Name.startswith("sse2.pcmp") ||
       unsigned NumElts = Op0->getType()->getVectorNumElements();
       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask, NumElts);
     } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
       unsigned NumElts =
           CI->getArgOperand(1)->getType()->getVectorNumElements();
       Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                           CI->getArgOperand(1));
+    } else if (IsX86 && (Name.startswith("avx512.kunpck"))) {
+      uint64_t Shift = CI->getType()->getScalarSizeInBits() / 2;
+      uint64_t And = (1 << Shift) - 1; 
+      Value* LowBits =  Builder.CreateAnd(CI->getArgOperand(0), And);
+      Value* HighBits =  Builder.CreateShl(CI->getArgOperand(1), Shift);
+      Rep = Builder.CreateOr(LowBits, HighBits);
     } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) {
       Type *I32Ty = Type::getInt32Ty(C);
       Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
                                                  ConstantInt::get(I32Ty, 0));
       Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
                                                  ConstantInt::get(I32Ty, 0));
       Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
                                         Builder.CreateFAdd(Elt0, Elt1),

lib/Target/X86/X86ISelLowering.cpp

 // before the illegal vector is scalarized on subtargets that don't have legal
 // vxi1 types.
 static SDValue combineBitcastvxi1(SelectionDAG &DAG, SDValue BitCast,
                                   const X86Subtarget &Subtarget) {
   EVT VT = BitCast.getValueType();
   SDValue N0 = BitCast.getOperand(0);
   EVT VecVT = N0->getValueType(0);
 
+  if (VT.isVector() && VecVT.isScalarInteger() && Subtarget.hasAVX512() &&
+      N0->getOpcode() == ISD::OR) {
+    SDValue Op0 = N0->getOperand(0);
+    SDValue Op1 = N0->getOperand(1);
+    MVT TrunckVT;
+    MVT BitcastVT;
+    switch (VT.getSimpleVT().SimpleTy) {
+    default:
+      return SDValue();
+    case MVT::v16i1:
+      TrunckVT = MVT::i8;
+      BitcastVT = MVT::v8i1;
+      break;
+    case MVT::v32i1:
+      TrunckVT = MVT::i16;
+      BitcastVT = MVT::v16i1;
+      break;
+    case MVT::v64i1:
+      TrunckVT = MVT::i32;
+      BitcastVT = MVT::v32i1;
+      break;
+    }
+    bool isArg0UndefRight = Op0->getOpcode() == ISD::SHL;
+    bool isArg0UndefLeft =
+        Op0->getOpcode() == ISD::ZERO_EXTEND || Op0->getOpcode() == ISD::AND;
+    bool isArg1UndefRight = Op1->getOpcode() == ISD::SHL;
+    bool isArg1UndefLeft =
+        Op1->getOpcode() == ISD::ZERO_EXTEND || Op1->getOpcode() == ISD::AND;
+    SDValue OpLeft;
+    SDValue OpRight;
+    if (isArg0UndefRight && isArg1UndefLeft) {
+      OpLeft = Op0;
+      OpRight = Op1;
+    } else if (isArg1UndefRight && isArg0UndefLeft) {
+      OpLeft = Op1;
+      OpRight = Op0;
+    } else
+      return SDValue();
+    SDLoc DL(BitCast);
+    SDValue Shr = OpLeft->getOperand(0);
+    SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, TrunckVT, Shr);
+    SDValue Bitcast1 = DAG.getBitcast(BitcastVT, Trunc1);
+    SDValue Trunc2 = DAG.getNode(ISD::TRUNCATE, DL, TrunckVT, OpRight);
+    SDValue Bitcast2 = DAG.getBitcast(BitcastVT, Trunc2);
+    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Bitcast1, Bitcast2);
+  }
+
   if (!VT.isScalarInteger() || !VecVT.isSimple())
     return SDValue();
 
   // With AVX512 vxi1 types are legal and we prefer using k-regs.
   // MOVMSK is supported in SSE2 or later.
   if (Subtarget.hasAVX512() || !Subtarget.hasSSE2())
     return SDValue();
 

lib/Target/X86/X86IntrinsicsInfo.h

static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_cvtusi642ss, INTR_TYPE_3OP, X86ISD::SCALAR_UINT_TO_FP_RND, 0),
   X86_INTRINSIC_DATA(avx512_cvtw2mask_128, CONVERT_TO_MASK, X86ISD::CVT2MASK, 0),
   X86_INTRINSIC_DATA(avx512_cvtw2mask_256, CONVERT_TO_MASK, X86ISD::CVT2MASK, 0),
   X86_INTRINSIC_DATA(avx512_cvtw2mask_512, CONVERT_TO_MASK, X86ISD::CVT2MASK, 0),
   X86_INTRINSIC_DATA(avx512_exp2_pd, INTR_TYPE_1OP_MASK_RM, X86ISD::EXP2, 0),
   X86_INTRINSIC_DATA(avx512_exp2_ps, INTR_TYPE_1OP_MASK_RM, X86ISD::EXP2, 0),
   X86_INTRINSIC_DATA(avx512_kand_w, MASK_BINOP, ISD::AND, 0),
   X86_INTRINSIC_DATA(avx512_kor_w, MASK_BINOP, ISD::OR, 0),
-  X86_INTRINSIC_DATA(avx512_kunpck_bw, KUNPCK, ISD::CONCAT_VECTORS, 0),
-  X86_INTRINSIC_DATA(avx512_kunpck_dq, KUNPCK, ISD::CONCAT_VECTORS, 0),
-  X86_INTRINSIC_DATA(avx512_kunpck_wd, KUNPCK, ISD::CONCAT_VECTORS, 0),
   X86_INTRINSIC_DATA(avx512_kxor_w, MASK_BINOP, ISD::XOR, 0),
   X86_INTRINSIC_DATA(avx512_mask_add_pd_512, INTR_TYPE_2OP_MASK, ISD::FADD,
   X86ISD::FADD_RND),
   X86_INTRINSIC_DATA(avx512_mask_add_ps_512, INTR_TYPE_2OP_MASK, ISD::FADD,
   X86ISD::FADD_RND),
   X86_INTRINSIC_DATA(avx512_mask_add_sd_round, INTR_TYPE_SCALAR_MASK_RM,
                      X86ISD::FADDS_RND, 0),
   X86_INTRINSIC_DATA(avx512_mask_add_ss_round, INTR_TYPE_SCALAR_MASK_RM,

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=X32
 ; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=X64
 
 ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512f-builtins.c
 
 
+define zeroext i16 @test_mm512_kunpackb(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> %__C, <8 x i64> %__D, <8 x i64> %__E, <8 x i64> %__F) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_kunpackb:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    andl $-64, %esp
+; X32-NEXT:    subl $64, %esp
+; X32-NEXT:    vmovdqa64 136(%ebp), %zmm3
+; X32-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X32-NEXT:    vpcmpneqd 8(%ebp), %zmm2, %k1
+; X32-NEXT:    kunpckbw %k0, %k1, %k1
+; X32-NEXT:    vpcmpneqd 72(%ebp), %zmm3, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    movl %ebp, %esp
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_kunpackb:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vpcmpneqd %zmm1, %zmm0, %k0
+; X64-NEXT:    vpcmpneqd %zmm3, %zmm2, %k1
+; X64-NEXT:    kunpckbw %k0, %k1, %k1
+; X64-NEXT:    vpcmpneqd %zmm5, %zmm4, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %__A to <16 x i32>
+  %1 = bitcast <8 x i64> %__B to <16 x i32>
+  %2 = icmp ne <16 x i32> %0, %1
+  %3 = bitcast <16 x i1> %2 to i16
+  %4 = bitcast <8 x i64> %__C to <16 x i32>
+  %5 = bitcast <8 x i64> %__D to <16 x i32>
+  %6 = icmp ne <16 x i32> %4, %5
+  %7 = bitcast <16 x i1> %6 to i16
+  %8 = and i16 %7, 255
+  %shl.i = shl i16 %3, 8
+  %or.i = or i16 %8, %shl.i
+  %9 = bitcast <8 x i64> %__E to <16 x i32>
+  %10 = bitcast <8 x i64> %__F to <16 x i32>
+  %11 = icmp ne <16 x i32> %9, %10
+  %12 = bitcast i16 %or.i to <16 x i1>
+  %13 = and <16 x i1> %11, %12
+  %14 = bitcast <16 x i1> %13 to i16
+  ret i16 %14
+}
+
 define <16 x float> @test_mm512_shuffle_f32x4(<16 x float> %__A, <16 x float> %__B) {
 ; X32-LABEL: test_mm512_shuffle_f32x4:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_shuffle_f32x4:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <16 x float> %__A, <16 x float> %__B, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 16, i32 17, i32 18, i32 19>
   ret <16 x float> %shuffle
 }
 
 
 define <16 x float> @test_mm512_mask_shuffle_f32x4(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X32-LABEL: test_mm512_mask_shuffle_f32x4:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3,4,5,6,7],zmm2[0,1,2,3,0,1,2,3]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shuffle_f32x4:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3,4,5,6,7],zmm2[0,1,2,3,0,1,2,3]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <16 x float> %__A, <16 x float> %__B, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 16, i32 17, i32 18, i32 19>
   %0 = bitcast i16 %__U to <16 x i1>
   %1 = select <16 x i1> %0, <16 x float> %shuffle, <16 x float> %__W
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_maskz_shuffle_f32x4(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X32-LABEL: test_mm512_maskz_shuffle_f32x4:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],zmm1[0,1,2,3,0,1,2,3]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shuffle_f32x4:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],zmm1[0,1,2,3,0,1,2,3]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <16 x float> %__A, <16 x float> %__B, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 16, i32 17, i32 18, i32 19>
   %0 = bitcast i16 %__U to <16 x i1>
   %1 = select <16 x i1> %0, <16 x float> %shuffle, <16 x float> zeroinitializer
   ret <16 x float> %1
 }
 
 define <8 x double> @test_mm512_shuffle_f64x2(<8 x double> %__A, <8 x double> %__B) {
 ; X32-LABEL: test_mm512_shuffle_f64x2:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_shuffle_f64x2:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x double> %__A, <8 x double> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   ret <8 x double> %shuffle
 }
 
 define <8 x double> @test_mm512_mask_shuffle_f64x2(<8 x double> %__W, i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
 ; X32-LABEL: test_mm512_mask_shuffle_f64x2:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw %eax, %k1
 ; X32-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3],zmm2[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shuffle_f64x2:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3],zmm2[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x double> %__A, <8 x double> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   %0 = bitcast i8 %__U to <8 x i1>
   %1 = select <8 x i1> %0, <8 x double> %shuffle, <8 x double> %__W
   ret <8 x double> %1
 }
 
 define <8 x double> @test_mm512_maskz_shuffle_f64x2(i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
 ; X32-LABEL: test_mm512_maskz_shuffle_f64x2:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw %eax, %k1
 ; X32-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shuffle_f64x2:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x double> %__A, <8 x double> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   %0 = bitcast i8 %__U to <8 x i1>
   %1 = select <8 x i1> %0, <8 x double> %shuffle, <8 x double> zeroinitializer
   ret <8 x double> %1
 }
 
 define <8 x i64> @test_mm512_shuffle_i32x4(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X32-LABEL: test_mm512_shuffle_i32x4:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_shuffle_i32x4:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i64> %__A, <8 x i64> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   ret <8 x i64> %shuffle
 }
 
 define <8 x i64> @test_mm512_mask_shuffle_i32x4(<8 x i64> %__W, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X32-LABEL: test_mm512_mask_shuffle_i32x4:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3,4,5,6,7],zmm2[0,1,2,3,0,1,2,3]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shuffle_i32x4:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3,4,5,6,7],zmm2[0,1,2,3,0,1,2,3]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i64> %__A, <8 x i64> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   %0 = bitcast <8 x i64> %shuffle to <16 x i32>
   %1 = bitcast <8 x i64> %__W to <16 x i32>
   %2 = bitcast i16 %__U to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %0, <16 x i32> %1
   %4 = bitcast <16 x i32> %3 to <8 x i64>
   ret <8 x i64> %4
 }
 
 define <8 x i64> @test_mm512_maskz_shuffle_i32x4(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X32-LABEL: test_mm512_maskz_shuffle_i32x4:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],zmm1[0,1,2,3,0,1,2,3]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shuffle_i32x4:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],zmm1[0,1,2,3,0,1,2,3]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i64> %__A, <8 x i64> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   %0 = bitcast <8 x i64> %shuffle to <16 x i32>
   %1 = bitcast i16 %__U to <16 x i1>
   %2 = select <16 x i1> %1, <16 x i32> %0, <16 x i32> zeroinitializer
   %3 = bitcast <16 x i32> %2 to <8 x i64>
   ret <8 x i64> %3
 }
 
 define <8 x i64> @test_mm512_shuffle_i64x2(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X32-LABEL: test_mm512_shuffle_i64x2:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_shuffle_i64x2:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i64> %__A, <8 x i64> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   ret <8 x i64> %shuffle
 }
 
 define <8 x i64> @test_mm512_mask_shuffle_i64x2(<8 x i64> %__W, i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X32-LABEL: test_mm512_mask_shuffle_i64x2:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw %eax, %k1
 ; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3],zmm2[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shuffle_i64x2:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3],zmm2[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i64> %__A, <8 x i64> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   %0 = bitcast i8 %__U to <8 x i1>
   %1 = select <8 x i1> %0, <8 x i64> %shuffle, <8 x i64> %__W
   ret <8 x i64> %1
 }
 
 define <8 x i64> @test_mm512_maskz_shuffle_i64x2(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X32-LABEL: test_mm512_maskz_shuffle_i64x2:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw %eax, %k1
 ; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shuffle_i64x2:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3],zmm1[0,1,0,1]
 ; X64-NEXT:    retq
 entry:
   %shuffle = shufflevector <8 x i64> %__A, <8 x i64> %__B, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 8, i32 9>
   %0 = bitcast i8 %__U to <8 x i1>
   %1 = select <8 x i1> %0, <8 x i64> %shuffle, <8 x i64> zeroinitializer
   ret <8 x i64> %1
 }
 
 
 define zeroext i16 @test_mm512_testn_epi32_mask(<8 x i64> %__A, <8 x i64> %__B) {
 ; X32-LABEL: test_mm512_testn_epi32_mask:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    vptestnmd %zmm0, %zmm1, %k0
 ; X32-NEXT:    kmovw %k0, %eax
 ; X32-NEXT:    movzwl %ax, %eax
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_testn_epi32_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vptestnmd %zmm0, %zmm1, %k0
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
 entry:
   %and1.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and1.i.i to <16 x i32>
   %1 = icmp eq <16 x i32> %0, zeroinitializer
   %2 = bitcast <16 x i1> %1 to i16
   ret i16 %2
 }
 
 define zeroext i16 @test_mm512_mask_testn_epi32_mask(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X32-LABEL: test_mm512_mask_testn_epi32_mask:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vptestnmd %zmm0, %zmm1, %k0 {%k1}
 ; X32-NEXT:    kmovw %k0, %eax
 ; X32-NEXT:    movzwl %ax, %eax
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_testn_epi32_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vptestnmd %zmm0, %zmm1, %k0 {%k1}
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
 entry:
   %and1.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and1.i.i to <16 x i32>
   %1 = icmp eq <16 x i32> %0, zeroinitializer
   %2 = bitcast i16 %__U to <16 x i1>
   %3 = and <16 x i1> %1, %2
   %4 = bitcast <16 x i1> %3 to i16
   ret i16 %4
 }
 
 define zeroext i8 @test_mm512_testn_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) {
 ; X32-LABEL: test_mm512_testn_epi64_mask:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    vptestnmq %zmm0, %zmm1, %k0
 ; X32-NEXT:    kmovw %k0, %eax
 ; X32-NEXT:    movzbl %al, %eax
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_testn_epi64_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vptestnmq %zmm0, %zmm1, %k0
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    movzbl %al, %eax
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
 entry:
   %and1.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> %and1.i.i, zeroinitializer
   %1 = bitcast <8 x i1> %0 to i8
   ret i8 %1
 }
 
 define zeroext i8 @test_mm512_mask_testn_epi64_mask(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X32-LABEL: test_mm512_mask_testn_epi64_mask:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw %eax, %k1
 ; X32-NEXT:    vptestnmq %zmm0, %zmm1, %k0 {%k1}
 ; X32-NEXT:    kmovw %k0, %eax
 ; X32-NEXT:    movzbl %al, %eax
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_testn_epi64_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vptestnmq %zmm0, %zmm1, %k0 {%k1}
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    movzbl %al, %eax
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
 entry:
   %and1.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> %and1.i.i, zeroinitializer
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = and <8 x i1> %0, %1
   %3 = bitcast <8 x i1> %2 to i8
   ret i8 %3
 }
 
 define zeroext i16 @test_mm512_mask_test_epi32_mask(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X32-LABEL: test_mm512_mask_test_epi32_mask:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vptestmd %zmm0, %zmm1, %k0 {%k1}
 ; X32-NEXT:    kmovw %k0, %eax
 ; X32-NEXT:    movzwl %ax, %eax
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_test_epi32_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vptestmd %zmm0, %zmm1, %k0 {%k1}
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
 entry:
   %and1.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and1.i.i to <16 x i32>
   %1 = icmp ne <16 x i32> %0, zeroinitializer
   %2 = bitcast i16 %__U to <16 x i1>
   %3 = and <16 x i1> %1, %2
   %4 = bitcast <16 x i1> %3 to i16
   ret i16 %4
 }
 
 define zeroext i8 @test_mm512_mask_test_epi64_mask(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X32-LABEL: test_mm512_mask_test_epi64_mask:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw %eax, %k1
 ; X32-NEXT:    vptestmq %zmm0, %zmm1, %k0 {%k1}
 ; X32-NEXT:    kmovw %k0, %eax
 ; X32-NEXT:    movzbl %al, %eax
 ; X32-NEXT:    vzeroupper
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_test_epi64_mask:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vptestmq %zmm0, %zmm1, %k0 {%k1}
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    movzbl %al, %eax
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
 entry:
   %and1.i.i = and <8 x i64> %__B, %__A
   %0 = icmp ne <8 x i64> %and1.i.i, zeroinitializer
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = and <8 x i1> %0, %1
   %3 = bitcast <8 x i1> %2 to i8
   ret i8 %3
 }
 
 define <8 x i64> @test_mm512_mask_set1_epi32(<8 x i64> %__O, i16 zeroext %__M, i32 %__A) {
 ; X32-LABEL: test_mm512_mask_set1_epi32:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vpbroadcastd %eax, %zmm0 {%k1}
 ; X32-NEXT:    retl
 ;

test/CodeGen/X86/avx512-intrinsics-upgrade.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s
 
- define <16 x i32>@test_int_x86_avx512_mask_pbroadcastd_gpr_512(i32 %x0, <16 x i32> %x1, i16 %mask) {
+declare i16 @llvm.x86.avx512.kunpck.bw(i16, i16) nounwind readnone
+
+define i16 @unpckbw_test(i16 %a0, i16 %a1) {
+; CHECK-LABEL: unpckbw_test:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    shll $8, %esi
+; CHECK-NEXT:    orl %esi, %eax
+; CHECK-NEXT:    ## kill: %ax<def> %ax<kill> %eax<kill>
+; CHECK-NEXT:    retq
+  %res = call i16 @llvm.x86.avx512.kunpck.bw(i16 %a0, i16 %a1)
+  ret i16 %res
+}
+
 ; CHECK-LABEL: test_int_x86_avx512_mask_pbroadcastd_gpr_512:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpbroadcastd %edi, %zmm1
 ; CHECK-NEXT:    kmovw %esi, %k1
 ; CHECK-NEXT:    vpbroadcastd %edi, %zmm0 {%k1}
 ; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
 ; CHECK-NEXT:    vpbroadcastd %edi, %zmm1 {%k1} {z}
 ; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0

test/CodeGen/X86/avx512-intrinsics.ll

 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    ## kill: %ax<def> %ax<kill> %eax<kill>
 ; CHECK-NEXT:    retq
   %t1 = call i16 @llvm.x86.avx512.kor.w(i16 %a0, i16 8)
   %t2 = call i16 @llvm.x86.avx512.kor.w(i16 %t1, i16 %a1)
   ret i16 %t2
 }
 
-declare i16 @llvm.x86.avx512.kunpck.bw(i16, i16) nounwind readnone
-
-define i16 @unpckbw_test(i16 %a0, i16 %a1) {
-; CHECK-LABEL: unpckbw_test:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k0
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    kunpckbw %k1, %k0, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    ## kill: %ax<def> %ax<kill> %eax<kill>
-; CHECK-NEXT:    retq
-  %res = call i16 @llvm.x86.avx512.kunpck.bw(i16 %a0, i16 %a1)
-  ret i16 %res
-}
-
 declare i16 @llvm.x86.avx512.kxnor.w(i16, i16) nounwind readnone
 ; TODO: the two kxnor instructions here a no op and should be elimintaed,
 ; probably by FoldConstantArithmetic in SelectionDAG.
 define i16 @test_kxnor(i16 %a0, i16 %a1) {
 ; CHECK-LABEL: test_kxnor:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    kmovw %esi, %k0
 ; CHECK-NEXT:    kmovw %edi, %k1

test/CodeGen/X86/avx512bw-intrinsics-fast-isel.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=X32
 ; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=X64
 
 ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512bw-builtins.c
 
+define i64 @test_mm512_kunpackd(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> %__C, <8 x i64> %__D, <8 x i64> %__E, <8 x i64> %__F) {
+; X32-LABEL: test_mm512_kunpackd:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    andl $-64, %esp
+; X32-NEXT:    subl $64, %esp
+; X32-NEXT:    vmovdqa64 136(%ebp), %zmm3
+; X32-NEXT:    vmovdqa64 72(%ebp), %zmm4
+; X32-NEXT:    vmovdqa64 8(%ebp), %zmm5
+; X32-NEXT:    vpcmpneqb %zmm0, %zmm1, %k0
+; X32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
+; X32-NEXT:    vpcmpneqb %zmm5, %zmm2, %k0
+; X32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
+; X32-NEXT:    kmovd {{[0-9]+}}(%esp), %k0
+; X32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    kunpckdq %k0, %k1, %k1
+; X32-NEXT:    vpcmpneqb %zmm3, %zmm4, %k0 {%k1}
+; X32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl %ebp, %esp
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_kunpackd:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vpcmpneqb %zmm0, %zmm1, %k0
+; X64-NEXT:    vpcmpneqb %zmm3, %zmm2, %k1
+; X64-NEXT:    kunpckdq %k0, %k1, %k1
+; X64-NEXT:    vpcmpneqb %zmm5, %zmm4, %k0 {%k1}
+; X64-NEXT:    kmovq %k0, %rax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %__B to <64 x i8>
+  %1 = bitcast <8 x i64> %__A to <64 x i8>
+  %2 = icmp ne <64 x i8> %0, %1
+  %3 = bitcast <64 x i1> %2 to i64
+  %4 = bitcast <8 x i64> %__C to <64 x i8>
+  %5 = bitcast <8 x i64> %__D to <64 x i8>
+  %6 = icmp ne <64 x i8> %4, %5
+  %7 = bitcast <64 x i1> %6 to i64
+  %and.i = and i64 %7, 4294967295
+  %shl.i = shl i64 %3, 32
+  %or.i = or i64 %and.i, %shl.i
+  %8 = bitcast <8 x i64> %__E to <64 x i8>
+  %9 = bitcast <8 x i64> %__F to <64 x i8>
+  %10 = icmp ne <64 x i8> %8, %9
+  %11 = bitcast i64 %or.i to <64 x i1>
+  %12 = and <64 x i1> %10, %11
+  %13 = bitcast <64 x i1> %12 to i64
+  ret i64 %13
+}
+
+define i32 @test_mm512_kunpackw(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> %__C, <8 x i64> %__D, <8 x i64> %__E, <8 x i64> %__F) {
+; X32-LABEL: test_mm512_kunpackw:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    andl $-64, %esp
+; X32-NEXT:    subl $64, %esp
+; X32-NEXT:    vmovdqa64 136(%ebp), %zmm3
+; X32-NEXT:    vpcmpneqw %zmm0, %zmm1, %k0
+; X32-NEXT:    vpcmpneqw 8(%ebp), %zmm2, %k1
+; X32-NEXT:    kunpckwd %k0, %k1, %k1
+; X32-NEXT:    vpcmpneqw 72(%ebp), %zmm3, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movl %ebp, %esp
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_kunpackw:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vpcmpneqw %zmm0, %zmm1, %k0
+; X64-NEXT:    vpcmpneqw %zmm3, %zmm2, %k1
+; X64-NEXT:    kunpckwd %k0, %k1, %k1
+; X64-NEXT:    vpcmpneqw %zmm5, %zmm4, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %__B to <32 x i16>
+  %1 = bitcast <8 x i64> %__A to <32 x i16>
+  %2 = icmp ne <32 x i16> %0, %1
+  %3 = bitcast <32 x i1> %2 to i32
+  %4 = bitcast <8 x i64> %__C to <32 x i16>
+  %5 = bitcast <8 x i64> %__D to <32 x i16>
+  %6 = icmp ne <32 x i16> %4, %5
+  %7 = bitcast <32 x i1> %6 to i32
+  %and.i = and i32 %7, 65535
+  %shl.i = shl i32 %3, 16
+  %or.i = or i32 %and.i, %shl.i
+  %8 = bitcast <8 x i64> %__E to <32 x i16>
+  %9 = bitcast <8 x i64> %__F to <32 x i16>
+  %10 = icmp ne <32 x i16> %8, %9
+  %11 = bitcast i32 %or.i to <32 x i1>
+  %12 = and <32 x i1> %10, %11
+  %13 = bitcast <32 x i1> %12 to i32
+  ret i32 %13
+}
+
+
 define <8 x i64> @test_mm512_mask_set1_epi8(<8 x i64> %__O, i64 %__M, i8 signext %__A)  {
 ; X32-LABEL: test_mm512_mask_set1_epi8:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    pushl %ebx
 ; X32-NEXT:    .cfi_def_cfa_offset 8
 ; X32-NEXT:    .cfi_offset %ebx, -8
 ; X32-NEXT:    vmovdqa64 %zmm0, %zmm3
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax

 ; X32-NEXT:    vpmovb2m %zmm0, %k0
 ; X32-NEXT:    vpmovm2b %k0, %zmm0
 ; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X32-NEXT:    vpmovm2b %k1, %zmm2
 ; X32-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]
 ; X32-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; X32-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
 ; X32-NEXT:    vpblendvb %ymm5, %ymm1, %ymm2, %ymm1
-; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; X32-NEXT:    vpmovb2m %zmm0, %k0
-; X32-NEXT:    vpmovm2b %k0, %zmm0
-; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X32-NEXT:    movl %eax, %ecx
 ; X32-NEXT:    shrl $30, %ecx
 ; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X32-NEXT:    vpmovm2b %k0, %zmm2
 ; X32-NEXT:    vpbroadcastw %xmm2, %xmm2
 ; X32-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; X32-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
 ; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X32-NEXT:    vpmovb2m %zmm0, %k0
 ; X32-NEXT:    shrl $31, %eax
 ; X32-NEXT:    kmovd %eax, %k1

 ; X32-NEXT:    vpmovb2m %zmm0, %k0
 ; X32-NEXT:    vpmovm2b %k0, %zmm0
 ; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X32-NEXT:    vpmovm2b %k2, %zmm2
 ; X32-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]
 ; X32-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; X32-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
 ; X32-NEXT:    vpblendvb %ymm4, %ymm1, %ymm2, %ymm1
-; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; X32-NEXT:    vpmovb2m %zmm0, %k0
-; X32-NEXT:    vpmovm2b %k0, %zmm0
-; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X32-NEXT:    movl %eax, %ecx
 ; X32-NEXT:    shrl $30, %ecx
 ; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X32-NEXT:    vpmovm2b %k0, %zmm2
 ; X32-NEXT:    vpbroadcastw %xmm2, %xmm2
 ; X32-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; X32-NEXT:    vpblendvb %ymm3, %ymm1, %ymm2, %ymm1
 ; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; X32-NEXT:    shrl $31, %eax
 ; X32-NEXT:    kmovd %eax, %k0
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al

test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512BW
 ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512F-32
 
+declare i32 @llvm.x86.avx512.kunpck.wd(i32, i32)
+
+define i32@test_int_x86_avx512_kunpck_wd(i32 %x0, i32 %x1) {
+; AVX512BW-LABEL: test_int_x86_avx512_kunpck_wd:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    movzwl %di, %eax
+; AVX512BW-NEXT:    shll $16, %esi
+; AVX512BW-NEXT:    orl %esi, %eax
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_int_x86_avx512_kunpck_wd:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    shll $16, %eax
+; AVX512F-32-NEXT:    orl %ecx, %eax
+; AVX512F-32-NEXT:    retl
+  %res = call i32 @llvm.x86.avx512.kunpck.wd(i32 %x0, i32 %x1)
+  ret i32 %res
+}
+
+declare i64 @llvm.x86.avx512.kunpck.dq(i64, i64)
+
+define i64@test_int_x86_avx512_kunpck_qd(i64 %x0, i64 %x1) {
+; AVX512BW-LABEL: test_int_x86_avx512_kunpck_qd:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    shlq $32, %rsi
+; AVX512BW-NEXT:    movq %rsi, %rax
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_int_x86_avx512_kunpck_qd:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; AVX512F-32-NEXT:    xorl %eax, %eax
+; AVX512F-32-NEXT:    retl
+  %res = call i64 @llvm.x86.avx512.kunpck.dq(i64 %x0, i64 %x1)
+  ret i64 %res
+}
+
 declare <64 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.512(i8, <64 x i8>, i64)
 
   define <64 x i8>@test_int_x86_avx512_mask_pbroadcast_b_gpr_512(i8 %x0, <64 x i8> %x1, i64 %mask) {
 ; AVX512BW-LABEL: test_int_x86_avx512_mask_pbroadcast_b_gpr_512:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    vpbroadcastb %edi, %zmm1
 ; AVX512BW-NEXT:    kmovq %rsi, %k1
 ; AVX512BW-NEXT:    vpbroadcastb %edi, %zmm0 {%k1}

test/CodeGen/X86/avx512bw-intrinsics.ll

 ; AVX512F-32-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
   %res = call  <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %x0, <64 x i8> %x1)
   %res1 = call  <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %x0, <64 x i8> %x2)
   %res2 = add  <8 x i64> %res, %res1
   ret  <8 x i64> %res2
 }
 
-declare i32 @llvm.x86.avx512.kunpck.wd(i32, i32)
-
-define i32@test_int_x86_avx512_kunpck_wd(i32 %x0, i32 %x1) {
-; AVX512BW-LABEL: test_int_x86_avx512_kunpck_wd:
-; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    kmovd %edi, %k0
-; AVX512BW-NEXT:    kmovd %esi, %k1
-; AVX512BW-NEXT:    kunpckwd %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    retq
-;
-; AVX512F-32-LABEL: test_int_x86_avx512_kunpck_wd:
-; AVX512F-32:       # %bb.0:
-; AVX512F-32-NEXT:    kmovw {{[0-9]+}}(%esp), %k0
-; AVX512F-32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; AVX512F-32-NEXT:    kunpckwd %k0, %k1, %k0
-; AVX512F-32-NEXT:    kmovd %k0, %eax
-; AVX512F-32-NEXT:    retl
-  %res = call i32 @llvm.x86.avx512.kunpck.wd(i32 %x0, i32 %x1)
-  ret i32 %res
-}
-
-declare i64 @llvm.x86.avx512.kunpck.dq(i64, i64)
-
-define i64@test_int_x86_avx512_kunpck_qd(i64 %x0, i64 %x1) {
-; AVX512BW-LABEL: test_int_x86_avx512_kunpck_qd:
-; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    kmovq %rdi, %k0
-; AVX512BW-NEXT:    kmovq %rsi, %k1
-; AVX512BW-NEXT:    kunpckdq %k1, %k0, %k0
-; AVX512BW-NEXT:    kmovq %k0, %rax
-; AVX512BW-NEXT:    retq
-;
-; AVX512F-32-LABEL: test_int_x86_avx512_kunpck_qd:
-; AVX512F-32:       # %bb.0:
-; AVX512F-32-NEXT:    subl $12, %esp
-; AVX512F-32-NEXT:    .cfi_def_cfa_offset 16
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k0
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; AVX512F-32-NEXT:    kunpckdq %k0, %k1, %k0
-; AVX512F-32-NEXT:    kmovq %k0, (%esp)
-; AVX512F-32-NEXT:    movl (%esp), %eax
-; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; AVX512F-32-NEXT:    addl $12, %esp
-; AVX512F-32-NEXT:    retl
-  %res = call i64 @llvm.x86.avx512.kunpck.dq(i64 %x0, i64 %x1)
-  ret i64 %res
-}
-
 declare i64 @llvm.x86.avx512.cvtb2mask.512(<64 x i8>)
 
 define i64@test_int_x86_avx512_cvtb2mask_512(<64 x i8> %x0) {
 ; AVX512BW-LABEL: test_int_x86_avx512_cvtb2mask_512:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    vpmovb2m %zmm0, %k0
 ; AVX512BW-NEXT:    kmovq %k0, %rax
 ; AVX512BW-NEXT:    retq