[X86] test/testn intrinsics lowering to IR. llvm part.

[uriel.k]

Summary

Added autoUpgrade support and removed builtins from llvm.

this commit will be submitted only after D38732 will submitted
as it depends on the IR code pattern that being folded to TESTM/NM DAG nodes.

There is another revision that deals the clang side: D38737

[RKSimon]

*-fast-isel.ll tests to match the clang *-builtins tests?

[uriel.k]

Added fast-isel tests.
Note that for avx512bw there is an open bug that responsible for that huge code size.

[craig.topper]

lib/IR/AutoUpgrade.cpp

829

'atleast' should be two words

831

Line up the last argument with the end of the opening parenthese above

843

These two lines are indented one space too far.

875–876

Move this bitcast into the ApplyX86MaskOn1BitsVec. The same one appears at the second call site.

1050

Extract space between Pred and comma

[RKSimon]

test/CodeGen/X86/avx512bw-intrinsics-fast-isel.ll

1908

Drop the attributes

1942

Drop the attributes

[craig.topper]

LGTM

lib/IR/AutoUpgrade.cpp

847

You can use Builder.getIntNTy(std::max(NumElts, 8U)) here and you won't need the Context argument. Builder already has a reference to the Context internally and getIntNTy is just a helper to call IntegerType::get using that reference

include/llvm/IR/IntrinsicsX86.td

def int_x86_avx_ptestz_256 : GCCBuiltin<"__builtin_ia32_ptestz256">,
         Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty,
                   llvm_v4i64_ty], [IntrNoMem]>;
   def int_x86_avx_ptestc_256 : GCCBuiltin<"__builtin_ia32_ptestc256">,
         Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty,
                   llvm_v4i64_ty], [IntrNoMem]>;
   def int_x86_avx_ptestnzc_256 : GCCBuiltin<"__builtin_ia32_ptestnzc256">,
         Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty,
                   llvm_v4i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_d_512 : GCCBuiltin<"__builtin_ia32_ptestmd512">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                  llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_q_512 : GCCBuiltin<"__builtin_ia32_ptestmq512">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                  llvm_i8_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_ptestm_b_128 : GCCBuiltin<"__builtin_ia32_ptestmb128">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty,
-                   llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_b_256 : GCCBuiltin<"__builtin_ia32_ptestmb256">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty,
-                   llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_b_512 : GCCBuiltin<"__builtin_ia32_ptestmb512">,
-        Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty,
-                   llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_d_128 : GCCBuiltin<"__builtin_ia32_ptestmd128">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty,
-                   llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_d_256 : GCCBuiltin<"__builtin_ia32_ptestmd256">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty,
-                   llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_q_128 : GCCBuiltin<"__builtin_ia32_ptestmq128">,
-        Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty,
-                   llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_q_256 : GCCBuiltin<"__builtin_ia32_ptestmq256">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty,
-                   llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_w_128 : GCCBuiltin<"__builtin_ia32_ptestmw128">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty,
-                   llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_w_256 : GCCBuiltin<"__builtin_ia32_ptestmw256">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty,
-                   llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestm_w_512 : GCCBuiltin<"__builtin_ia32_ptestmw512">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty,
-                   llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_ptestnm_b_128 : GCCBuiltin<"__builtin_ia32_ptestnmb128">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty,
-                   llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_b_256 : GCCBuiltin<"__builtin_ia32_ptestnmb256">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty,
-                   llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_b_512 : GCCBuiltin<"__builtin_ia32_ptestnmb512">,
-        Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty,
-                   llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_d_128 : GCCBuiltin<"__builtin_ia32_ptestnmd128">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty,
-                   llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_d_256 : GCCBuiltin<"__builtin_ia32_ptestnmd256">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty,
-                   llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_d_512 : GCCBuiltin<"__builtin_ia32_ptestnmd512">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty,
-                   llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_q_128 : GCCBuiltin<"__builtin_ia32_ptestnmq128">,
-        Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty,
-                   llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_q_256 : GCCBuiltin<"__builtin_ia32_ptestnmq256">,
-        Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty,
-                   llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_q_512 : GCCBuiltin<"__builtin_ia32_ptestnmq512">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty,
-                   llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_w_128 : GCCBuiltin<"__builtin_ia32_ptestnmw128">,
-        Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty,
-                   llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_w_256 : GCCBuiltin<"__builtin_ia32_ptestnmw256">,
-        Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty,
-                   llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_ptestnm_w_512 : GCCBuiltin<"__builtin_ia32_ptestnmw512">,
-        Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty,
-                   llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
 
   def int_x86_avx512_mask_fpclass_pd_128 :
          GCCBuiltin<"__builtin_ia32_fpclasspd128_mask">,
           Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty],
           [IntrNoMem]>;
   def int_x86_avx512_mask_fpclass_pd_256 :
          GCCBuiltin<"__builtin_ia32_fpclasspd256_mask">,
           Intrinsic<[llvm_i8_ty], [llvm_v4f64_ty, llvm_i32_ty, llvm_i8_ty],

lib/IR/AutoUpgrade.cpp

if (Name=="ssse3.pabs.b.128" || // Added in 6.0
       Name.startswith("avx512.mask.broadcastf") || // Added in 6.0
       Name.startswith("avx512.mask.broadcasti") || // Added in 6.0
       Name == "xop.vpcmov" || // Added in 3.8
       Name == "xop.vpcmov.256" || // Added in 5.0
       Name.startswith("avx512.mask.move.s") || // Added in 4.0
       Name.startswith("avx512.cvtmask2") || // Added in 5.0
       (Name.startswith("xop.vpcom") && // Added in 3.2
        F->arg_size() == 2) ||
+      Name.startswith("avx512.ptestm") || //Added in 6.0
+      Name.startswith("avx512.ptestnm") || //Added in 6.0
       Name.startswith("sse2.pavg") || // Added in 6.0
       Name.startswith("avx2.pavg") || // Added in 6.0
       Name.startswith("avx512.mask.pavg")) // Added in 6.0
     return true;
 
   return false;
 }
 
static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI,
   Value *Res = Builder.CreateSelect(Cmp, Op0, Op1);
 
   if (CI.getNumArgOperands() == 4)
     Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
 
   return Res;
 }
 
+// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
+static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder,Value *Vec, Value *Mask,
+                                     unsigned NumElts, LLVMContext &Context) {
+  const auto *C = dyn_cast<Constant>(Mask);
+  if (!C || !C->isAllOnesValue())
+    Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
+
+  if (NumElts < 8) {
+    uint32_t Indices[8];
+    for (unsigned i = 0; i != NumElts; ++i)
+      Indices[i] = i;
+    for (unsigned i = NumElts; i != 8; ++i)
+      Indices[i] = NumElts + i % NumElts;
+    Vec = Builder.CreateShuffleVector(Vec,
+                                      Constant::getNullValue(Vec->getType()),
+                                      Indices);
+  }
+  return Builder.CreateBitCast(Vec,
+                               IntegerType::get(Context, std::max(NumElts, 8U)));
+}
+
 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
                                    unsigned CC, bool Signed) {
   Value *Op0 = CI.getArgOperand(0);
   unsigned NumElts = Op0->getType()->getVectorNumElements();
 
   Value *Cmp;
   if (CC == 3) {
     Cmp = Constant::getNullValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts));
if (CC == 3) {
     case 4: Pred = ICmpInst::ICMP_NE;  break;
     case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
     case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
     }
     Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
   }
 
   Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);
-  const auto *C = dyn_cast<Constant>(Mask);
-  if (!C || !C->isAllOnesValue())
-    Cmp = Builder.CreateAnd(Cmp, getX86MaskVec(Builder, Mask, NumElts));
 
-  if (NumElts < 8) {
-    uint32_t Indices[8];
-    for (unsigned i = 0; i != NumElts; ++i)
-      Indices[i] = i;
-    for (unsigned i = NumElts; i != 8; ++i)
-      Indices[i] = NumElts + i % NumElts;
-    Cmp = Builder.CreateShuffleVector(Cmp,
-                                      Constant::getNullValue(Cmp->getType()),
-                                      Indices);
-  }
-  return Builder.CreateBitCast(Cmp, IntegerType::get(CI.getContext(),
-                                                     std::max(NumElts, 8U)));
+  return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask, NumElts, CI.getContext());
 }
 
 // Replace a masked intrinsic with an older unmasked intrinsic.
 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
                                     Intrinsic::ID IID) {
   Function *F = CI.getCalledFunction();
   Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);
   Value *Rep = Builder.CreateCall(Intrin,
if (!NewFn) {
     } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
       Type *ExtTy = Type::getInt32Ty(C);
       if (CI->getOperand(0)->getType()->isIntegerTy(8))
         ExtTy = Type::getInt64Ty(C);
       unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
                          ExtTy->getPrimitiveSizeInBits();
       Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
       Rep = Builder.CreateVectorSplat(NumElts, Rep);
-    } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))) {
+    } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
+                         Name.startswith("avx512.ptestnm"))) {
+      Value *Op0 = CI->getArgOperand(0);
+      Value *Op1 = CI->getArgOperand(1);
+      Value *Mask = CI->getArgOperand(2);
+      Rep = Builder.CreateAnd(Op0, Op1);
+      llvm::Type *Ty = Op0->getType();
+      Value *Zero = llvm::Constant::getNullValue(Ty);
+      ICmpInst::Predicate Pred =
+        Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
+      Rep = Builder.CreateICmp(Pred, Rep, Zero);
+      unsigned NumElts = Op0->getType()->getVectorNumElements();
+      Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask, NumElts,
+                                   CI->getContext());
+    } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
       unsigned NumElts =
           CI->getArgOperand(1)->getType()->getVectorNumElements();
       Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                           CI->getArgOperand(1));
     } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) {
       Type *I32Ty = Type::getInt32Ty(C);
       Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

lib/Target/X86/X86IntrinsicsInfo.h

   X86_INTRINSIC_DATA(avx512_psrli_d_512, VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx512_psrli_q_512, VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx512_psrli_w_512, VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx512_psrlv_d_512, INTR_TYPE_2OP, ISD::SRL, 0),
   X86_INTRINSIC_DATA(avx512_psrlv_q_512, INTR_TYPE_2OP, ISD::SRL, 0),
   X86_INTRINSIC_DATA(avx512_psrlv_w_128, INTR_TYPE_2OP, ISD::SRL, 0),
   X86_INTRINSIC_DATA(avx512_psrlv_w_256, INTR_TYPE_2OP, ISD::SRL, 0),
   X86_INTRINSIC_DATA(avx512_psrlv_w_512, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_b_128, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_b_256, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_b_512, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_d_128, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_d_256, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_d_512, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_q_128, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_q_256, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_q_512, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_w_128, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_w_256, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestm_w_512, CMP_MASK, X86ISD::TESTM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_b_128, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_b_256, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_b_512, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_d_128, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_d_256, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_d_512, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_q_128, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_q_256, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_q_512, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_w_128, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_w_256, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_ptestnm_w_512, CMP_MASK, X86ISD::TESTNM, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_pd_128, INTR_TYPE_1OP_MASK, X86ISD::RCP14, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_pd_256, INTR_TYPE_1OP_MASK, X86ISD::RCP14, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_pd_512, INTR_TYPE_1OP_MASK, X86ISD::RCP14, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_ps_128, INTR_TYPE_1OP_MASK, X86ISD::RCP14, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_ps_256, INTR_TYPE_1OP_MASK, X86ISD::RCP14, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_ps_512, INTR_TYPE_1OP_MASK, X86ISD::RCP14, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_sd, INTR_TYPE_SCALAR_MASK, X86ISD::RCP14S, 0),
-  X86_INTRINSIC_DATA(avx512_rcp14_ss, INTR_TYPE_SCALAR_MASK, X86ISD::RCP14S, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_pd_128, INTR_TYPE_1OP_MASK, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_pd_256, INTR_TYPE_1OP_MASK, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_pd_512, INTR_TYPE_1OP_MASK, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_ps_128, INTR_TYPE_1OP_MASK, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_ps_256, INTR_TYPE_1OP_MASK, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_ps_512, INTR_TYPE_1OP_MASK, X86ISD::FRCP, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_sd, INTR_TYPE_SCALAR_MASK, X86ISD::FRCPS, 0),
+  X86_INTRINSIC_DATA(avx512_rcp14_ss, INTR_TYPE_SCALAR_MASK, X86ISD::FRCPS, 0),
   X86_INTRINSIC_DATA(avx512_rcp28_pd, INTR_TYPE_1OP_MASK_RM, X86ISD::RCP28, 0),
   X86_INTRINSIC_DATA(avx512_rcp28_ps, INTR_TYPE_1OP_MASK_RM, X86ISD::RCP28, 0),
   X86_INTRINSIC_DATA(avx512_rcp28_sd, INTR_TYPE_SCALAR_MASK_RM, X86ISD::RCP28S, 0),
   X86_INTRINSIC_DATA(avx512_rcp28_ss, INTR_TYPE_SCALAR_MASK_RM, X86ISD::RCP28S, 0),
   X86_INTRINSIC_DATA(avx512_rsqrt14_pd_128, INTR_TYPE_1OP_MASK, X86ISD::RSQRT14, 0),
   X86_INTRINSIC_DATA(avx512_rsqrt14_pd_256, INTR_TYPE_1OP_MASK, X86ISD::RSQRT14, 0),
   X86_INTRINSIC_DATA(avx512_rsqrt14_pd_512, INTR_TYPE_1OP_MASK, X86ISD::RSQRT14, 0),
   X86_INTRINSIC_DATA(avx512_rsqrt14_ps_128, INTR_TYPE_1OP_MASK, X86ISD::RSQRT14, 0),

test/CodeGen/X86/avx512-intrinsics-fast-isel.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=X32
 ; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=X64
 
 ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512f-builtins.c
 
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm512_testn_epi32_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #2 {
+; X32-LABEL: test_mm512_testn_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmd %zmm0, %zmm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_testn_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmd %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <16 x i32>
+  %1 = icmp eq <16 x i32> %0, zeroinitializer
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm512_mask_testn_epi32_mask(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #2 {
+; X32-LABEL: test_mm512_mask_testn_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestnmd %zmm0, %zmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_testn_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestnmd %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <16 x i32>
+  %1 = icmp eq <16 x i32> %0, zeroinitializer
+  %2 = bitcast i16 %__U to <16 x i1>
+  %3 = and <16 x i1> %1, %2
+  %4 = bitcast <16 x i1> %3 to i16
+  ret i16 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm512_testn_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #2 {
+; X32-LABEL: test_mm512_testn_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmq %zmm0, %zmm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_testn_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmq %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = icmp eq <8 x i64> %and1.i.i, zeroinitializer
+  %1 = bitcast <8 x i1> %0 to i8
+  ret i8 %1
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm512_mask_testn_epi64_mask(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #2 {
+; X32-LABEL: test_mm512_mask_testn_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestnmq %zmm0, %zmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_testn_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestnmq %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = icmp eq <8 x i64> %and1.i.i, zeroinitializer
+  %1 = bitcast i8 %__U to <8 x i1>
+  %2 = and <8 x i1> %0, %1
+  %3 = bitcast <8 x i1> %2 to i8
+  ret i8 %3
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm512_mask_test_epi32_mask(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #2 {
+; X32-LABEL: test_mm512_mask_test_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestmd %zmm0, %zmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_test_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestmd %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <16 x i32>
+  %1 = icmp ne <16 x i32> %0, zeroinitializer
+  %2 = bitcast i16 %__U to <16 x i1>
+  %3 = and <16 x i1> %1, %2
+  %4 = bitcast <16 x i1> %3 to i16
+  ret i16 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm512_mask_test_epi64_mask(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #2 {
+; X32-LABEL: test_mm512_mask_test_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestmq %zmm0, %zmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_test_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestmq %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = icmp ne <8 x i64> %and1.i.i, zeroinitializer
+  %1 = bitcast i8 %__U to <8 x i1>
+  %2 = and <8 x i1> %0, %1
+  %3 = bitcast <8 x i1> %2 to i8
+  ret i8 %3
+}
+
 define <8 x i64> @test_mm512_mask_set1_epi32(<8 x i64> %__O, i16 zeroext %__M, i32 %__A) {
 ; X32-LABEL: test_mm512_mask_set1_epi32:
 ; X32:       # BB#0: # %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vpbroadcastd %eax, %zmm0 {%k1}
 ; X32-NEXT:    retl
 ;

test/CodeGen/X86/avx512-intrinsics-upgrade.ll

 ; CHECK-NEXT:    vpaddq %zmm2, %zmm1, %zmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
   %res1 = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 -1)
   %res2 = add <8 x i64> %res, %res1
   ret <8 x i64> %res2
 }
 
+define i8 @test_vptestmq(<8 x i64> %a0, <8 x i64> %a1, i8 %m) {
+; CHECK-LABEL: test_vptestmq:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmq %zmm1, %zmm0, %k0
+; CHECK-NEXT:    kmovw %k0, %ecx
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vptestmq %zmm1, %zmm0, %k0 {%k1}
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    addb %cl, %al
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq
+  %res = call i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 -1)
+  %res1 = call i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 %m)
+  %res2 = add i8 %res1, %res
+  ret i8 %res2
+}
+declare i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64>, <8 x i64>, i8)
+
+define i16 @test_vptestmd(<16 x i32> %a0, <16 x i32> %a1, i16 %m) {
+; CHECK-LABEL: test_vptestmd:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmd %zmm1, %zmm0, %k0
+; CHECK-NEXT:    kmovw %k0, %ecx
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vptestmd %zmm1, %zmm0, %k0 {%k1}
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
+; CHECK-NEXT:    retq
+  %res = call i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32> %a0, <16 x i32> %a1, i16 -1)
+  %res1 = call i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32> %a0, <16 x i32> %a1, i16 %m)
+  %res2 = add i16 %res1, %res
+  ret i16 %res2
+}
+declare i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32>, <16 x i32>, i16)
+
+declare i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32>, <16 x i32>, i16 %x2)
+
+define i16@test_int_x86_avx512_ptestnm_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_d_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmd %zmm1, %zmm0, %k0
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vptestnmd %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    kmovw %k1, %ecx
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
+; CHECK-NEXT:    retq
+  %res = call i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)
+  %res1 = call i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32> %x0, <16 x i32> %x1, i16-1)
+  %res2 = add i16 %res, %res1
+  ret i16 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64>, <8 x i64>, i8 %x2)
+
+define i8@test_int_x86_avx512_ptestnm_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_q_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmq %zmm1, %zmm0, %k0
+; CHECK-NEXT:    kmovw %edi, %k1
+; CHECK-NEXT:    vptestnmq %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    kmovw %k1, %ecx
+; CHECK-NEXT:    kmovw %k0, %eax
+; CHECK-NEXT:    addb %cl, %al
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq
+  %res = call i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64> %x0, <8 x i64> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+

test/CodeGen/X86/avx512-intrinsics.ll

 ; CHECK-NEXT:    retq
   %res = call <8 x double> @llvm.x86.avx512.mask.min.pd.512(<8 x double> %a0, <8 x double> %a1,
                     <8 x double>zeroinitializer, i8 -1, i32 4)
   ret <8 x double> %res
 }
 declare <8 x double> @llvm.x86.avx512.mask.min.pd.512(<8 x double>, <8 x double>,
                     <8 x double>, i8, i32)
 
-define i8 @test_vptestmq(<8 x i64> %a0, <8 x i64> %a1, i8 %m) {
-; CHECK-LABEL: test_vptestmq:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vptestmq %zmm1, %zmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    vptestmq %zmm1, %zmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    addb %cl, %al
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq
-  %res = call i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 -1)
-  %res1 = call i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 %m)
-  %res2 = add i8 %res1, %res
-  ret i8 %res2
-}
-declare i8 @llvm.x86.avx512.ptestm.q.512(<8 x i64>, <8 x i64>, i8)
-
-define i16 @test_vptestmd(<16 x i32> %a0, <16 x i32> %a1, i16 %m) {
-; CHECK-LABEL: test_vptestmd:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vptestmd %zmm1, %zmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    vptestmd %zmm1, %zmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
-; CHECK-NEXT:    retq
-  %res = call i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32> %a0, <16 x i32> %a1, i16 -1)
-  %res1 = call i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32> %a0, <16 x i32> %a1, i16 %m)
-  %res2 = add i16 %res1, %res
-  ret i16 %res2
-}
-declare i16 @llvm.x86.avx512.ptestm.d.512(<16 x i32>, <16 x i32>, i16)
-
 define void @test_mask_store_ss(i8* %ptr, <4 x float> %data, i8 %mask) {
 ; CHECK-LABEL: test_mask_store_ss:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    kmovw %esi, %k1
 ; CHECK-NEXT:    vmovss %xmm0, (%rdi) {%k1}
 ; CHECK-NEXT:    retq
  call void @llvm.x86.avx512.mask.store.ss(i8* %ptr, <4 x float> %data, i8 %mask)
  ret void
; CHECK-NEXT:    retq
   %res = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> zeroinitializer, i32 5, i8 %x4, i32 8)
   %res2 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 8)
   %res3 = fadd <2 x double> %res, %res1
   %res4 = fadd <2 x double> %res3, %res2
   ret <2 x double> %res4
 }
 
-declare i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32>, <16 x i32>, i16 %x2)
-
-define i16@test_int_x86_avx512_ptestnm_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_d_512:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vptestnmd %zmm1, %zmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    vptestnmd %zmm1, %zmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
-; CHECK-NEXT:    retq
-  %res = call i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)
-  %res1 = call i16 @llvm.x86.avx512.ptestnm.d.512(<16 x i32> %x0, <16 x i32> %x1, i16-1)
-  %res2 = add i16 %res, %res1
-  ret i16 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64>, <8 x i64>, i8 %x2)
-
-define i8@test_int_x86_avx512_ptestnm_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_q_512:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vptestnmq %zmm1, %zmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    vptestnmq %zmm1, %zmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    addb %cl, %al
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq
-  %res = call i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestnm.q.512(<8 x i64> %x0, <8 x i64> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-
-
-
 declare <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
 
 define <2 x double>@test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
 ; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vmovapd %xmm0, %xmm3
 ; CHECK-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm3
 ; CHECK-NEXT:    kmovw %edi, %k1

test/CodeGen/X86/avx512bw-intrinsics-fast-isel.ll

; X64-NEXT:    retq
   %arg1 = bitcast <8 x i64> %a1 to <32 x i16>
   %arg2 = bitcast <8 x i64> %a2 to <32 x i16>
   %res0 = shufflevector <32 x i16> %arg1, <32 x i16> %arg2, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
   %res1 = select <32 x i1> %arg0, <32 x i16> %res0, <32 x i16> zeroinitializer
   %res2 = bitcast <32 x i16> %res1 to <8 x i64>
   ret <8 x i64> %res2
 }
 
+; Function Attrs: norecurse nounwind readnone
+define i64 @test_mm512_test_epi8_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_test_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    andl $-8, %esp
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    vptestmb %zmm0, %zmm1, %k0
+; X32-NEXT:    kmovq %k0, (%esp)
+; X32-NEXT:    movl (%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl %ebp, %esp
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_test_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmb %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovq %k0, %rax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <64 x i8>
+  %1 = icmp ne <64 x i8> %0, zeroinitializer
+  %2 = bitcast <64 x i1> %1 to i64
+  ret i64 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i64 @test_mm512_mask_test_epi8_mask(i64 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_mask_test_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    pushl %ebx
+; X32-NEXT:    andl $-64, %esp
+; X32-NEXT:    subl $256, %esp # imm = 0x100
+; X32-NEXT:    .cfi_offset %ebx, -12
+; X32-NEXT:    vmovaps %zmm1, {{[0-9]+}}(%esp) # 64-byte Spill
+; X32-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%esp) # 64-byte Spill
+; X32-NEXT:    movl 8(%ebp), %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    kmovd %eax, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpbroadcastw %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpslld $24, %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $4, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpbroadcastd %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $5, %cl
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpsllq $40, %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $6, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $12, %ecx
+; X32-NEXT:    andl $15, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $13, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $14, %ecx
+; X32-NEXT:    andl $3, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $15, %ecx
+; X32-NEXT:    andl $1, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $16, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $15, %dl
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    shrb $2, %bl
+; X32-NEXT:    kmovd %ebx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslld $24, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $4, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $5, %dl
+; X32-NEXT:    andb $1, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $40, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $6, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $24, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm5, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $28, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
+; X32-NEXT:    vpblendvb %ymm4, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $29, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
+; X32-NEXT:    vpblendvb %ymm3, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $30, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrl $31, %eax
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0]
+; X32-NEXT:    vpblendvb %ymm7, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl 12(%ebp), %eax
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    vpmovm2b %k1, %zmm7
+; X32-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm6, %ymm1, %ymm7, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslld $24, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $4, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $5, %cl
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $40, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $6, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $12, %ecx
+; X32-NEXT:    andl $15, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $13, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $14, %ecx
+; X32-NEXT:    andl $3, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $15, %ecx
+; X32-NEXT:    andl $1, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $16, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $15, %dl
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    shrb $2, %bl
+; X32-NEXT:    kmovd %ebx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslld $24, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $4, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $5, %dl
+; X32-NEXT:    andb $1, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $40, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $6, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $24, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k0
+; X32-NEXT:    vpmovb2m %zmm0, %k1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k1, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4]
+; X32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
+; X32-NEXT:    vpblendvb %ymm5, %ymm6, %ymm1, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $28, %ecx
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm5
+; X32-NEXT:    vpbroadcastd %xmm5, %xmm5
+; X32-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; X32-NEXT:    vpblendvb %ymm4, %ymm1, %ymm5, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $29, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm4
+; X32-NEXT:    vpslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2]
+; X32-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; X32-NEXT:    vpblendvb %ymm3, %ymm1, %ymm4, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $30, %ecx
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm3
+; X32-NEXT:    vpbroadcastw %xmm3, %xmm3
+; X32-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm3, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    kshiftlq $1, %k0, %k0
+; X32-NEXT:    kshiftrq $1, %k0, %k0
+; X32-NEXT:    shrl $31, %eax
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    kshiftlq $63, %k1, %k1
+; X32-NEXT:    korq %k1, %k0, %k1
+; X32-NEXT:    vmovaps {{[0-9]+}}(%esp), %zmm0 # 64-byte Reload
+; X32-NEXT:    vmovaps {{[0-9]+}}(%esp), %zmm1 # 64-byte Reload
+; X32-NEXT:    vptestmb %zmm1, %zmm0, %k0 {%k1}
+; X32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    leal -4(%ebp), %esp
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_test_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovq %rdi, %k1
+; X64-NEXT:    vptestmb %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovq %k0, %rax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <64 x i8>
+  %1 = icmp ne <64 x i8> %0, zeroinitializer
+  %2 = bitcast i64 %__U to <64 x i1>
+  %3 = and <64 x i1> %1, %2
+  %4 = bitcast <64 x i1> %3 to i64
+  ret i64 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm512_test_epi16_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_test_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmw %zmm0, %zmm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_test_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmw %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <32 x i16>
+  %1 = icmp ne <32 x i16> %0, zeroinitializer
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm512_mask_test_epi16_mask(i32 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_mask_test_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestmw %zmm0, %zmm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_test_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestmw %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <32 x i16>
+  %1 = icmp ne <32 x i16> %0, zeroinitializer
+  %2 = bitcast i32 %__U to <32 x i1>
+  %3 = and <32 x i1> %1, %2
+  %4 = bitcast <32 x i1> %3 to i32
+  ret i32 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i64 @test_mm512_testn_epi8_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_testn_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    andl $-8, %esp
+; X32-NEXT:    subl $8, %esp
+; X32-NEXT:    vptestnmb %zmm0, %zmm1, %k0
+; X32-NEXT:    kmovq %k0, (%esp)
+; X32-NEXT:    movl (%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl %ebp, %esp
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_testn_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmb %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovq %k0, %rax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <64 x i8>
+  %1 = icmp eq <64 x i8> %0, zeroinitializer
+  %2 = bitcast <64 x i1> %1 to i64
+  ret i64 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i64 @test_mm512_mask_testn_epi8_mask(i64 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_mask_testn_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %ebp, -8
+; X32-NEXT:    movl %esp, %ebp
+; X32-NEXT:    .cfi_def_cfa_register %ebp
+; X32-NEXT:    pushl %ebx
+; X32-NEXT:    andl $-64, %esp
+; X32-NEXT:    subl $256, %esp # imm = 0x100
+; X32-NEXT:    .cfi_offset %ebx, -12
+; X32-NEXT:    vmovaps %zmm1, {{[0-9]+}}(%esp) # 64-byte Spill
+; X32-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%esp) # 64-byte Spill
+; X32-NEXT:    movl 8(%ebp), %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    kmovd %eax, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpbroadcastw %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpslld $24, %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $4, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpbroadcastd %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $5, %cl
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpsllq $40, %xmm1, %xmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $6, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $12, %ecx
+; X32-NEXT:    andl $15, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $13, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $14, %ecx
+; X32-NEXT:    andl $3, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $15, %ecx
+; X32-NEXT:    andl $1, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $16, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $15, %dl
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    shrb $2, %bl
+; X32-NEXT:    kmovd %ebx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslld $24, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $4, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $5, %dl
+; X32-NEXT:    andb $1, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $40, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $6, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $24, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm5, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $28, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255]
+; X32-NEXT:    vpblendvb %ymm4, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $29, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255]
+; X32-NEXT:    vpblendvb %ymm3, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $30, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255]
+; X32-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm1
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrl $31, %eax
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0]
+; X32-NEXT:    vpblendvb %ymm7, %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[4,5,6,7]
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl 12(%ebp), %eax
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    vpmovm2b %k1, %zmm7
+; X32-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm6, %ymm1, %ymm7, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslld $24, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $4, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $5, %cl
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $40, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $6, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    andb $2, %cl
+; X32-NEXT:    shrb %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movb %ah, %cl
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $12, %ecx
+; X32-NEXT:    andl $15, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $13, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $14, %ecx
+; X32-NEXT:    andl $3, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $15, %ecx
+; X32-NEXT:    andl $1, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0]
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $16, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllw $8, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $15, %dl
+; X32-NEXT:    movl %edx, %ebx
+; X32-NEXT:    shrb $2, %bl
+; X32-NEXT:    kmovd %ebx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $3, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslld $24, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $4, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $5, %dl
+; X32-NEXT:    andb $1, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $40, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $6, %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    shrb $7, %cl
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpsllq $56, %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $24, %ecx
+; X32-NEXT:    kmovd %ecx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpbroadcastq %xmm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    andb $2, %dl
+; X32-NEXT:    shrb %dl
+; X32-NEXT:    kmovd %edx, %k1
+; X32-NEXT:    vpmovm2b %k1, %zmm0
+; X32-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    andb $15, %cl
+; X32-NEXT:    movl %ecx, %edx
+; X32-NEXT:    shrb $2, %dl
+; X32-NEXT:    kmovd %edx, %k0
+; X32-NEXT:    vpmovb2m %zmm0, %k1
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vpbroadcastw %xmm0, %xmm0
+; X32-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    vpmovm2b %k1, %zmm1
+; X32-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; X32-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255]
+; X32-NEXT:    vpblendvb %ymm7, %ymm6, %ymm0, %ymm0
+; X32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    shrb $3, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm1
+; X32-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4]
+; X32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm6
+; X32-NEXT:    vpblendvb %ymm5, %ymm6, %ymm1, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $28, %ecx
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm5
+; X32-NEXT:    vpbroadcastd %xmm5, %xmm5
+; X32-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; X32-NEXT:    vpblendvb %ymm4, %ymm1, %ymm5, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $29, %ecx
+; X32-NEXT:    andb $1, %cl
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm4
+; X32-NEXT:    vpslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2]
+; X32-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; X32-NEXT:    vpblendvb %ymm3, %ymm1, %ymm4, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm0
+; X32-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $30, %ecx
+; X32-NEXT:    kmovd %ecx, %k0
+; X32-NEXT:    vpmovm2b %k0, %zmm3
+; X32-NEXT:    vpbroadcastw %xmm3, %xmm3
+; X32-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; X32-NEXT:    vpblendvb %ymm2, %ymm1, %ymm3, %ymm1
+; X32-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-NEXT:    vpmovb2m %zmm0, %k0
+; X32-NEXT:    kshiftlq $1, %k0, %k0
+; X32-NEXT:    kshiftrq $1, %k0, %k0
+; X32-NEXT:    shrl $31, %eax
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    kshiftlq $63, %k1, %k1
+; X32-NEXT:    korq %k1, %k0, %k1
+; X32-NEXT:    vmovaps {{[0-9]+}}(%esp), %zmm0 # 64-byte Reload
+; X32-NEXT:    vmovaps {{[0-9]+}}(%esp), %zmm1 # 64-byte Reload
+; X32-NEXT:    vptestnmb %zmm1, %zmm0, %k0 {%k1}
+; X32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    leal -4(%ebp), %esp
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_testn_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovq %rdi, %k1
+; X64-NEXT:    vptestnmb %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovq %k0, %rax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <64 x i8>
+  %1 = icmp eq <64 x i8> %0, zeroinitializer
+  %2 = bitcast i64 %__U to <64 x i1>
+  %3 = and <64 x i1> %1, %2
+  %4 = bitcast <64 x i1> %3 to i64
+  ret i64 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm512_testn_epi16_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_testn_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmw %zmm0, %zmm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_testn_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmw %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <32 x i16>
+  %1 = icmp eq <32 x i16> %0, zeroinitializer
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm512_mask_testn_epi16_mask(i32 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm512_mask_testn_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestnmw %zmm0, %zmm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_testn_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestnmw %zmm0, %zmm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and1.i.i = and <8 x i64> %__B, %__A
+  %0 = bitcast <8 x i64> %and1.i.i to <32 x i16>
+  %1 = icmp eq <32 x i16> %0, zeroinitializer
+  %2 = bitcast i32 %__U to <32 x i1>
+  %3 = and <32 x i1> %1, %2
+  %4 = bitcast <32 x i1> %3 to i32
+  ret i32 %4
+}
+
 !0 = !{i32 1}
 

test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll

 ; AVX512F-32-NEXT:    vpaddb %zmm2, %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
   %res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
   %res1 = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 -1)
   %res2 = add <64 x i8> %res, %res1
   ret <64 x i8> %res2
 }
 
+declare i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8>, <64 x i8>, i64)
+
+define i64@test_int_x86_avx512_ptestm_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) {
+; AVX512BW-LABEL: test_int_x86_avx512_ptestm_b_512:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    kmovq %rdi, %k1
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm0, %k1 {%k1}
+; AVX512BW-NEXT:    kmovq %k1, %rcx
+; AVX512BW-NEXT:    kmovq %k0, %rax
+; AVX512BW-NEXT:    addq %rcx, %rax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_int_x86_avx512_ptestm_b_512:
+; AVX512F-32:       # BB#0:
+; AVX512F-32-NEXT:    subl $20, %esp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 24
+; AVX512F-32-NEXT:    vptestmb %zmm1, %zmm0, %k0
+; AVX512F-32-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vptestmb %zmm1, %zmm0, %k1 {%k1}
+; AVX512F-32-NEXT:    kmovq %k1, (%esp)
+; AVX512F-32-NEXT:    movl (%esp), %eax
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; AVX512F-32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; AVX512F-32-NEXT:    addl $20, %esp
+; AVX512F-32-NEXT:    vzeroupper
+; AVX512F-32-NEXT:    retl
+  %res = call i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
+  %res1 = call i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64-1)
+  %res2 = add i64 %res, %res1
+  ret i64 %res2
+}
+
+declare i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16>, <32 x i16>, i32)
+
+define i32@test_int_x86_avx512_ptestm_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) {
+; AVX512BW-LABEL: test_int_x86_avx512_ptestm_w_512:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vptestmw %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vptestmw %zmm1, %zmm0, %k1 {%k1}
+; AVX512BW-NEXT:    kmovd %k1, %ecx
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    addl %ecx, %eax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_int_x86_avx512_ptestm_w_512:
+; AVX512F-32:       # BB#0:
+; AVX512F-32-NEXT:    vptestmw %zmm1, %zmm0, %k0
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vptestmw %zmm1, %zmm0, %k1 {%k1}
+; AVX512F-32-NEXT:    kmovd %k1, %ecx
+; AVX512F-32-NEXT:    kmovd %k0, %eax
+; AVX512F-32-NEXT:    addl %ecx, %eax
+; AVX512F-32-NEXT:    vzeroupper
+; AVX512F-32-NEXT:    retl
+  %res = call i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
+  %res1 = call i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32-1)
+  %res2 = add i32 %res, %res1
+  ret i32 %res2
+}
+
+declare i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8>, <64 x i8>, i64 %x2)
+
+define i64@test_int_x86_avx512_ptestnm_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) {
+; AVX512BW-LABEL: test_int_x86_avx512_ptestnm_b_512:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vptestnmb %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    kmovq %rdi, %k1
+; AVX512BW-NEXT:    vptestnmb %zmm1, %zmm0, %k1 {%k1}
+; AVX512BW-NEXT:    kmovq %k1, %rcx
+; AVX512BW-NEXT:    kmovq %k0, %rax
+; AVX512BW-NEXT:    addq %rcx, %rax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_int_x86_avx512_ptestnm_b_512:
+; AVX512F-32:       # BB#0:
+; AVX512F-32-NEXT:    subl $20, %esp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 24
+; AVX512F-32-NEXT:    vptestnmb %zmm1, %zmm0, %k0
+; AVX512F-32-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vptestnmb %zmm1, %zmm0, %k1 {%k1}
+; AVX512F-32-NEXT:    kmovq %k1, (%esp)
+; AVX512F-32-NEXT:    movl (%esp), %eax
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; AVX512F-32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
+; AVX512F-32-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; AVX512F-32-NEXT:    addl $20, %esp
+; AVX512F-32-NEXT:    vzeroupper
+; AVX512F-32-NEXT:    retl
+  %res = call i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
+  %res1 = call i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64-1)
+  %res2 = add i64 %res, %res1
+  ret i64 %res2
+}
+
+declare i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16>, <32 x i16>, i32 %x2)
+
+define i32@test_int_x86_avx512_ptestnm_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) {
+; AVX512BW-LABEL: test_int_x86_avx512_ptestnm_w_512:
+; AVX512BW:       ## BB#0:
+; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm0, %k0
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm0, %k1 {%k1}
+; AVX512BW-NEXT:    kmovd %k1, %ecx
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    addl %ecx, %eax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_int_x86_avx512_ptestnm_w_512:
+; AVX512F-32:       # BB#0:
+; AVX512F-32-NEXT:    vptestnmw %zmm1, %zmm0, %k0
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vptestnmw %zmm1, %zmm0, %k1 {%k1}
+; AVX512F-32-NEXT:    kmovd %k1, %ecx
+; AVX512F-32-NEXT:    kmovd %k0, %eax
+; AVX512F-32-NEXT:    addl %ecx, %eax
+; AVX512F-32-NEXT:    vzeroupper
+; AVX512F-32-NEXT:    retl
+  %res = call i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
+  %res1 = call i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32-1)
+  %res2 = add i32 %res, %res1
+  ret i32 %res2
+}
+

test/CodeGen/X86/avx512bw-intrinsics.ll

; AVX512F-32-NEXT:    retl
   %res = call <32 x i16> @llvm.x86.avx512.mask.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
   %res1 = call <32 x i16> @llvm.x86.avx512.mask.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
   %res2 = call <32 x i16> @llvm.x86.avx512.mask.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
   %res3 = add <32 x i16> %res, %res1
   %res4 = add <32 x i16> %res3, %res2
   ret <32 x i16> %res4
 }
 
-declare i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8>, <64 x i8>, i64)
-
-define i64@test_int_x86_avx512_ptestm_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) {
-; AVX512BW-LABEL: test_int_x86_avx512_ptestm_b_512:
-; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    kmovq %rdi, %k1
-; AVX512BW-NEXT:    vptestmb %zmm1, %zmm0, %k0 {%k1}
-; AVX512BW-NEXT:    kmovq %k0, %rcx
-; AVX512BW-NEXT:    vptestmb %zmm1, %zmm0, %k0
-; AVX512BW-NEXT:    kmovq %k0, %rax
-; AVX512BW-NEXT:    addq %rcx, %rax
-; AVX512BW-NEXT:    retq
-;
-; AVX512F-32-LABEL: test_int_x86_avx512_ptestm_b_512:
-; AVX512F-32:       # BB#0:
-; AVX512F-32-NEXT:    subl $20, %esp
-; AVX512F-32-NEXT:    .cfi_def_cfa_offset 24
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k0
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; AVX512F-32-NEXT:    kunpckdq %k0, %k1, %k1
-; AVX512F-32-NEXT:    vptestmb %zmm1, %zmm0, %k0 {%k1}
-; AVX512F-32-NEXT:    kmovq %k0, (%esp)
-; AVX512F-32-NEXT:    vptestmb %zmm1, %zmm0, %k0
-; AVX512F-32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
-; AVX512F-32-NEXT:    movl (%esp), %eax
-; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; AVX512F-32-NEXT:    addl {{[0-9]+}}(%esp), %eax
-; AVX512F-32-NEXT:    adcxl {{[0-9]+}}(%esp), %edx
-; AVX512F-32-NEXT:    addl $20, %esp
-; AVX512F-32-NEXT:    retl
-  %res = call i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
-  %res1 = call i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64-1)
-  %res2 = add i64 %res, %res1
-  ret i64 %res2
-}
-
-declare i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16>, <32 x i16>, i32)
-
-define i32@test_int_x86_avx512_ptestm_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) {
-; AVX512BW-LABEL: test_int_x86_avx512_ptestm_w_512:
-; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    kmovd %edi, %k1
-; AVX512BW-NEXT:    vptestmw %zmm1, %zmm0, %k0 {%k1}
-; AVX512BW-NEXT:    kmovd %k0, %ecx
-; AVX512BW-NEXT:    vptestmw %zmm1, %zmm0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    addl %ecx, %eax
-; AVX512BW-NEXT:    retq
-;
-; AVX512F-32-LABEL: test_int_x86_avx512_ptestm_w_512:
-; AVX512F-32:       # BB#0:
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; AVX512F-32-NEXT:    vptestmw %zmm1, %zmm0, %k0 {%k1}
-; AVX512F-32-NEXT:    kmovd %k0, %ecx
-; AVX512F-32-NEXT:    vptestmw %zmm1, %zmm0, %k0
-; AVX512F-32-NEXT:    kmovd %k0, %eax
-; AVX512F-32-NEXT:    addl %ecx, %eax
-; AVX512F-32-NEXT:    retl
-  %res = call i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
-  %res1 = call i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32-1)
-  %res2 = add i32 %res, %res1
-  ret i32 %res2
-}
-
-declare i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8>, <64 x i8>, i64 %x2)
-
-define i64@test_int_x86_avx512_ptestnm_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) {
-; AVX512BW-LABEL: test_int_x86_avx512_ptestnm_b_512:
-; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    kmovq %rdi, %k1
-; AVX512BW-NEXT:    vptestnmb %zmm1, %zmm0, %k0 {%k1}
-; AVX512BW-NEXT:    kmovq %k0, %rcx
-; AVX512BW-NEXT:    vptestnmb %zmm1, %zmm0, %k0
-; AVX512BW-NEXT:    kmovq %k0, %rax
-; AVX512BW-NEXT:    addq %rcx, %rax
-; AVX512BW-NEXT:    retq
-;
-; AVX512F-32-LABEL: test_int_x86_avx512_ptestnm_b_512:
-; AVX512F-32:       # BB#0:
-; AVX512F-32-NEXT:    subl $20, %esp
-; AVX512F-32-NEXT:    .cfi_def_cfa_offset 24
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k0
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; AVX512F-32-NEXT:    kunpckdq %k0, %k1, %k1
-; AVX512F-32-NEXT:    vptestnmb %zmm1, %zmm0, %k0 {%k1}
-; AVX512F-32-NEXT:    kmovq %k0, (%esp)
-; AVX512F-32-NEXT:    vptestnmb %zmm1, %zmm0, %k0
-; AVX512F-32-NEXT:    kmovq %k0, {{[0-9]+}}(%esp)
-; AVX512F-32-NEXT:    movl (%esp), %eax
-; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; AVX512F-32-NEXT:    addl {{[0-9]+}}(%esp), %eax
-; AVX512F-32-NEXT:    adcxl {{[0-9]+}}(%esp), %edx
-; AVX512F-32-NEXT:    addl $20, %esp
-; AVX512F-32-NEXT:    retl
-  %res = call i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
-  %res1 = call i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64-1)
-  %res2 = add i64 %res, %res1
-  ret i64 %res2
-}
-
-declare i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16>, <32 x i16>, i32 %x2)
-
-define i32@test_int_x86_avx512_ptestnm_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) {
-; AVX512BW-LABEL: test_int_x86_avx512_ptestnm_w_512:
-; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    kmovd %edi, %k1
-; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm0, %k0 {%k1}
-; AVX512BW-NEXT:    kmovd %k0, %ecx
-; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm0, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    addl %ecx, %eax
-; AVX512BW-NEXT:    retq
-;
-; AVX512F-32-LABEL: test_int_x86_avx512_ptestnm_w_512:
-; AVX512F-32:       # BB#0:
-; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; AVX512F-32-NEXT:    vptestnmw %zmm1, %zmm0, %k0 {%k1}
-; AVX512F-32-NEXT:    kmovd %k0, %ecx
-; AVX512F-32-NEXT:    vptestnmw %zmm1, %zmm0, %k0
-; AVX512F-32-NEXT:    kmovd %k0, %eax
-; AVX512F-32-NEXT:    addl %ecx, %eax
-; AVX512F-32-NEXT:    retl
-  %res = call i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
-  %res1 = call i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32-1)
-  %res2 = add i32 %res, %res1
-  ret i32 %res2
-}
-
 define <32 x i16> @test_x86_avx512_psll_w_512(<32 x i16> %a0, <8 x i16> %a1) {
 ; AVX512BW-LABEL: test_x86_avx512_psll_w_512:
 ; AVX512BW:       ## BB#0:
 ; AVX512BW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512F-32-LABEL: test_x86_avx512_psll_w_512:
 ; AVX512F-32:       # BB#0:

test/CodeGen/X86/avx512bwvl-intrinsics-fast-isel.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=X32
 ; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=X64
 
 ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512vlbw-builtins.c
+; Function Attrs: norecurse nounwind readnone
+
+define zeroext i16 @test_mm_test_epi8_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_test_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmb %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_test_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmb %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <16 x i8>
+  %1 = icmp ne <16 x i8> %0, zeroinitializer
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm_mask_test_epi8_mask(i16 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_test_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestmb %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_test_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestmb %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <16 x i8>
+  %1 = icmp ne <16 x i8> %0, zeroinitializer
+  %2 = bitcast i16 %__U to <16 x i1>
+  %3 = and <16 x i1> %1, %2
+  %4 = bitcast <16 x i1> %3 to i16
+  ret i16 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm256_test_epi8_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_test_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmb %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_test_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmb %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <32 x i8>
+  %1 = icmp ne <32 x i8> %0, zeroinitializer
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm256_mask_test_epi8_mask(i32 %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_test_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestmb %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_test_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestmb %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <32 x i8>
+  %1 = icmp ne <32 x i8> %0, zeroinitializer
+  %2 = bitcast i32 %__U to <32 x i1>
+  %3 = and <32 x i1> %1, %2
+  %4 = bitcast <32 x i1> %3 to i32
+  ret i32 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_test_epi16_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_test_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmw %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_test_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmw %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <8 x i16>
+  %1 = icmp ne <8 x i16> %0, zeroinitializer
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_mask_test_epi16_mask(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_test_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    vptestmw %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_test_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestmw %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <8 x i16>
+  %1 = icmp ne <8 x i16> %0, zeroinitializer
+  %2 = bitcast i8 %__U to <8 x i1>
+  %3 = and <8 x i1> %1, %2
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm256_test_epi16_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_test_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmw %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_test_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmw %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <16 x i16>
+  %1 = icmp ne <16 x i16> %0, zeroinitializer
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm256_mask_test_epi16_mask(i16 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_test_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestmw %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_test_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestmw %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <16 x i16>
+  %1 = icmp ne <16 x i16> %0, zeroinitializer
+  %2 = bitcast i16 %__U to <16 x i1>
+  %3 = and <16 x i1> %1, %2
+  %4 = bitcast <16 x i1> %3 to i16
+  ret i16 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm_testn_epi8_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_testn_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmb %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_testn_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmb %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <16 x i8>
+  %1 = icmp eq <16 x i8> %0, zeroinitializer
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm_mask_testn_epi8_mask(i16 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_testn_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestnmb %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_testn_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestnmb %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <16 x i8>
+  %1 = icmp eq <16 x i8> %0, zeroinitializer
+  %2 = bitcast i16 %__U to <16 x i1>
+  %3 = and <16 x i1> %1, %2
+  %4 = bitcast <16 x i1> %3 to i16
+  ret i16 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm256_testn_epi8_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_testn_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmb %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_testn_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmb %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <32 x i8>
+  %1 = icmp eq <32 x i8> %0, zeroinitializer
+  %2 = bitcast <32 x i1> %1 to i32
+  ret i32 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i32 @test_mm256_mask_testn_epi8_mask(i32 %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_testn_epi8_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestnmb %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_testn_epi8_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestnmb %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <32 x i8>
+  %1 = icmp eq <32 x i8> %0, zeroinitializer
+  %2 = bitcast i32 %__U to <32 x i1>
+  %3 = and <32 x i1> %1, %2
+  %4 = bitcast <32 x i1> %3 to i32
+  ret i32 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_testn_epi16_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_testn_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmw %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_testn_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmw %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <8 x i16>
+  %1 = icmp eq <8 x i16> %0, zeroinitializer
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_mask_testn_epi16_mask(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_testn_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovd %eax, %k1
+; X32-NEXT:    vptestnmw %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_testn_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestnmw %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <8 x i16>
+  %1 = icmp eq <8 x i16> %0, zeroinitializer
+  %2 = bitcast i8 %__U to <8 x i1>
+  %3 = and <8 x i1> %1, %2
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm256_testn_epi16_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_testn_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmw %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_testn_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmw %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <16 x i16>
+  %1 = icmp eq <16 x i16> %0, zeroinitializer
+  %2 = bitcast <16 x i1> %1 to i16
+  ret i16 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i16 @test_mm256_mask_testn_epi16_mask(i16 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_testn_epi16_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X32-NEXT:    vptestnmw %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovd %k0, %eax
+; X32-NEXT:    movzwl %ax, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_testn_epi16_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vptestnmw %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovd %k0, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <16 x i16>
+  %1 = icmp eq <16 x i16> %0, zeroinitializer
+  %2 = bitcast i16 %__U to <16 x i1>
+  %3 = and <16 x i1> %1, %2
+  %4 = bitcast <16 x i1> %3 to i16
+  ret i16 %4
+}
 
 define <2 x i64> @test_mm_mask_set1_epi8(<2 x i64> %__O, i16 zeroext %__M, i8 signext %__A) local_unnamed_addr #0 {
 ; X32-LABEL: test_mm_mask_set1_epi8:
 ; X32:       # BB#0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X32-NEXT:    vpbroadcastb %eax, %xmm0 {%k1}
 ; X32-NEXT:    retl

test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll

; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
   %res1 = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 -1)
   %res2 = add <16 x i16> %res, %res1
   ret <16 x i16> %res2
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+declare i16 @llvm.x86.avx512.ptestm.b.128(<16 x i8>, <16 x i8>, i16)
+
+define i16@test_int_x86_avx512_ptestm_b_128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_b_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmb %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x08,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestmb %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i16 @llvm.x86.avx512.ptestm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2)
+  %res1 = call i16 @llvm.x86.avx512.ptestm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16-1)
+  %res2 = add i16 %res, %res1
+  ret i16 %res2
+}
+
+declare i32 @llvm.x86.avx512.ptestm.b.256(<32 x i8>, <32 x i8>, i32)
+
+define i32@test_int_x86_avx512_ptestm_b_256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_b_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmb %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x28,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestmb %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x29,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
+; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i32 @llvm.x86.avx512.ptestm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2)
+  %res1 = call i32 @llvm.x86.avx512.ptestm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32-1)
+  %res2 = add i32 %res, %res1
+  ret i32 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestm.w.128(<8 x i16>, <8 x i16>, i8)
+
+define i8@test_int_x86_avx512_ptestm_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_w_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmw %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x08,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestmw %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i16 @llvm.x86.avx512.ptestm.w.256(<16 x i16>, <16 x i16>, i16)
+
+define i16@test_int_x86_avx512_ptestm_w_256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_w_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmw %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x28,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestmw %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
+; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i16 @llvm.x86.avx512.ptestm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
+  %res1 = call i16 @llvm.x86.avx512.ptestm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16-1)
+  %res2 = add i16 %res, %res1
+  ret i16 %res2
+}
+
+declare i16 @llvm.x86.avx512.ptestnm.b.128(<16 x i8>, <16 x i8>, i16)
+
+define i16@test_int_x86_avx512_ptestnm_b_128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_b_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmb %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x08,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestnmb %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7e,0x09,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i16 @llvm.x86.avx512.ptestnm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2)
+  %res1 = call i16 @llvm.x86.avx512.ptestnm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16-1)
+  %res2 = add i16 %res, %res1
+  ret i16 %res2
+}
+
+declare i32 @llvm.x86.avx512.ptestnm.b.256(<32 x i8>, <32 x i8>, i32)
+
+define i32@test_int_x86_avx512_ptestnm_b_256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_b_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmb %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x28,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestnmb %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7e,0x29,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
+; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i32 @llvm.x86.avx512.ptestnm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2)
+  %res1 = call i32 @llvm.x86.avx512.ptestnm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32-1)
+  %res2 = add i32 %res, %res1
+  ret i32 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestnm.w.128(<8 x i16>, <8 x i16>, i8 %x2)
+
+define i8@test_int_x86_avx512_ptestnm_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_w_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmw %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x08,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestnmw %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfe,0x09,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestnm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestnm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i16 @llvm.x86.avx512.ptestnm.w.256(<16 x i16>, <16 x i16>, i16 %x2)
+
+define i16@test_int_x86_avx512_ptestnm_w_256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_w_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmw %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x28,0x26,0xc1]
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vptestnmw %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfe,0x29,0x26,0xc9]
+; CHECK-NEXT:    kmovd %k1, %ecx ## encoding: [0xc5,0xfb,0x93,0xc9]
+; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
+; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i16 @llvm.x86.avx512.ptestnm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
+  %res1 = call i16 @llvm.x86.avx512.ptestnm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16-1)
+  %res2 = add i16 %res, %res1
+  ret i16 %res2
+}
+

test/CodeGen/X86/avx512bwvl-intrinsics.ll

; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.permvar.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
   %res1 = call <16 x i16> @llvm.x86.avx512.mask.permvar.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
   %res2 = call <16 x i16> @llvm.x86.avx512.mask.permvar.hi.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
   %res3 = add <16 x i16> %res, %res1
   %res4 = add <16 x i16> %res3, %res2
   ret <16 x i16> %res4
 }
 
-declare i16 @llvm.x86.avx512.ptestm.b.128(<16 x i8>, <16 x i8>, i16)
-
-define i16@test_int_x86_avx512_ptestm_b_128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_b_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestmb %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x08,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestmb %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
-; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i16 @llvm.x86.avx512.ptestm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2)
-  %res1 = call i16 @llvm.x86.avx512.ptestm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16-1)
-  %res2 = add i16 %res, %res1
-  ret i16 %res2
-}
-
-declare i32 @llvm.x86.avx512.ptestm.b.256(<32 x i8>, <32 x i8>, i32)
-
-define i32@test_int_x86_avx512_ptestm_b_256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_b_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestmb %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7d,0x29,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestmb %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x28,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i32 @llvm.x86.avx512.ptestm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2)
-  %res1 = call i32 @llvm.x86.avx512.ptestm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32-1)
-  %res2 = add i32 %res, %res1
-  ret i32 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestm.w.128(<8 x i16>, <8 x i16>, i8)
-
-define i8@test_int_x86_avx512_ptestm_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_w_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestmw %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x08,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestmw %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i16 @llvm.x86.avx512.ptestm.w.256(<16 x i16>, <16 x i16>, i16)
-
-define i16@test_int_x86_avx512_ptestm_w_256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_w_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestmw %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x28,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestmw %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
-; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i16 @llvm.x86.avx512.ptestm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
-  %res1 = call i16 @llvm.x86.avx512.ptestm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16-1)
-  %res2 = add i16 %res, %res1
-  ret i16 %res2
-}
-
-declare i16 @llvm.x86.avx512.ptestnm.b.128(<16 x i8>, <16 x i8>, i16)
-
-define i16@test_int_x86_avx512_ptestnm_b_128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_b_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestnmb %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x08,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestnmb %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7e,0x09,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
-; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i16 @llvm.x86.avx512.ptestnm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2)
-  %res1 = call i16 @llvm.x86.avx512.ptestnm.b.128(<16 x i8> %x0, <16 x i8> %x1, i16-1)
-  %res2 = add i16 %res, %res1
-  ret i16 %res2
-}
-
-declare i32 @llvm.x86.avx512.ptestnm.b.256(<32 x i8>, <32 x i8>, i32)
-
-define i32@test_int_x86_avx512_ptestnm_b_256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_b_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestnmb %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7e,0x29,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestnmb %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x28,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i32 @llvm.x86.avx512.ptestnm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2)
-  %res1 = call i32 @llvm.x86.avx512.ptestnm.b.256(<32 x i8> %x0, <32 x i8> %x1, i32-1)
-  %res2 = add i32 %res, %res1
-  ret i32 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestnm.w.128(<8 x i16>, <8 x i16>, i8 %x2)
-
-define i8@test_int_x86_avx512_ptestnm_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_w_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestnmw %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x08,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestnmw %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfe,0x09,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestnm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestnm.w.128(<8 x i16> %x0, <8 x i16> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i16 @llvm.x86.avx512.ptestnm.w.256(<16 x i16>, <16 x i16>, i16 %x2)
-
-define i16@test_int_x86_avx512_ptestnm_w_256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_w_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
-; CHECK-NEXT:    vptestnmw %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x28,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %ecx ## encoding: [0xc5,0xfb,0x93,0xc8]
-; CHECK-NEXT:    vptestnmw %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfe,0x29,0x26,0xc1]
-; CHECK-NEXT:    kmovd %k0, %eax ## encoding: [0xc5,0xfb,0x93,0xc0]
-; CHECK-NEXT:    addl %ecx, %eax ## encoding: [0x01,0xc8]
-; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i16 @llvm.x86.avx512.ptestnm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
-  %res1 = call i16 @llvm.x86.avx512.ptestnm.w.256(<16 x i16> %x0, <16 x i16> %x1, i16-1)
-  %res2 = add i16 %res, %res1
-  ret i16 %res2
-}
-
-

test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=X32
 ; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=X64
 
 ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512vl-builtins.c
 
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_test_epi32_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_test_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmd %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_test_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmd %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <4 x i32>
+  %1 = icmp ne <4 x i32> %0, zeroinitializer
+  %2 = shufflevector <4 x i1> %1, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %3 = bitcast <8 x i1> %2 to i8
+  ret i8 %3
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_mask_test_epi32_mask(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_test_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestmd %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_test_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestmd %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <4 x i32>
+  %1 = icmp ne <4 x i32> %0, zeroinitializer
+  %2 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = and <4 x i1> %1, %extract.i
+  %4 = shufflevector <4 x i1> %3, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %5 = bitcast <8 x i1> %4 to i8
+  ret i8 %5
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_test_epi32_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_test_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmd %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_test_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmd %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <8 x i32>
+  %1 = icmp ne <8 x i32> %0, zeroinitializer
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_mask_test_epi32_mask(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_test_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestmd %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_test_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestmd %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <8 x i32>
+  %1 = icmp ne <8 x i32> %0, zeroinitializer
+  %2 = bitcast i8 %__U to <8 x i1>
+  %3 = and <8 x i1> %1, %2
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_test_epi64_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_test_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmq %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_test_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmq %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = icmp ne <2 x i64> %and.i.i, zeroinitializer
+  %1 = shufflevector <2 x i1> %0, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_mask_test_epi64_mask(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_test_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestmq %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_test_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestmq %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = icmp ne <2 x i64> %and.i.i, zeroinitializer
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %2 = and <2 x i1> %0, %extract.i
+  %3 = shufflevector <2 x i1> %2, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_test_epi64_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_test_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestmq %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_test_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestmq %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = icmp ne <4 x i64> %and.i.i, zeroinitializer
+  %1 = shufflevector <4 x i1> %0, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_mask_test_epi64_mask(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_test_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestmq %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_test_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestmq %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = icmp ne <4 x i64> %and.i.i, zeroinitializer
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = and <4 x i1> %0, %extract.i
+  %3 = shufflevector <4 x i1> %2, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_testn_epi32_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_testn_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmd %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_testn_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmd %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <4 x i32>
+  %1 = icmp eq <4 x i32> %0, zeroinitializer
+  %2 = shufflevector <4 x i1> %1, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %3 = bitcast <8 x i1> %2 to i8
+  ret i8 %3
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_mask_testn_epi32_mask(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_testn_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestnmd %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_testn_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestnmd %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = bitcast <2 x i64> %and.i.i to <4 x i32>
+  %1 = icmp eq <4 x i32> %0, zeroinitializer
+  %2 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = and <4 x i1> %1, %extract.i
+  %4 = shufflevector <4 x i1> %3, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %5 = bitcast <8 x i1> %4 to i8
+  ret i8 %5
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_testn_epi32_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_testn_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmd %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_testn_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmd %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <8 x i32>
+  %1 = icmp eq <8 x i32> %0, zeroinitializer
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_mask_testn_epi32_mask(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_testn_epi32_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestnmd %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_testn_epi32_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestnmd %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = bitcast <4 x i64> %and.i.i to <8 x i32>
+  %1 = icmp eq <8 x i32> %0, zeroinitializer
+  %2 = bitcast i8 %__U to <8 x i1>
+  %3 = and <8 x i1> %1, %2
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_testn_epi64_mask(<2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_testn_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmq %xmm0, %xmm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_testn_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmq %xmm0, %xmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = icmp eq <2 x i64> %and.i.i, zeroinitializer
+  %1 = shufflevector <2 x i1> %0, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm_mask_testn_epi64_mask(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm_mask_testn_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestnmq %xmm0, %xmm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_testn_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestnmq %xmm0, %xmm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <2 x i64> %__B, %__A
+  %0 = icmp eq <2 x i64> %and.i.i, zeroinitializer
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %2 = and <2 x i1> %0, %extract.i
+  %3 = shufflevector <2 x i1> %2, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_testn_epi64_mask(<4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_testn_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    vptestnmq %ymm0, %ymm1, %k0
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_testn_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    vptestnmq %ymm0, %ymm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = icmp eq <4 x i64> %and.i.i, zeroinitializer
+  %1 = shufflevector <4 x i1> %0, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = bitcast <8 x i1> %1 to i8
+  ret i8 %2
+}
+
+; Function Attrs: norecurse nounwind readnone
+define zeroext i8 @test_mm256_mask_testn_epi64_mask(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) local_unnamed_addr #0 {
+; X32-LABEL: test_mm256_mask_testn_epi64_mask:
+; X32:       # BB#0: # %entry
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X32-NEXT:    kmovw %eax, %k1
+; X32-NEXT:    vptestnmq %ymm0, %ymm1, %k0 {%k1}
+; X32-NEXT:    kmovw %k0, %eax
+; X32-NEXT:    movzbl %al, %eax
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_testn_epi64_mask:
+; X64:       # BB#0: # %entry
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vptestnmq %ymm0, %ymm1, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+entry:
+  %and.i.i = and <4 x i64> %__B, %__A
+  %0 = icmp eq <4 x i64> %and.i.i, zeroinitializer
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = and <4 x i1> %0, %extract.i
+  %3 = shufflevector <4 x i1> %2, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %4 = bitcast <8 x i1> %3 to i8
+  ret i8 %4
+}
+
 define <2 x i64> @test_mm_mask_set1_epi32(<2 x i64> %__O, i8 zeroext %__M)  {
 ; X32-LABEL: test_mm_mask_set1_epi32:
 ; X32:       # BB#0: # %entry
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X32-NEXT:    kmovw %eax, %k1
 ; X32-NEXT:    vpbroadcastd {{\.LCPI.*}}, %xmm0 {%k1}
 ; X32-NEXT:    retl
 ;

test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll

 ; CHECK-NEXT:    vpaddd %ymm2, %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
   %res1 = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
   %res2 = add <8 x i32> %res, %res1
   ret <8 x i32> %res2
 }
 
+declare i8 @llvm.x86.avx512.ptestm.d.128(<4 x i32>, <4 x i32>,i8)
+
+define i8@test_int_x86_avx512_ptestm_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_d_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmd %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x08,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestmd %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestm.d.256(<8 x i32>, <8 x i32>, i8)
+
+define i8@test_int_x86_avx512_ptestm_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_d_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmd %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x28,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestmd %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x29,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestm.q.128(<2 x i64>, <2 x i64>, i8)
+
+define i8@test_int_x86_avx512_ptestm_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_q_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmq %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x08,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestmq %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestm.q.256(<4 x i64>, <4 x i64>, i8)
+
+define i8@test_int_x86_avx512_ptestm_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestm_q_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestmq %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x28,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestmq %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestnm.d.128(<4 x i32>, <4 x i32>, i8 %x2)
+
+define i8@test_int_x86_avx512_ptestnm_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_d_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmd %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x08,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestnmd %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7e,0x09,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestnm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestnm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestnm.d.256(<8 x i32>, <8 x i32>, i8 %x2)
+
+define i8@test_int_x86_avx512_ptestnm_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_d_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmd %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x28,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestnmd %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0x7e,0x29,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestnm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestnm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestnm.q.128(<2 x i64>, <2 x i64>, i8 %x2)
+
+define i8@test_int_x86_avx512_ptestnm_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_q_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmq %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x08,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestnmq %xmm1, %xmm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfe,0x09,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestnm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestnm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+
+declare i8 @llvm.x86.avx512.ptestnm.q.256(<4 x i64>, <4 x i64>, i8 %x2)
+
+define i8@test_int_x86_avx512_ptestnm_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ptestnm_q_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    vptestnmq %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x28,0x27,0xc1]
+; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT:    vptestnmq %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf2,0xfe,0x29,0x27,0xc9]
+; CHECK-NEXT:    kmovw %k1, %ecx ## encoding: [0xc5,0xf8,0x93,0xc9]
+; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
+; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %res = call i8 @llvm.x86.avx512.ptestnm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call i8 @llvm.x86.avx512.ptestnm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8-1)
+  %res2 = add i8 %res, %res1
+  ret i8 %res2
+}
+

test/CodeGen/X86/avx512vl-intrinsics.ll

; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 %x4)
   %res1 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> zeroinitializer, i32 5, i8 %x4)
   %res2 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 -1)
   %res3 = fadd <8 x float> %res, %res1
   %res4 = fadd <8 x float> %res3, %res2
   ret <8 x float> %res4
 }
 
-declare i8 @llvm.x86.avx512.ptestm.d.128(<4 x i32>, <4 x i32>,i8)
-
-define i8@test_int_x86_avx512_ptestm_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_d_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestmd %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestmd %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x08,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestm.d.256(<8 x i32>, <8 x i32>, i8)
-
-define i8@test_int_x86_avx512_ptestm_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_d_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestmd %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7d,0x28,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestmd %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7d,0x29,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestm.q.128(<2 x i64>, <2 x i64>, i8)
-
-define i8@test_int_x86_avx512_ptestm_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_q_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestmq %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestmq %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x08,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestm.q.256(<4 x i64>, <4 x i64>, i8)
-
-define i8@test_int_x86_avx512_ptestm_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestm_q_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestmq %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfd,0x29,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestmq %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfd,0x28,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestnm.d.128(<4 x i32>, <4 x i32>, i8 %x2)
-
-define i8@test_int_x86_avx512_ptestnm_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_d_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestnmd %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7e,0x09,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestnmd %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x08,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestnm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestnm.d.128(<4 x i32> %x0, <4 x i32> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestnm.d.256(<8 x i32>, <8 x i32>, i8 %x2)
-
-define i8@test_int_x86_avx512_ptestnm_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_d_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestnmd %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0x7e,0x28,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestnmd %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0x7e,0x29,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestnm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestnm.d.256(<8 x i32> %x0, <8 x i32> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestnm.q.128(<2 x i64>, <2 x i64>, i8 %x2)
-
-define i8@test_int_x86_avx512_ptestnm_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_q_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestnmq %xmm1, %xmm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfe,0x09,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestnmq %xmm1, %xmm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x08,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestnm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestnm.q.128(<2 x i64> %x0, <2 x i64> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-declare i8 @llvm.x86.avx512.ptestnm.q.256(<4 x i64>, <4 x i64>, i8 %x2)
-
-define i8@test_int_x86_avx512_ptestnm_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_ptestnm_q_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; CHECK-NEXT:    vptestnmq %ymm1, %ymm0, %k0 {%k1} ## encoding: [0x62,0xf2,0xfe,0x29,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %ecx ## encoding: [0xc5,0xf8,0x93,0xc8]
-; CHECK-NEXT:    vptestnmq %ymm1, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x28,0x27,0xc1]
-; CHECK-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
-; CHECK-NEXT:    addb %cl, %al ## encoding: [0x00,0xc8]
-; CHECK-NEXT:    ## kill: %AL<def> %AL<kill> %EAX<kill>
-; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %res = call i8 @llvm.x86.avx512.ptestnm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
-  %res1 = call i8 @llvm.x86.avx512.ptestnm.q.256(<4 x i64> %x0, <4 x i64> %x1, i8-1)
-  %res2 = add i8 %res, %res1
-  ret i8 %res2
-}
-
-
-
 define <2 x i64> @test_x86_avx512_psra_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psra_q_128:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpsraq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0xe2,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
   ret <2 x i64> %res
 }