[x86][icelake]VNNI

[coby]

Summary

Introducing Vector Neural Network Instructions, consisting of:
vpdpbusd{s}
vpdpwssd{s}

[craig.topper]

LGTM

include/llvm/IR/IntrinsicsX86.td

def int_x86_avx512_mask_vpmadd52l_uq_512 :
               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
   def int_x86_avx512_maskz_vpmadd52l_uq_512 :
               GCCBuiltin<"__builtin_ia32_vpmadd52luq512_maskz">,
               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
                          llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
 }
 
+// VNNI
+let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
+  def int_x86_avx512_mask_vpdpbusd_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusd128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpbusd_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusd128_maskz">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpbusd_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusd256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpbusd_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusd256_maskz">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpbusd_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusd512_mask">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpbusd_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusd512_maskz">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpdpbusds_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusds128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpbusds_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusds128_maskz">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpbusds_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusds256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpbusds_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusds256_maskz">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpbusds_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusds512_mask">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpbusds_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpbusds512_maskz">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpdpwssd_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssd128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpwssd_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssd128_maskz">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpwssd_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssd256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpwssd_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssd256_maskz">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpwssd_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssd512_mask">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpwssd_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssd512_maskz">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_mask_vpdpwssds_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssds128_mask">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpwssds_128 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssds128_maskz">,
+              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
+                         llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpwssds_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssds256_mask">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpwssds_256 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssds256_maskz">,
+              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
+                         llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_vpdpwssds_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssds512_mask">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_maskz_vpdpwssds_512 :
+              GCCBuiltin<"__builtin_ia32_vpdpwssds512_maskz">,
+              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
+                         llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+}
+
 //===----------------------------------------------------------------------===//
 // XOP
 
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_xop_vpermil2pd : GCCBuiltin<"__builtin_ia32_vpermil2pd">,
               Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
                                           llvm_v2i64_ty, llvm_i8_ty],
                         [IntrNoMem]>;

lib/Support/Host.cpp

bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   Features["avx512vl"] = HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save;
 
   Features["prefetchwt1"] = HasLeaf7 && (ECX & 1);
   Features["avx512vbmi"] = HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save;
   Features["avx512vpopcntdq"] = HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save;
   // Enable protection keys
   Features["pku"] = HasLeaf7 && ((ECX >> 4) & 1);
 
+  // Enable Vector Neural Network Instructions
+  Features["avx512vnni"] = HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save;
+
   bool HasLeafD = MaxLevel >= 0xd &&
                   !getX86CpuIDAndInfoEx(0xd, 0x1, &EAX, &EBX, &ECX, &EDX);
 
   // Only enable XSAVE if OS has enabled support for saving YMM state.
   Features["xsaveopt"] = HasAVXSave && HasLeafD && ((EAX >> 0) & 1);
   Features["xsavec"] = HasAVXSave && HasLeafD && ((EAX >> 1) & 1);
   Features["xsaves"] = HasAVXSave && HasLeafD && ((EAX >> 3) & 1);
 

lib/Target/X86/X86.td

 def FeatureVBMI     : SubtargetFeature<"avx512vbmi", "HasVBMI", "true",
                       "Enable AVX-512 Vector Byte Manipulation Instructions",
                                       [FeatureBWI]>;
 def FeatureIFMA     : SubtargetFeature<"avx512ifma", "HasIFMA", "true",
                       "Enable AVX-512 Integer Fused Multiple-Add",
                                       [FeatureAVX512]>;
 def FeaturePKU   : SubtargetFeature<"pku", "HasPKU", "true",
                       "Enable protection keys">;
+def FeatureVNNI    : SubtargetFeature<"avx512vnni", "HasVNNI", "true",
+                          "Enable AVX-512 Vector Neural Network Instructions",
+                                      [FeatureAVX512]>;
 def FeaturePCLMUL  : SubtargetFeature<"pclmul", "HasPCLMUL", "true",
                          "Enable packed carry-less multiplication instructions",
                                [FeatureSSE2]>;
 def FeatureFMA     : SubtargetFeature<"fma", "HasFMA", "true",
                                       "Enable three-operand fused multiple-add",
                                       [FeatureAVX]>;
 def FeatureFMA4    : SubtargetFeature<"fma4", "HasFMA4", "true",
                                       "Enable four-operand fused multiple-add",

lib/Target/X86/X86ISelLowering.h

enum NodeType : unsigned {
       // Multiply and Add Packed Integers.
       VPMADDUBSW, VPMADDWD,
 
       // AVX512IFMA multiply and add.
       // NOTE: These are different than the instruction and perform
       // op0 x op1 + op2.
       VPMADD52L, VPMADD52H,
 
+      // VNNI
+      VPDPBUSD,
+      VPDPBUSDS,
+      VPDPWSSD,
+      VPDPWSSDS,
+
       // FMA nodes.
       // We use the target independent ISD::FMA for the non-inverted case.
       FNMADD,
       FMSUB,
       FNMSUB,
       FMADDSUB,
       FMSUBADD,
 

lib/Target/X86/X86ISelLowering.cpp

const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::CVTP2SI:            return "X86ISD::CVTP2SI";
   case X86ISD::CVTP2UI:            return "X86ISD::CVTP2UI";
   case X86ISD::CVTP2SI_RND:        return "X86ISD::CVTP2SI_RND";
   case X86ISD::CVTP2UI_RND:        return "X86ISD::CVTP2UI_RND";
   case X86ISD::CVTS2SI_RND:        return "X86ISD::CVTS2SI_RND";
   case X86ISD::CVTS2UI_RND:        return "X86ISD::CVTS2UI_RND";
   case X86ISD::LWPINS:             return "X86ISD::LWPINS";
   case X86ISD::MGATHER:            return "X86ISD::MGATHER";
+  case X86ISD::VPDPBUSD:           return "X86ISD::VPDPBUSD";
+  case X86ISD::VPDPBUSDS:          return "X86ISD::VPDPBUSDS";
+  case X86ISD::VPDPWSSD:           return "X86ISD::VPDPWSSD";
+  case X86ISD::VPDPWSSDS:          return "X86ISD::VPDPWSSDS";
   }
   return nullptr;
 }
 
 /// Return true if the addressing mode represented by AM is legal for this
 /// target, for a load/store of the specified type.
 bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                               const AddrMode &AM, Type *Ty,

lib/Target/X86/X86InstrAVX512.td

def : Pat<(X86Movsd (v2f64 VR128X:$src1),
           (COPY_TO_REGCLASS FR64X:$src2, VR128X))>;
   }
 }
 
 defm : AVX512_scalar_math_f64_patterns<fadd, "ADD">;
 defm : AVX512_scalar_math_f64_patterns<fsub, "SUB">;
 defm : AVX512_scalar_math_f64_patterns<fmul, "MUL">;
 defm : AVX512_scalar_math_f64_patterns<fdiv, "DIV">;
+
+//===----------------------------------------------------------------------===//
+// VNNI
+//===----------------------------------------------------------------------===//
+
+let Constraints = "$src1 = $dst" in
+multiclass VNNI_rmb<bits<8> Op, string OpStr, SDNode OpNode,
+                    X86VectorVTInfo VTI> {
+  defm r  :   AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst),
+                                   (ins VTI.RC:$src2, VTI.RC:$src3), OpStr,
+                                   "$src3, $src2", "$src2, $src3",
+                                   (VTI.VT (OpNode VTI.RC:$src1,
+                                            VTI.RC:$src2, VTI.RC:$src3))>,
+              EVEX_4V, T8PD;
+  defm m  :   AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
+                                   (ins VTI.RC:$src2, VTI.MemOp:$src3), OpStr,
+                                   "$src3, $src2", "$src2, $src3",
+                                   (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2,
+                                            (VTI.VT (bitconvert
+                                                     (VTI.LdFrag addr:$src3)))))>,
+              EVEX_4V, EVEX_CD8<32, CD8VF>, T8PD;
+  defm mb :   AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
+                                   (ins VTI.RC:$src2, VTI.ScalarMemOp:$src3),
+                                   OpStr, "${src3}"##VTI.BroadcastStr##", $src2",
+                                   "$src2, ${src3}"##VTI.BroadcastStr,
+                                   (OpNode VTI.RC:$src1, VTI.RC:$src2,
+                                    (VTI.VT (X86VBroadcast
+                                             (VTI.ScalarLdFrag addr:$src3))))>,
+              EVEX_4V, EVEX_CD8<32, CD8VF>, EVEX_B, T8PD;
+}
+
+multiclass VNNI_common<bits<8> Op, string OpStr, SDNode OpNode> {
+  let Predicates = [HasVNNI] in
+  defm Z      :   VNNI_rmb<Op, OpStr, OpNode, v16i32_info>, EVEX_V512;
+  let Predicates = [HasVNNI, HasVLX] in {
+    defm Z256 :   VNNI_rmb<Op, OpStr, OpNode, v8i32x_info>, EVEX_V256;
+    defm Z128 :   VNNI_rmb<Op, OpStr, OpNode, v4i32x_info>, EVEX_V128;
+  }
+}
+
+defm VPDPBUSD   : VNNI_common<0x50, "vpdpbusd", X86Vpdpbusd>;
+defm VPDPBUSDS  : VNNI_common<0x51, "vpdpbusds", X86Vpdpbusds>;
+defm VPDPWSSD   : VNNI_common<0x52, "vpdpwssd", X86Vpdpwssd>;
+defm VPDPWSSDS  : VNNI_common<0x53, "vpdpwssds", X86Vpdpwssds>;
+

lib/Target/X86/X86InstrFragmentsSIMD.td

 def X86FmsubRnds3   : SDNode<"X86ISD::FMSUBS3_RND",     SDTFmaRound, [SDNPCommutative]>;
 def X86FnmsubRnds3  : SDNode<"X86ISD::FNMSUBS3_RND",    SDTFmaRound, [SDNPCommutative]>;
 
 def SDTIFma : SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<0,1>,
                            SDTCisSameAs<1,2>, SDTCisSameAs<1,3>]>;
 def x86vpmadd52l     : SDNode<"X86ISD::VPMADD52L",     SDTIFma, [SDNPCommutative]>;
 def x86vpmadd52h     : SDNode<"X86ISD::VPMADD52H",     SDTIFma, [SDNPCommutative]>;
 
+// VNNI
+def SDTVnni : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
+                                   SDTCisSameAs<1,2>, SDTCisSameAs<1,3>]>;
+def X86Vpdpbusd  : SDNode<"X86ISD::VPDPBUSD", SDTVnni>;
+def X86Vpdpbusds : SDNode<"X86ISD::VPDPBUSDS", SDTVnni>;
+def X86Vpdpwssd  : SDNode<"X86ISD::VPDPWSSD", SDTVnni>;
+def X86Vpdpwssds : SDNode<"X86ISD::VPDPWSSDS", SDTVnni>;
+
 def X86rsqrt28   : SDNode<"X86ISD::RSQRT28",  SDTFPUnaryOpRound>;
 def X86rcp28     : SDNode<"X86ISD::RCP28",    SDTFPUnaryOpRound>;
 def X86exp2      : SDNode<"X86ISD::EXP2",     SDTFPUnaryOpRound>;
 
 def X86rsqrt28s  : SDNode<"X86ISD::RSQRT28S",   SDTFPBinOpRound>;
 def X86rcp28s    : SDNode<"X86ISD::RCP28S",     SDTFPBinOpRound>;
 def X86RndScales : SDNode<"X86ISD::VRNDSCALES", SDTFPBinOpImmRound>;
 def X86Reduces   : SDNode<"X86ISD::VREDUCES",   SDTFPBinOpImmRound>;

lib/Target/X86/X86InstrInfo.td

def HasBWI       : Predicate<"Subtarget->hasBWI()">,
                      AssemblerPredicate<"FeatureBWI", "AVX-512 BW ISA">;
 def NoBWI        : Predicate<"!Subtarget->hasBWI()">;
 def HasVLX       : Predicate<"Subtarget->hasVLX()">,
                      AssemblerPredicate<"FeatureVLX", "AVX-512 VL ISA">;
 def NoVLX        : Predicate<"!Subtarget->hasVLX()">;
 def NoVLX_Or_NoBWI : Predicate<"!Subtarget->hasVLX() || !Subtarget->hasBWI()">;
 def NoVLX_Or_NoDQI : Predicate<"!Subtarget->hasVLX() || !Subtarget->hasDQI()">;
 def PKU        : Predicate<"Subtarget->hasPKU()">;
+def HasVNNI    : Predicate<"Subtarget->hasVNNI()">;
 
 def HasPOPCNT    : Predicate<"Subtarget->hasPOPCNT()">;
 def HasAES       : Predicate<"Subtarget->hasAES()">;
 def HasFXSR      : Predicate<"Subtarget->hasFXSR()">;
 def HasXSAVE     : Predicate<"Subtarget->hasXSAVE()">;
 def HasXSAVEOPT  : Predicate<"Subtarget->hasXSAVEOPT()">;
 def HasXSAVEC    : Predicate<"Subtarget->hasXSAVEC()">;
 def HasXSAVES    : Predicate<"Subtarget->hasXSAVES()">;

lib/Target/X86/X86IntrinsicsInfo.h

static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_vfnmsub_pd_256, FMA_OP_MASK, X86ISD::FNMSUB, 0),
   X86_INTRINSIC_DATA(avx512_mask_vfnmsub_pd_512, FMA_OP_MASK, X86ISD::FNMSUB,
                      X86ISD::FNMSUB_RND),
   X86_INTRINSIC_DATA(avx512_mask_vfnmsub_ps_128, FMA_OP_MASK, X86ISD::FNMSUB, 0),
   X86_INTRINSIC_DATA(avx512_mask_vfnmsub_ps_256, FMA_OP_MASK, X86ISD::FNMSUB, 0),
   X86_INTRINSIC_DATA(avx512_mask_vfnmsub_ps_512, FMA_OP_MASK, X86ISD::FNMSUB,
                      X86ISD::FNMSUB_RND),
 
+  X86_INTRINSIC_DATA(avx512_mask_vpdpbusd_128,  FMA_OP_MASK, X86ISD::VPDPBUSD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpbusd_256,  FMA_OP_MASK, X86ISD::VPDPBUSD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpbusd_512,  FMA_OP_MASK, X86ISD::VPDPBUSD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpbusds_128, FMA_OP_MASK, X86ISD::VPDPBUSDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpbusds_256, FMA_OP_MASK, X86ISD::VPDPBUSDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpbusds_512, FMA_OP_MASK, X86ISD::VPDPBUSDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpwssd_128,  FMA_OP_MASK, X86ISD::VPDPWSSD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpwssd_256,  FMA_OP_MASK, X86ISD::VPDPWSSD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpwssd_512,  FMA_OP_MASK, X86ISD::VPDPWSSD, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpwssds_128, FMA_OP_MASK, X86ISD::VPDPWSSDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpwssds_256, FMA_OP_MASK, X86ISD::VPDPWSSDS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vpdpwssds_512, FMA_OP_MASK, X86ISD::VPDPWSSDS, 0),
+
   X86_INTRINSIC_DATA(avx512_mask_vpermi2var_d_128, VPERM_3OP_MASK,
                     X86ISD::VPERMIV3, 0),
   X86_INTRINSIC_DATA(avx512_mask_vpermi2var_d_256, VPERM_3OP_MASK,
                     X86ISD::VPERMIV3, 0),
   X86_INTRINSIC_DATA(avx512_mask_vpermi2var_d_512, VPERM_3OP_MASK,
                     X86ISD::VPERMIV3, 0),
   X86_INTRINSIC_DATA(avx512_mask_vpermi2var_hi_128, VPERM_3OP_MASK,
                     X86ISD::VPERMIV3, 0),
static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_pd_256, FMA_OP_MASKZ, X86ISD::FMADDSUB, 0),
   X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_pd_512, FMA_OP_MASKZ, X86ISD::FMADDSUB,
                      X86ISD::FMADDSUB_RND),
   X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_ps_128, FMA_OP_MASKZ, X86ISD::FMADDSUB, 0),
   X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_ps_256, FMA_OP_MASKZ, X86ISD::FMADDSUB, 0),
   X86_INTRINSIC_DATA(avx512_maskz_vfmaddsub_ps_512, FMA_OP_MASKZ, X86ISD::FMADDSUB,
                      X86ISD::FMADDSUB_RND),
 
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpbusd_128,  FMA_OP_MASKZ, X86ISD::VPDPBUSD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpbusd_256,  FMA_OP_MASKZ, X86ISD::VPDPBUSD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpbusd_512,  FMA_OP_MASKZ, X86ISD::VPDPBUSD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpbusds_128, FMA_OP_MASKZ, X86ISD::VPDPBUSDS, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpbusds_256, FMA_OP_MASKZ, X86ISD::VPDPBUSDS, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpbusds_512, FMA_OP_MASKZ, X86ISD::VPDPBUSDS, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpwssd_128,  FMA_OP_MASKZ, X86ISD::VPDPWSSD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpwssd_256,  FMA_OP_MASKZ, X86ISD::VPDPWSSD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpwssd_512,  FMA_OP_MASKZ, X86ISD::VPDPWSSD, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpwssds_128, FMA_OP_MASKZ, X86ISD::VPDPWSSDS, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpwssds_256, FMA_OP_MASKZ, X86ISD::VPDPWSSDS, 0),
+  X86_INTRINSIC_DATA(avx512_maskz_vpdpwssds_512, FMA_OP_MASKZ, X86ISD::VPDPWSSDS, 0),
+
   X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_d_128, VPERM_3OP_MASKZ,
                      X86ISD::VPERMV3, 0),
   X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_d_256, VPERM_3OP_MASKZ,
                      X86ISD::VPERMV3, 0),
   X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_d_512, VPERM_3OP_MASKZ,
                      X86ISD::VPERMV3, 0),
   X86_INTRINSIC_DATA(avx512_maskz_vpermt2var_hi_128, VPERM_3OP_MASKZ,
                      X86ISD::VPERMV3, 0),

lib/Target/X86/X86Subtarget.h

protected:
   bool HasBWI;
 
   /// Processor has AVX-512 Vector Length eXtenstions
   bool HasVLX;
 
   /// Processor has PKU extenstions
   bool HasPKU;
 
+  /// Processor has AVX-512 Vector Neural Network Instructions
+  bool HasVNNI;
+
   /// Processor supports MPX - Memory Protection Extensions
   bool HasMPX;
 
   /// Processor has Software Guard Extensions
   bool HasSGX;
 
   /// Processor supports Flush Cache Line instruction
   bool HasCLFLUSHOPT;
public:
   bool hasCDI() const { return HasCDI; }
   bool hasVPOPCNTDQ() const { return HasVPOPCNTDQ; }
   bool hasPFI() const { return HasPFI; }
   bool hasERI() const { return HasERI; }
   bool hasDQI() const { return HasDQI; }
   bool hasBWI() const { return HasBWI; }
   bool hasVLX() const { return HasVLX; }
   bool hasPKU() const { return HasPKU; }
+  bool hasVNNI() const { return HasVNNI; }
   bool hasMPX() const { return HasMPX; }
   bool hasCLFLUSHOPT() const { return HasCLFLUSHOPT; }
   bool hasCLWB() const { return HasCLWB; }
 
   bool isXRaySupported() const override { return is64Bit(); }
 
   X86ProcFamilyEnum getProcFamily() const { return X86ProcFamily; }
 

lib/Target/X86/X86Subtarget.cpp

void X86Subtarget::initializeEnvironment() {
   HasCDI = false;
   HasPFI = false;
   HasDQI = false;
   HasVPOPCNTDQ = false;
   HasBWI = false;
   HasVLX = false;
   HasADX = false;
   HasPKU = false;
+  HasVNNI = false;
   HasSHA = false;
   HasPRFCHW = false;
   HasRDSEED = false;
   HasLAHFSAHF = false;
   HasMWAITX = false;
   HasCLZERO = false;
   HasMPX = false;
   HasSGX = false;

test/CodeGen/X86/avx512vl_vnni-intrinsics.ll

+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512vnni,+avx512vl| FileCheck %s
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpbusd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %ymm0, %ymm3
+; CHECK-NEXT:    vpdpbusd (%rdi), %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vmovaps %ymm0, %ymm4
+; CHECK-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4
+; CHECK-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm4, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %x2 = load <8 x i32>, <8 x i32>* %x2p
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res3 = add <8 x i32> %res, %res1
+  %res4 = add <8 x i32> %res2, %res3
+  ret <8 x i32> %res4
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpbusd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm3
+; CHECK-NEXT:    vpdpbusd (%rdi), %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vmovaps %xmm0, %xmm4
+; CHECK-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4
+; CHECK-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %x2 = load <4 x i32>, <4 x i32>* %x2p
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res3 = add <4 x i32> %res, %res1
+  %res4 = add <4 x i32> %res2, %res3
+  ret <4 x i32> %res4
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpbusds_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %ymm0, %ymm3
+; CHECK-NEXT:    vpdpbusds (%rdi), %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vmovaps %ymm0, %ymm4
+; CHECK-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4
+; CHECK-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm4, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %x2 = load <8 x i32>, <8 x i32>* %x2p
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res3 = add <8 x i32> %res, %res1
+  %res4 = add <8 x i32> %res2, %res3
+  ret <8 x i32> %res4
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpbusds_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm3
+; CHECK-NEXT:    vpdpbusds (%rdi), %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vmovaps %xmm0, %xmm4
+; CHECK-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4
+; CHECK-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %x2 = load <4 x i32>, <4 x i32>* %x2p
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res3 = add <4 x i32> %res, %res1
+  %res4 = add <4 x i32> %res2, %res3
+  ret <4 x i32> %res4
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpwssd_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %ymm0, %ymm3
+; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vmovaps %ymm0, %ymm4
+; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4
+; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm4, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %x2 = load <8 x i32>, <8 x i32>* %x2p
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res3 = add <8 x i32> %res, %res1
+  %res4 = add <8 x i32> %res2, %res3
+  ret <8 x i32> %res4
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpwssd_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm3
+; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vmovaps %xmm0, %xmm4
+; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4
+; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %x2 = load <4 x i32>, <4 x i32>* %x2p
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res3 = add <4 x i32> %res, %res1
+  %res4 = add <4 x i32> %res2, %res3
+  ret <4 x i32> %res4
+}
+
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpwssds_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %ymm0, %ymm3
+; CHECK-NEXT:    vpdpwssds (%rdi), %ymm1, %ymm3 {%k1}
+; CHECK-NEXT:    vmovaps %ymm0, %ymm4
+; CHECK-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4
+; CHECK-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm4, %ymm0
+; CHECK-NEXT:    vpaddd %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    retq
+  %x2 = load <8 x i32>, <8 x i32>* %x2p
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res3 = add <8 x i32> %res, %res1
+  %res4 = add <8 x i32> %res2, %res3
+  ret <8 x i32> %res4
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpwssds_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm3
+; CHECK-NEXT:    vpdpwssds (%rdi), %xmm1, %xmm3 {%k1}
+; CHECK-NEXT:    vmovaps %xmm0, %xmm4
+; CHECK-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4
+; CHECK-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %x2 = load <4 x i32>, <4 x i32>* %x2p
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res3 = add <4 x i32> %res, %res1
+  %res4 = add <4 x i32> %res2, %res3
+  ret <4 x i32> %res4
+}
+

test/CodeGen/X86/avx512vnni-intrinsics.ll

+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512vnni | FileCheck %s
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpbusd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vpdpbusd (%rdi), %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vmovaps %zmm0, %zmm4
+; CHECK-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4
+; CHECK-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %zmm0, %zmm4, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %x2 = load <16 x i32>, <16 x i32>* %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res3 = add <16 x i32> %res, %res1
+  %res4 = add <16 x i32> %res2, %res3
+  ret <16 x i32> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpbusds_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vpdpbusds (%rdi), %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vmovaps %zmm0, %zmm4
+; CHECK-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4
+; CHECK-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %zmm0, %zmm4, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %x2 = load <16 x i32>, <16 x i32>* %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res3 = add <16 x i32> %res, %res1
+  %res4 = add <16 x i32> %res2, %res3
+  ret <16 x i32> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpwssd_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vmovaps %zmm0, %zmm4
+; CHECK-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4
+; CHECK-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %zmm0, %zmm4, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %x2 = load <16 x i32>, <16 x i32>* %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res3 = add <16 x i32> %res, %res1
+  %res4 = add <16 x i32> %res2, %res3
+  ret <16 x i32> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_vpdpwssds_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    kmovw %esi, %k1
+; CHECK-NEXT:    vmovaps %zmm0, %zmm3
+; CHECK-NEXT:    vpdpwssds (%rdi), %zmm1, %zmm3 {%k1}
+; CHECK-NEXT:    vmovaps %zmm0, %zmm4
+; CHECK-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4
+; CHECK-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vpaddd %zmm0, %zmm4, %zmm0
+; CHECK-NEXT:    vpaddd %zmm0, %zmm3, %zmm0
+; CHECK-NEXT:    retq
+  %x2 = load <16 x i32>, <16 x i32>* %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res3 = add <16 x i32> %res, %res1
+  %res4 = add <16 x i32> %res2, %res3
+  ret <16 x i32> %res4
+}
+

test/MC/X86/avx512vl_vnni-encoding.s

+// RUN: llvm-mc -triple x86_64-unknown-unknown -mcpu=knl -mattr=+avx512vnni,+avx512vl --show-encoding < %s | FileCheck %s
+
+// CHECK: vpdpbusd %xmm3, %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x50,0xcb]
+          vpdpbusd %xmm3, %xmm2, %xmm1
+
+// CHECK: vpdpbusds %xmm3, %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x51,0xcb]
+          vpdpbusds %xmm3, %xmm2, %xmm1
+
+// CHECK: vpdpwssd %xmm3, %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x52,0xcb]
+          vpdpwssd %xmm3, %xmm2, %xmm1
+
+// CHECK: vpdpwssds %xmm3, %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x53,0xcb]
+          vpdpwssds %xmm3, %xmm2, %xmm1
+
+// CHECK: vpdpbusd %xmm23, %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x50,0xef]
+          vpdpbusd %xmm23, %xmm22, %xmm21
+
+// CHECK: vpdpbusds %xmm23, %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x51,0xef]
+          vpdpbusds %xmm23, %xmm22, %xmm21
+
+// CHECK: vpdpwssd %xmm23, %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x52,0xef]
+          vpdpwssd %xmm23, %xmm22, %xmm21
+
+// CHECK: vpdpwssds %xmm23, %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x53,0xef]
+          vpdpwssds %xmm23, %xmm22, %xmm21
+
+// CHECK: vpdpbusd %xmm3, %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x50,0xcb]
+          vpdpbusd %xmm3, %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusds %xmm3, %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x51,0xcb]
+          vpdpbusds %xmm3, %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssd %xmm3, %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x52,0xcb]
+          vpdpwssd %xmm3, %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssds %xmm3, %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x53,0xcb]
+          vpdpwssds %xmm3, %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusd %xmm23, %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x50,0xef]
+          vpdpbusd %xmm23, %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusds %xmm23, %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x51,0xef]
+          vpdpbusds %xmm23, %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssd %xmm23, %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x52,0xef]
+          vpdpwssd %xmm23, %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssds %xmm23, %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x53,0xef]
+          vpdpwssds %xmm23, %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusd  (%rcx), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x50,0x09]
+          vpdpbusd  (%rcx), %xmm2, %xmm1
+
+// CHECK: vpdpbusd  -64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x50,0x4c,0x24,0xfc]
+          vpdpbusd  -64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpbusd  64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x50,0x4c,0x24,0x04]
+          vpdpbusd  64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x50,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x50,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x50,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpbusds  (%rcx), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x51,0x09]
+          vpdpbusds  (%rcx), %xmm2, %xmm1
+
+// CHECK: vpdpbusds  -64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x51,0x4c,0x24,0xfc]
+          vpdpbusds  -64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpbusds  64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x51,0x4c,0x24,0x04]
+          vpdpbusds  64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x51,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x51,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x51,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpwssd  (%rcx), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x52,0x09]
+          vpdpwssd  (%rcx), %xmm2, %xmm1
+
+// CHECK: vpdpwssd  -64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x52,0x4c,0x24,0xfc]
+          vpdpwssd  -64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpwssd  64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x52,0x4c,0x24,0x04]
+          vpdpwssd  64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x52,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x52,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x52,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpwssds  (%rcx), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x53,0x09]
+          vpdpwssds  (%rcx), %xmm2, %xmm1
+
+// CHECK: vpdpwssds  -64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x53,0x4c,0x24,0xfc]
+          vpdpwssds  -64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpwssds  64(%rsp), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x08,0x53,0x4c,0x24,0x04]
+          vpdpwssds  64(%rsp), %xmm2, %xmm1
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x53,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x53,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x08,0x53,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %xmm2, %xmm1
+
+// CHECK: vpdpbusd  (%rcx), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x50,0x29]
+          vpdpbusd  (%rcx), %xmm22, %xmm21
+
+// CHECK: vpdpbusd  -64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x50,0x6c,0x24,0xfc]
+          vpdpbusd  -64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpbusd  64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x50,0x6c,0x24,0x04]
+          vpdpbusd  64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x50,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x50,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x50,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpbusds  (%rcx), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x51,0x29]
+          vpdpbusds  (%rcx), %xmm22, %xmm21
+
+// CHECK: vpdpbusds  -64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x51,0x6c,0x24,0xfc]
+          vpdpbusds  -64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpbusds  64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x51,0x6c,0x24,0x04]
+          vpdpbusds  64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x51,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x51,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x51,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpwssd  (%rcx), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x52,0x29]
+          vpdpwssd  (%rcx), %xmm22, %xmm21
+
+// CHECK: vpdpwssd  -64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x52,0x6c,0x24,0xfc]
+          vpdpwssd  -64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpwssd  64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x52,0x6c,0x24,0x04]
+          vpdpwssd  64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x52,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x52,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x52,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpwssds  (%rcx), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x53,0x29]
+          vpdpwssds  (%rcx), %xmm22, %xmm21
+
+// CHECK: vpdpwssds  -64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x53,0x6c,0x24,0xfc]
+          vpdpwssds  -64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpwssds  64(%rsp), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x00,0x53,0x6c,0x24,0x04]
+          vpdpwssds  64(%rsp), %xmm22, %xmm21
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x53,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x53,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x00,0x53,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %xmm22, %xmm21
+
+// CHECK: vpdpbusd  (%rcx), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x50,0x09]
+          vpdpbusd  (%rcx), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusd  -64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x50,0x4c,0x24,0xfc]
+          vpdpbusd  -64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusd  64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x50,0x4c,0x24,0x04]
+          vpdpbusd  64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x50,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x50,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x50,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusds  (%rcx), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x51,0x09]
+          vpdpbusds  (%rcx), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusds  -64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x51,0x4c,0x24,0xfc]
+          vpdpbusds  -64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusds  64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x51,0x4c,0x24,0x04]
+          vpdpbusds  64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x51,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x51,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x51,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssd  (%rcx), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x52,0x09]
+          vpdpwssd  (%rcx), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssd  -64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x52,0x4c,0x24,0xfc]
+          vpdpwssd  -64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssd  64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x52,0x4c,0x24,0x04]
+          vpdpwssd  64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x52,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x52,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x52,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssds  (%rcx), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x53,0x09]
+          vpdpwssds  (%rcx), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssds  -64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x53,0x4c,0x24,0xfc]
+          vpdpwssds  -64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssds  64(%rsp), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x0a,0x53,0x4c,0x24,0x04]
+          vpdpwssds  64(%rsp), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x53,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x53,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x0a,0x53,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %xmm2, %xmm1 {%k2}
+
+// CHECK: vpdpbusd  (%rcx), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x50,0x29]
+          vpdpbusd  (%rcx), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusd  -64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x50,0x6c,0x24,0xfc]
+          vpdpbusd  -64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusd  64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x50,0x6c,0x24,0x04]
+          vpdpbusd  64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x50,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x50,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x50,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusds  (%rcx), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x51,0x29]
+          vpdpbusds  (%rcx), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusds  -64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x51,0x6c,0x24,0xfc]
+          vpdpbusds  -64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusds  64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x51,0x6c,0x24,0x04]
+          vpdpbusds  64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x51,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x51,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x51,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssd  (%rcx), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x52,0x29]
+          vpdpwssd  (%rcx), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssd  -64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x52,0x6c,0x24,0xfc]
+          vpdpwssd  -64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssd  64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x52,0x6c,0x24,0x04]
+          vpdpwssd  64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x52,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x52,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x52,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssds  (%rcx), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x53,0x29]
+          vpdpwssds  (%rcx), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssds  -64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x53,0x6c,0x24,0xfc]
+          vpdpwssds  -64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssds  64(%rsp), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x02,0x53,0x6c,0x24,0x04]
+          vpdpwssds  64(%rsp), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x53,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x53,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x02,0x53,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %xmm22, %xmm21 {%k2}
+
+// CHECK: vpdpbusd %ymm3, %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x50,0xcb]
+          vpdpbusd %ymm3, %ymm2, %ymm1
+
+// CHECK: vpdpbusds %ymm3, %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x51,0xcb]
+          vpdpbusds %ymm3, %ymm2, %ymm1
+
+// CHECK: vpdpwssd %ymm3, %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x52,0xcb]
+          vpdpwssd %ymm3, %ymm2, %ymm1
+
+// CHECK: vpdpwssds %ymm3, %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x53,0xcb]
+          vpdpwssds %ymm3, %ymm2, %ymm1
+
+// CHECK: vpdpbusd %ymm23, %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x50,0xef]
+          vpdpbusd %ymm23, %ymm22, %ymm21
+
+// CHECK: vpdpbusds %ymm23, %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x51,0xef]
+          vpdpbusds %ymm23, %ymm22, %ymm21
+
+// CHECK: vpdpwssd %ymm23, %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x52,0xef]
+          vpdpwssd %ymm23, %ymm22, %ymm21
+
+// CHECK: vpdpwssds %ymm23, %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x53,0xef]
+          vpdpwssds %ymm23, %ymm22, %ymm21
+
+// CHECK: vpdpbusd %ymm3, %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x50,0xcb]
+          vpdpbusd %ymm3, %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusds %ymm3, %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x51,0xcb]
+          vpdpbusds %ymm3, %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssd %ymm3, %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x52,0xcb]
+          vpdpwssd %ymm3, %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssds %ymm3, %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x53,0xcb]
+          vpdpwssds %ymm3, %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusd %ymm23, %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x50,0xef]
+          vpdpbusd %ymm23, %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusds %ymm23, %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x51,0xef]
+          vpdpbusds %ymm23, %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssd %ymm23, %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x52,0xef]
+          vpdpwssd %ymm23, %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssds %ymm23, %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x53,0xef]
+          vpdpwssds %ymm23, %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusd  (%rcx), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x50,0x09]
+          vpdpbusd  (%rcx), %ymm2, %ymm1
+
+// CHECK: vpdpbusd  -128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x50,0x4c,0x24,0xfc]
+          vpdpbusd  -128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpbusd  128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x50,0x4c,0x24,0x04]
+          vpdpbusd  128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x50,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x50,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x50,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpbusds  (%rcx), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x51,0x09]
+          vpdpbusds  (%rcx), %ymm2, %ymm1
+
+// CHECK: vpdpbusds  -128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x51,0x4c,0x24,0xfc]
+          vpdpbusds  -128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpbusds  128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x51,0x4c,0x24,0x04]
+          vpdpbusds  128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x51,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x51,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x51,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpwssd  (%rcx), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x52,0x09]
+          vpdpwssd  (%rcx), %ymm2, %ymm1
+
+// CHECK: vpdpwssd  -128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x52,0x4c,0x24,0xfc]
+          vpdpwssd  -128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpwssd  128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x52,0x4c,0x24,0x04]
+          vpdpwssd  128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x52,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x52,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x52,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpwssds  (%rcx), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x53,0x09]
+          vpdpwssds  (%rcx), %ymm2, %ymm1
+
+// CHECK: vpdpwssds  -128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x53,0x4c,0x24,0xfc]
+          vpdpwssds  -128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpwssds  128(%rsp), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x28,0x53,0x4c,0x24,0x04]
+          vpdpwssds  128(%rsp), %ymm2, %ymm1
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x53,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x53,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x28,0x53,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %ymm2, %ymm1
+
+// CHECK: vpdpbusd  (%rcx), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x50,0x29]
+          vpdpbusd  (%rcx), %ymm22, %ymm21
+
+// CHECK: vpdpbusd  -128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x50,0x6c,0x24,0xfc]
+          vpdpbusd  -128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpbusd  128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x50,0x6c,0x24,0x04]
+          vpdpbusd  128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x50,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x50,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x50,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpbusds  (%rcx), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x51,0x29]
+          vpdpbusds  (%rcx), %ymm22, %ymm21
+
+// CHECK: vpdpbusds  -128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x51,0x6c,0x24,0xfc]
+          vpdpbusds  -128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpbusds  128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x51,0x6c,0x24,0x04]
+          vpdpbusds  128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x51,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x51,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x51,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpwssd  (%rcx), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x52,0x29]
+          vpdpwssd  (%rcx), %ymm22, %ymm21
+
+// CHECK: vpdpwssd  -128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x52,0x6c,0x24,0xfc]
+          vpdpwssd  -128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpwssd  128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x52,0x6c,0x24,0x04]
+          vpdpwssd  128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x52,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x52,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x52,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpwssds  (%rcx), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x53,0x29]
+          vpdpwssds  (%rcx), %ymm22, %ymm21
+
+// CHECK: vpdpwssds  -128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x53,0x6c,0x24,0xfc]
+          vpdpwssds  -128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpwssds  128(%rsp), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x20,0x53,0x6c,0x24,0x04]
+          vpdpwssds  128(%rsp), %ymm22, %ymm21
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x53,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x53,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x20,0x53,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %ymm22, %ymm21
+
+// CHECK: vpdpbusd  (%rcx), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x50,0x09]
+          vpdpbusd  (%rcx), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusd  -128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x50,0x4c,0x24,0xfc]
+          vpdpbusd  -128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusd  128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x50,0x4c,0x24,0x04]
+          vpdpbusd  128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x50,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x50,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x50,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusds  (%rcx), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x51,0x09]
+          vpdpbusds  (%rcx), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusds  -128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x51,0x4c,0x24,0xfc]
+          vpdpbusds  -128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusds  128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x51,0x4c,0x24,0x04]
+          vpdpbusds  128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x51,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x51,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x51,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssd  (%rcx), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x52,0x09]
+          vpdpwssd  (%rcx), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssd  -128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x52,0x4c,0x24,0xfc]
+          vpdpwssd  -128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssd  128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x52,0x4c,0x24,0x04]
+          vpdpwssd  128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x52,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x52,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x52,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssds  (%rcx), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x53,0x09]
+          vpdpwssds  (%rcx), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssds  -128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x53,0x4c,0x24,0xfc]
+          vpdpwssds  -128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssds  128(%rsp), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x2a,0x53,0x4c,0x24,0x04]
+          vpdpwssds  128(%rsp), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x53,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x53,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x2a,0x53,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %ymm2, %ymm1 {%k2}
+
+// CHECK: vpdpbusd  (%rcx), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x50,0x29]
+          vpdpbusd  (%rcx), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusd  -128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x50,0x6c,0x24,0xfc]
+          vpdpbusd  -128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusd  128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x50,0x6c,0x24,0x04]
+          vpdpbusd  128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x50,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x50,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x50,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusds  (%rcx), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x51,0x29]
+          vpdpbusds  (%rcx), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusds  -128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x51,0x6c,0x24,0xfc]
+          vpdpbusds  -128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusds  128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x51,0x6c,0x24,0x04]
+          vpdpbusds  128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x51,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x51,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x51,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssd  (%rcx), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x52,0x29]
+          vpdpwssd  (%rcx), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssd  -128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x52,0x6c,0x24,0xfc]
+          vpdpwssd  -128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssd  128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x52,0x6c,0x24,0x04]
+          vpdpwssd  128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x52,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x52,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x52,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssds  (%rcx), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x53,0x29]
+          vpdpwssds  (%rcx), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssds  -128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x53,0x6c,0x24,0xfc]
+          vpdpwssds  -128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssds  128(%rsp), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x22,0x53,0x6c,0x24,0x04]
+          vpdpwssds  128(%rsp), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x53,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x53,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x22,0x53,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %ymm22, %ymm21 {%k2}
+

test/MC/X86/avx512vnni-encoding.s

+// RUN: llvm-mc -triple x86_64-unknown-unknown -mcpu=knl -mattr=+avx512vnni --show-encoding < %s | FileCheck %s
+
+// CHECK: vpdpbusd %zmm3, %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x50,0xcb]
+          vpdpbusd %zmm3, %zmm2, %zmm1
+
+// CHECK: vpdpbusds %zmm3, %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x51,0xcb]
+          vpdpbusds %zmm3, %zmm2, %zmm1
+
+// CHECK: vpdpwssd %zmm3, %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x52,0xcb]
+          vpdpwssd %zmm3, %zmm2, %zmm1
+
+// CHECK: vpdpwssds %zmm3, %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x53,0xcb]
+          vpdpwssds %zmm3, %zmm2, %zmm1
+
+// CHECK: vpdpbusd %zmm23, %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x50,0xef]
+          vpdpbusd %zmm23, %zmm22, %zmm21
+
+// CHECK: vpdpbusds %zmm23, %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x51,0xef]
+          vpdpbusds %zmm23, %zmm22, %zmm21
+
+// CHECK: vpdpwssd %zmm23, %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x52,0xef]
+          vpdpwssd %zmm23, %zmm22, %zmm21
+
+// CHECK: vpdpwssds %zmm23, %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x53,0xef]
+          vpdpwssds %zmm23, %zmm22, %zmm21
+
+// CHECK: vpdpbusd %zmm3, %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x50,0xcb]
+          vpdpbusd %zmm3, %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusds %zmm3, %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x51,0xcb]
+          vpdpbusds %zmm3, %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssd %zmm3, %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x52,0xcb]
+          vpdpwssd %zmm3, %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssds %zmm3, %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x53,0xcb]
+          vpdpwssds %zmm3, %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusd %zmm23, %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x50,0xef]
+          vpdpbusd %zmm23, %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusds %zmm23, %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x51,0xef]
+          vpdpbusds %zmm23, %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssd %zmm23, %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x52,0xef]
+          vpdpwssd %zmm23, %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssds %zmm23, %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x53,0xef]
+          vpdpwssds %zmm23, %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusd  (%rcx), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x50,0x09]
+          vpdpbusd  (%rcx), %zmm2, %zmm1
+
+// CHECK: vpdpbusd  -256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x50,0x4c,0x24,0xfc]
+          vpdpbusd  -256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpbusd  256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x50,0x4c,0x24,0x04]
+          vpdpbusd  256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x50,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x50,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x50,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpbusds  (%rcx), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x51,0x09]
+          vpdpbusds  (%rcx), %zmm2, %zmm1
+
+// CHECK: vpdpbusds  -256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x51,0x4c,0x24,0xfc]
+          vpdpbusds  -256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpbusds  256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x51,0x4c,0x24,0x04]
+          vpdpbusds  256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x51,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x51,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x51,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpwssd  (%rcx), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x52,0x09]
+          vpdpwssd  (%rcx), %zmm2, %zmm1
+
+// CHECK: vpdpwssd  -256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x52,0x4c,0x24,0xfc]
+          vpdpwssd  -256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpwssd  256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x52,0x4c,0x24,0x04]
+          vpdpwssd  256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x52,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x52,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x52,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpwssds  (%rcx), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x53,0x09]
+          vpdpwssds  (%rcx), %zmm2, %zmm1
+
+// CHECK: vpdpwssds  -256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x53,0x4c,0x24,0xfc]
+          vpdpwssds  -256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpwssds  256(%rsp), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xf2,0x6d,0x48,0x53,0x4c,0x24,0x04]
+          vpdpwssds  256(%rsp), %zmm2, %zmm1
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x53,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x53,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+// CHECK: encoding: [0x62,0xb2,0x6d,0x48,0x53,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %zmm2, %zmm1
+
+// CHECK: vpdpbusd  (%rcx), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x50,0x29]
+          vpdpbusd  (%rcx), %zmm22, %zmm21
+
+// CHECK: vpdpbusd  -256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x50,0x6c,0x24,0xfc]
+          vpdpbusd  -256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpbusd  256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x50,0x6c,0x24,0x04]
+          vpdpbusd  256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x50,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x50,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x50,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpbusds  (%rcx), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x51,0x29]
+          vpdpbusds  (%rcx), %zmm22, %zmm21
+
+// CHECK: vpdpbusds  -256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x51,0x6c,0x24,0xfc]
+          vpdpbusds  -256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpbusds  256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x51,0x6c,0x24,0x04]
+          vpdpbusds  256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x51,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x51,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x51,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpwssd  (%rcx), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x52,0x29]
+          vpdpwssd  (%rcx), %zmm22, %zmm21
+
+// CHECK: vpdpwssd  -256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x52,0x6c,0x24,0xfc]
+          vpdpwssd  -256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpwssd  256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x52,0x6c,0x24,0x04]
+          vpdpwssd  256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x52,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x52,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x52,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpwssds  (%rcx), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x53,0x29]
+          vpdpwssds  (%rcx), %zmm22, %zmm21
+
+// CHECK: vpdpwssds  -256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x53,0x6c,0x24,0xfc]
+          vpdpwssds  -256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpwssds  256(%rsp), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xe2,0x4d,0x40,0x53,0x6c,0x24,0x04]
+          vpdpwssds  256(%rsp), %zmm22, %zmm21
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x53,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x53,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+// CHECK: encoding: [0x62,0xa2,0x4d,0x40,0x53,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %zmm22, %zmm21
+
+// CHECK: vpdpbusd  (%rcx), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x50,0x09]
+          vpdpbusd  (%rcx), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusd  -256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x50,0x4c,0x24,0xfc]
+          vpdpbusd  -256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusd  256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x50,0x4c,0x24,0x04]
+          vpdpbusd  256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x50,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x50,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x50,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusds  (%rcx), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x51,0x09]
+          vpdpbusds  (%rcx), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusds  -256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x51,0x4c,0x24,0xfc]
+          vpdpbusds  -256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusds  256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x51,0x4c,0x24,0x04]
+          vpdpbusds  256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x51,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x51,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x51,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssd  (%rcx), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x52,0x09]
+          vpdpwssd  (%rcx), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssd  -256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x52,0x4c,0x24,0xfc]
+          vpdpwssd  -256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssd  256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x52,0x4c,0x24,0x04]
+          vpdpwssd  256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x52,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x52,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x52,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssds  (%rcx), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x53,0x09]
+          vpdpwssds  (%rcx), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssds  -256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x53,0x4c,0x24,0xfc]
+          vpdpwssds  -256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssds  256(%rsp), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x6d,0x4a,0x53,0x4c,0x24,0x04]
+          vpdpwssds  256(%rsp), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x53,0x8c,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x53,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+// CHECK: encoding: [0x62,0xb2,0x6d,0x4a,0x53,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %zmm2, %zmm1 {%k2}
+
+// CHECK: vpdpbusd  (%rcx), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x50,0x29]
+          vpdpbusd  (%rcx), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusd  -256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x50,0x6c,0x24,0xfc]
+          vpdpbusd  -256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusd  256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x50,0x6c,0x24,0x04]
+          vpdpbusd  256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusd  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x50,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusd  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusd  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x50,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusd  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusd  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x50,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusd  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusds  (%rcx), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x51,0x29]
+          vpdpbusds  (%rcx), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusds  -256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x51,0x6c,0x24,0xfc]
+          vpdpbusds  -256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusds  256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x51,0x6c,0x24,0x04]
+          vpdpbusds  256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusds  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x51,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpbusds  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusds  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x51,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpbusds  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpbusds  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x51,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpbusds  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssd  (%rcx), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x52,0x29]
+          vpdpwssd  (%rcx), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssd  -256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x52,0x6c,0x24,0xfc]
+          vpdpwssd  -256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssd  256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x52,0x6c,0x24,0x04]
+          vpdpwssd  256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssd  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x52,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssd  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssd  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x52,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssd  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssd  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x52,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssd  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssds  (%rcx), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x53,0x29]
+          vpdpwssds  (%rcx), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssds  -256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x53,0x6c,0x24,0xfc]
+          vpdpwssds  -256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssds  256(%rsp), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xe2,0x4d,0x42,0x53,0x6c,0x24,0x04]
+          vpdpwssds  256(%rsp), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssds  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x53,0xac,0xf1,0x00,0x00,0x00,0x10]
+          vpdpwssds  268435456(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssds  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x53,0xac,0xf1,0x00,0x00,0x00,0xe0]
+          vpdpwssds  -536870912(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+
+// CHECK: vpdpwssds  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+// CHECK: encoding: [0x62,0xa2,0x4d,0x42,0x53,0xac,0xf1,0x02,0x00,0x00,0xe0]
+          vpdpwssds  -536870910(%rcx,%r14,8), %zmm22, %zmm21 {%k2}
+