[DAGCombiner][X86] Disable narrowExtractedVectorLoad if the element type size isn't byte sized

[craig.topper]

Summary

The address calculation for the offset assumes that you can calculate the offset by multiplying the index by the store size of the element. But that only works if the element'ss store size is exactly its real size since we store vectors tightly packed in memory. There are improvements we could make to this like special casing extracting element 0. I think we could also handle cases where the extracted VT is byte sized and the index is aligned with the extract element count.

[craig.topper]

llvm/test/CodeGen/X86/avx512-extract-subvector-load-store.ll

624

I think this is because we're now hitting scalarizing code in the type legalizer.

[RKSimon]

LGTM for correctness - are you intending to improve the codegen later on?

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
   auto *Ld = dyn_cast<LoadSDNode>(Extract->getOperand(0));
   auto *ExtIdx = dyn_cast<ConstantSDNode>(Extract->getOperand(1));
   if (!Ld || Ld->getExtensionType() || !Ld->isSimple() ||
       !ExtIdx)
     return SDValue();
 
   // Allow targets to opt-out.
   EVT VT = Extract->getValueType(0);
+
+  // Only handle byte sized scalars otherwise the offset is incorrect.
+  // FIXME: We might be able to do better if the VT is byte sized and the index
+  // is aligned.
+  if (!VT.getScalarType().isByteSized())
+    return SDValue();
+
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (!TLI.shouldReduceLoadWidth(Ld, Ld->getExtensionType(), VT))
     return SDValue();
 
   // The narrow load will be offset from the base address of the old load if
   // we are extracting from something besides index 0 (little-endian).
   SDLoc DL(Extract);
   SDValue BaseAddr = Ld->getOperand(1);

llvm/test/CodeGen/X86/avx512-extract-subvector-load-store.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq,+fast-variable-shuffle -O2 | FileCheck %s --check-prefix=AVX512
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+fast-variable-shuffle -O2 | FileCheck %s --check-prefix=AVX512NOTDQ
 
 define void @load_v8i1_broadcast_4_v2i1(<8 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v8i1_broadcast_4_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 4(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $4, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 4(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $4, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 4,i32 4>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v8i1_broadcast_7_v2i1(<8 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v8i1_broadcast_7_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 6(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $6, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 6(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $6, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 7,i32 7>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v16i1_broadcast_8_v2i1(<16 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v16i1_broadcast_8_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 8(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $8, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $8, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 8,i32 8>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v16i1_broadcast_8_v4i1(<16 x i1>* %a0,<4 x float> %a1,<4 x float> %a2,<4 x float>* %a3) {
 ; AVX512-LABEL: load_v16i1_broadcast_8_v4i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 8(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $8, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm2
 ; AVX512-NEXT:    vpbroadcastd %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovd2m %xmm2, %k1
 ; AVX512-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $8, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 8,i32 8,i32 8,i32 8>
     %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2
     store <4 x float> %d2, <4 x float>* %a3
     ret void
 }
 define void @load_v16i1_broadcast_15_v2i1(<16 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v16i1_broadcast_15_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 14(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $14, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 14(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 15,i32 15>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v16i1_broadcast_15_v4i1(<16 x i1>* %a0,<4 x float> %a1,<4 x float> %a2,<4 x float>* %a3) {
 ; AVX512-LABEL: load_v16i1_broadcast_15_v4i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 12(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vpmovd2m %xmm2, %k1
 ; AVX512-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 12(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $12, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 15,i32 15,i32 15,i32 15>
     %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2
     store <4 x float> %d2, <4 x float>* %a3
     ret void
 }
 define void @load_v32i1_broadcast_16_v2i1(<32 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 16(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $16, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 16,i32 16>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v32i1_broadcast_16_v4i1(<32 x i1>* %a0,<4 x float> %a1,<4 x float> %a2,<4 x float>* %a3) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v4i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 16(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $16, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm2
 ; AVX512-NEXT:    vpbroadcastd %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovd2m %xmm2, %k1
 ; AVX512-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 16,i32 16,i32 16,i32 16>
     %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2
     store <4 x float> %d2, <4 x float>* %a3
     ret void
 }
 define void @load_v32i1_broadcast_16_v8i1(<32 x i1>* %a0,<8 x float> %a1,<8 x float> %a2,<8 x float>* %a3) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v8i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 16(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $16, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm2
 ; AVX512-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512-NEXT:    vpmovd2m %ymm2, %k1
 ; AVX512-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm2, %ymm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16>
     %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2
     store <8 x float> %d2, <8 x float>* %a3
     ret void
 }
 define void @load_v32i1_broadcast_31_v2i1(<32 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 30(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $30, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 30(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $30, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 31,i32 31>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v32i1_broadcast_31_v4i1(<32 x i1>* %a0,<4 x float> %a1,<4 x float> %a2,<4 x float>* %a3) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v4i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 28(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $28, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vpmovd2m %xmm2, %k1
 ; AVX512-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 28(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $28, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 31,i32 31,i32 31,i32 31>
     %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2
     store <4 x float> %d2, <4 x float>* %a3
     ret void
 }
 define void @load_v32i1_broadcast_31_v8i1(<32 x i1>* %a0,<8 x float> %a1,<8 x float> %a2,<8 x float>* %a3) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v8i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 24(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $24, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm2
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]
 ; AVX512-NEXT:    vpermd %ymm2, %ymm3, %ymm2
 ; AVX512-NEXT:    vpmovd2m %ymm2, %k1
 ; AVX512-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 24(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $24, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]
 ; AVX512NOTDQ-NEXT:    vpermd %ymm2, %ymm3, %ymm2
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm2, %ymm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31>
     %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2
     store <8 x float> %d2, <8 x float>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_32_v2i1(<64 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 32,i32 32>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_32_v4i1(<64 x i1>* %a0,<4 x float> %a1,<4 x float> %a2,<4 x float>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v4i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm2
 ; AVX512-NEXT:    vpbroadcastd %xmm2, %xmm2
 ; AVX512-NEXT:    vpmovd2m %xmm2, %k1
 ; AVX512-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 32,i32 32,i32 32,i32 32>
     %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2
     store <4 x float> %d2, <4 x float>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_32_v8i1(<64 x i1>* %a0,<8 x float> %a1,<8 x float> %a2,<8 x float>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v8i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm2
 ; AVX512-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512-NEXT:    vpmovd2m %ymm2, %k1
 ; AVX512-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm2, %ymm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>
     %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2
     store <8 x float> %d2, <8 x float>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_32_v16i1(<64 x i1>* %a0,<16 x float> %a1,<16 x float> %a2,<16 x float>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v16i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovw 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %zmm2
 ; AVX512-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512-NEXT:    vpmovd2m %zmm2, %k1
 ; AVX512-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %zmm1, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v16i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512NOTDQ-NEXT:    vptestmd %zmm2, %zmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %zmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>
     %d2 = select <16 x i1> %d1, <16 x float> %a1, <16 x float> %a2
     store <16 x float> %d2, <16 x float>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_63_v2i1(<64 x i1>* %a0,<2 x double> %a1,<2 x double> %a2,<2 x double>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v2i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 62(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $62, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm2, %k1
 ; AVX512-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 62(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $62, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovapd %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 63,i32 63>
     %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2
     store <2 x double> %d2, <2 x double>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_63_v4i1(<64 x i1>* %a0,<4 x float> %a1,<4 x float> %a2,<4 x float>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v4i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 60(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $60, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vpmovd2m %xmm2, %k1
 ; AVX512-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 60(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $60, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm2, %xmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm0, %xmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %xmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 63,i32 63,i32 63,i32 63>
     %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2
     store <4 x float> %d2, <4 x float>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_63_v8i1(<64 x i1>* %a0,<8 x float> %a1,<8 x float> %a2,<8 x float>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v8i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 56(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $56, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm2
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]
 ; AVX512-NEXT:    vpermd %ymm2, %ymm3, %ymm2
 ; AVX512-NEXT:    vpmovd2m %ymm2, %k1
 ; AVX512-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 56(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $56, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]
 ; AVX512NOTDQ-NEXT:    vpermd %ymm2, %ymm3, %ymm2
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm2, %ymm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm0, %ymm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>
     %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2
     store <8 x float> %d2, <8 x float>* %a3
     ret void
 }
 define void @load_v64i1_broadcast_63_v16i1(<64 x i1>* %a0,<16 x float> %a1,<16 x float> %a2,<16 x float>* %a3) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v16i1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovw 48(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $48, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %zmm2
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512-NEXT:    vpermd %zmm2, %zmm3, %zmm2
 ; AVX512-NEXT:    vpmovd2m %zmm2, %k1
 ; AVX512-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %zmm1, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v16i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 48(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $48, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512NOTDQ-NEXT:    vpermd %zmm2, %zmm3, %zmm2
 ; AVX512NOTDQ-NEXT:    vptestmd %zmm2, %zmm2, %k1
 ; AVX512NOTDQ-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
 ; AVX512NOTDQ-NEXT:    vmovaps %zmm1, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>
     %d2 = select <16 x i1> %d1, <16 x float> %a1, <16 x float> %a2
     store <16 x float> %d2, <16 x float>* %a3
     ret void
 }
 define void @load_v2i1_broadcast_1_v1i1_store(<2 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v2i1_broadcast_1_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 1(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $1, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v2i1_broadcast_1_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 1(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <2 x i1>, <2 x i1>* %a0
     %d1 = shufflevector <2 x i1> %d0,<2 x i1> undef,<1 x i32><i32 1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v3i1_broadcast_1_v1i1_store(<3 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v3i1_broadcast_1_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 1(%rdi), %k0
+; AVX512-NEXT:    movzbl (%rdi), %eax
+; AVX512-NEXT:    xorl %ecx, %ecx
+; AVX512-NEXT:    btl $1, %eax
+; AVX512-NEXT:    movl $255, %eax
+; AVX512-NEXT:    cmovael %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
+; AVX512-NEXT:    kshiftrb $1, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v3i1_broadcast_1_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 1(%rdi), %al
+; AVX512NOTDQ-NEXT:    movzbl (%rdi), %eax
+; AVX512NOTDQ-NEXT:    xorl %ecx, %ecx
+; AVX512NOTDQ-NEXT:    btl $1, %eax
+; AVX512NOTDQ-NEXT:    movl $255, %eax
+; AVX512NOTDQ-NEXT:    cmovael %ecx, %eax
+; AVX512NOTDQ-NEXT:    kmovd %eax, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <3 x i1>, <3 x i1>* %a0
     %d1 = shufflevector <3 x i1> %d0,<3 x i1> undef,<1 x i32><i32 1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v3i1_broadcast_2_v1i1_store(<3 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v3i1_broadcast_2_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 2(%rdi), %k0
+; AVX512-NEXT:    movzbl (%rdi), %eax
+; AVX512-NEXT:    xorl %ecx, %ecx
+; AVX512-NEXT:    btl $2, %eax
+; AVX512-NEXT:    movl $255, %eax
+; AVX512-NEXT:    cmovael %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
+; AVX512-NEXT:    kshiftrb $2, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v3i1_broadcast_2_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 2(%rdi), %al
+; AVX512NOTDQ-NEXT:    movzbl (%rdi), %eax
+; AVX512NOTDQ-NEXT:    xorl %ecx, %ecx
+; AVX512NOTDQ-NEXT:    btl $2, %eax
+; AVX512NOTDQ-NEXT:    movl $255, %eax
+; AVX512NOTDQ-NEXT:    cmovael %ecx, %eax
+; AVX512NOTDQ-NEXT:    kmovd %eax, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <3 x i1>, <3 x i1>* %a0
     %d1 = shufflevector <3 x i1> %d0,<3 x i1> undef,<1 x i32><i32 2>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v4i1_broadcast_2_v1i1_store(<4 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v4i1_broadcast_2_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 2(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $2, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v4i1_broadcast_2_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 2(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <4 x i1>, <4 x i1>* %a0
     %d1 = shufflevector <4 x i1> %d0,<4 x i1> undef,<1 x i32><i32 2>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v4i1_broadcast_3_v1i1_store(<4 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v4i1_broadcast_3_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 3(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $3, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v4i1_broadcast_3_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 3(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <4 x i1>, <4 x i1>* %a0
     %d1 = shufflevector <4 x i1> %d0,<4 x i1> undef,<1 x i32><i32 3>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v8i1_broadcast_4_v1i1_store(<8 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v8i1_broadcast_4_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 4(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $4, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 4(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<1 x i32><i32 4>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v8i1_broadcast_4_v2i1_store(<8 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v8i1_broadcast_4_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 4(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $4, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 4(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $4, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 4,i32 4>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v8i1_broadcast_7_v1i1_store(<8 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v8i1_broadcast_7_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 7(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 7(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<1 x i32><i32 7>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v8i1_broadcast_7_v2i1_store(<8 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v8i1_broadcast_7_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 6(%rdi), %k0
+; AVX512-NEXT:    kmovb (%rdi), %k0
+; AVX512-NEXT:    kshiftrb $6, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 6(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $6, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 7,i32 7>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_8_v1i1_store(<16 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_8_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 8(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $8, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 8(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<1 x i32><i32 8>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_8_v2i1_store(<16 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_8_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 8(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $8, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $8, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 8,i32 8>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_8_v4i1_store(<16 x i1>* %a0,<4 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_8_v4i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 8(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $8, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovd2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $8, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 8,i32 8,i32 8,i32 8>
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_15_v1i1_store(<16 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_15_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 15(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 15(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<1 x i32><i32 15>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_15_v2i1_store(<16 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_15_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 14(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $14, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 14(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 15,i32 15>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_15_v4i1_store(<16 x i1>* %a0,<4 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_15_v4i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 12(%rdi), %k0
+; AVX512-NEXT:    kmovw (%rdi), %k0
+; AVX512-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vpmovd2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 12(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $12, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 15,i32 15,i32 15,i32 15>
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_16_v1i1_store(<32 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 16(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $16, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 16(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<1 x i32><i32 16>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_16_v2i1_store(<32 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 16(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $16, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 16,i32 16>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_16_v4i1_store(<32 x i1>* %a0,<4 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v4i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 16(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $16, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovd2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 16,i32 16,i32 16,i32 16>
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_16_v8i1_store(<32 x i1>* %a0,<8 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v8i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 16(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $16, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm0
 ; AVX512-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512-NEXT:    vpmovd2m %ymm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16>
     store <8 x i1> %d1, <8 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_31_v1i1_store(<32 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 31(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $31, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 31(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $31, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<1 x i32><i32 31>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_31_v2i1_store(<32 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 30(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $30, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 30(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $30, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 31,i32 31>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_31_v4i1_store(<32 x i1>* %a0,<4 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v4i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 28(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $28, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vpmovd2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 28(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $28, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 31,i32 31,i32 31,i32 31>
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_31_v8i1_store(<32 x i1>* %a0,<8 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v8i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 24(%rdi), %k0
+; AVX512-NEXT:    kmovd (%rdi), %k0
+; AVX512-NEXT:    kshiftrd $24, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]
 ; AVX512-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vpmovd2m %ymm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 24(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrd $24, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]
 ; AVX512NOTDQ-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31>
     store <8 x i1> %d1, <8 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_32_v1i1_store(<64 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 32(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<1 x i32><i32 32>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_32_v2i1_store(<64 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 32,i32 32>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_32_v4i1_store(<64 x i1>* %a0,<4 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v4i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovd2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 32,i32 32,i32 32,i32 32>
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_32_v8i1_store(<64 x i1>* %a0,<8 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v8i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm0
 ; AVX512-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512-NEXT:    vpmovd2m %ymm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>
     store <8 x i1> %d1, <8 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_32_v16i1_store(<64 x i1>* %a0,<16 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v16i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovw 32(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $32, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %zmm0
 ; AVX512-NEXT:    vpbroadcastd %xmm0, %zmm0
 ; AVX512-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512-NEXT:    kmovw %k0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v16i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %zmm0
 ; AVX512NOTDQ-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovw %k0, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>
     store <16 x i1> %d1, <16 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_63_v1i1_store(<64 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v1i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 63(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $63, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movb 63(%rdi), %al
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $63, %k0, %k0
+; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<1 x i32><i32 63>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_63_v2i1_store(<64 x i1>* %a0,<2 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v2i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 62(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $62, %k0, %k0
 ; AVX512-NEXT:    vpmovm2q %k0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpmovq2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 62(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $62, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX512NOTDQ-NEXT:    vptestmq %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 63,i32 63>
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_63_v4i1_store(<64 x i1>* %a0,<4 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v4i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 60(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $60, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512-NEXT:    vpmovd2m %xmm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 60(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $60, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; AVX512NOTDQ-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 63,i32 63,i32 63,i32 63>
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_63_v8i1_store(<64 x i1>* %a0,<8 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v8i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovb 56(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $56, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %ymm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]
 ; AVX512-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vpmovd2m %ymm0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 56(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $56, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]
 ; AVX512NOTDQ-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX512NOTDQ-NEXT:    vptestmd %ymm0, %ymm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>
     store <8 x i1> %d1, <8 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_63_v16i1_store(<64 x i1>* %a0,<16 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v16i1_store:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    kmovw 48(%rdi), %k0
+; AVX512-NEXT:    kmovq (%rdi), %k0
+; AVX512-NEXT:    kshiftrq $48, %k0, %k0
 ; AVX512-NEXT:    vpmovm2d %k0, %zmm0
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512-NEXT:    kmovw %k0, (%rsi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v16i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    kmovw 48(%rdi), %k1
+; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
+; AVX512NOTDQ-NEXT:    kshiftrq $48, %k0, %k1
 ; AVX512NOTDQ-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512NOTDQ-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512NOTDQ-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512NOTDQ-NEXT:    kmovw %k0, (%rsi)
 ; AVX512NOTDQ-NEXT:    vzeroupper
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>
     store <16 x i1> %d1, <16 x i1>* %a1
     ret void
 }
 

llvm/test/CodeGen/X86/avx512-mask-op.ll

 ; AVX512DQ-NEXT:    kmovw %edi, %k0
 ; AVX512DQ-NEXT:    kshiftrw $8, %k0, %k0
 ; AVX512DQ-NEXT:    kmovw %k0, %eax
 ; AVX512DQ-NEXT:    ## kill: def $al killed $al killed $eax
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: shuf_test1:
 ; X86:       ## %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k0
+; X86-NEXT:    kshiftrw $8, %k0, %k0
+; X86-NEXT:    kmovd %k0, %eax
+; X86-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
    %v1 = bitcast i16 %v to <16 x i1>
    %mask = shufflevector <16 x i1> %v1, <16 x i1> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
    %mask1 = bitcast <8 x i1> %mask to i8
    ret i8 %mask1
 }
 
 define i32 @zext_test1(<16 x i32> %a, <16 x i32> %b) {

llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512bw --show-mc-encoding | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw --show-mc-encoding | FileCheck %s --check-prefixes=CHECK,X64
 
 declare i32 @llvm.x86.avx512.kunpck.wd(i32, i32)
 
 define i32@test_int_x86_avx512_kunpck_wd(i32 %x0, i32 %x1) {
 ; X86-LABEL: test_int_x86_avx512_kunpck_wd:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k0 # encoding: [0xc5,0xf8,0x90,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0 # encoding: [0xc4,0xe1,0xf9,0x90,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    kunpckwd %k1, %k0, %k0 # encoding: [0xc5,0xfc,0x4b,0xc1]
 ; X86-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_kunpck_wd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k0 # encoding: [0xc5,0xfb,0x92,0xc7]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    kunpckwd %k1, %k0, %k0 # encoding: [0xc5,0xfc,0x4b,0xc1]
 ; X64-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call i32 @llvm.x86.avx512.kunpck.wd(i32 %x0, i32 %x1)
   ret i32 %res
 }
 
 declare i64 @llvm.x86.avx512.kunpck.dq(i64, i64)
 
 define i64@test_int_x86_avx512_kunpck_qd(i64 %x0, i64 %x1) {
 ; X86-LABEL: test_int_x86_avx512_kunpck_qd:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx # encoding: [0x8b,0x54,0x24,0x04]
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x0c]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k0 # encoding: [0xc4,0xe1,0xf8,0x90,0x44,0x24,0x04]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x0c]
+; X86-NEXT:    kmovd %k0, %edx # encoding: [0xc5,0xfb,0x93,0xd0]
+; X86-NEXT:    kmovd %k1, %eax # encoding: [0xc5,0xfb,0x93,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_kunpck_qd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovq %rdi, %k0 # encoding: [0xc4,0xe1,0xfb,0x92,0xc7]
 ; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
 ; X64-NEXT:    kunpckdq %k1, %k0, %k0 # encoding: [0xc4,0xe1,0xfc,0x4b,0xc1]
 ; X64-NEXT:    kmovq %k0, %rax # encoding: [0xc4,0xe1,0xfb,0x93,0xc0]