[X86] When storing v1i1/v2i1/v4i1 to memory, make sure we store zeros in the rest of the byte

[craig.topper]

Summary

We can't store garbage in the unused bits. It possible that something like zextload from i1/i2/i4 is created to read the memory. Those zextloads would be legalized assuming the extra bits are 0.

I'm not sure that the code in lowerStore is executed for the v1i1/v2i1/v4i1 case. It looks like the DAG combine in combineStore may have converted them to v8i1 first. And I think we're missing some cases to avoid going to the stack in the first place. But I don't have time to investigate those things at the moment so I wanted to focus on the correctness issue.

Should fix PR48147.

[RKSimon]

LGTM - there's a number of optimisations in the new kshift mask clearing codegen that we're missing but we can fix those up later.

llvm/lib/Target/X86/X86ISelLowering.cpp

23882

Any idea why we pad with a INSERT_SUBVECTOR here but use a CONCAT_VECTORS in the other case?

[craig.topper]

llvm/lib/Target/X86/X86ISelLowering.cpp

23882

I think because combineStore runs before type legalization and we try to avoid INSERT_SUBVECTOR early. LowerSTORE runs later, but I guess combineStore is called from LegalizeVectorOps and CONCAT_VECTORS wouldn't become INSERT_SUBVECTOR until LegalizeDAG. So maybe lowerStore should be using CONCAT_VECTORS as well?

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

23882

Yes, consistently using CONCAT_VECTORS makes sense to me assuming there's no post-legalization gotchas.

llvm/lib/Target/X86/X86ISelLowering.cpp

static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
                           SelectionDAG &DAG) {
   StoreSDNode *St = cast<StoreSDNode>(Op.getNode());
   SDLoc dl(St);
   SDValue StoredVal = St->getValue();
 
   // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 stores.
   if (StoredVal.getValueType().isVector() &&
       StoredVal.getValueType().getVectorElementType() == MVT::i1) {
-    assert(StoredVal.getValueType().getVectorNumElements() <= 8 &&
-           "Unexpected VT");
+    unsigned NumElts = StoredVal.getValueType().getVectorNumElements();
+    assert(NumElts <= 8 && "Unexpected VT");
     assert(!St->isTruncatingStore() && "Expected non-truncating store");
     assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
            "Expected AVX512F without AVX512DQI");
 
+    // We must pad with zeros to ensure we store zeroes to any unused bits.
     StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
                             DAG.getUNDEF(MVT::v16i1), StoredVal,
                             DAG.getIntPtrConstant(0, dl));
     StoredVal = DAG.getBitcast(MVT::i16, StoredVal);
     StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);
+    // Make sure we store zeros in the extra bits.
+    if (NumElts < 8)
+      StoredVal = DAG.getZeroExtendInReg(StoredVal, dl,
+                                         MVT::getIntegerVT(NumElts));
 
     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
                         St->getPointerInfo(), St->getOriginalAlign(),
                         St->getMemOperand()->getFlags());
   }
 
   if (St->isTruncatingStore())
     return SDValue();
return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
                         St->getMemOperand()->getFlags());
   }
 
   // If this is a store of a scalar_to_vector to v1i1, just use a scalar store.
   // This will avoid a copy to k-register.
   if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() &&
       StoredVal.getOpcode() == ISD::SCALAR_TO_VECTOR &&
       StoredVal.getOperand(0).getValueType() == MVT::i8) {
-    return DAG.getStore(St->getChain(), dl, StoredVal.getOperand(0),
+    SDValue Val = StoredVal.getOperand(0);
+    // We must store zeros to the unused bits.
+    Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1);
+    return DAG.getStore(St->getChain(), dl, Val,
                         St->getBasePtr(), St->getPointerInfo(),
                         St->getOriginalAlign(),
                         St->getMemOperand()->getFlags());
   }
 
   // Widen v2i1/v4i1 stores to v8i1.
-  if ((VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&
+  if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&
       Subtarget.hasAVX512()) {
     unsigned NumConcats = 8 / VT.getVectorNumElements();
-    SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT));
+    // We must store zeros to the unused bits.
+    SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT));
     Ops[0] = StoredVal;
     StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
                         St->getPointerInfo(), St->getOriginalAlign(),
                         St->getMemOperand()->getFlags());
   }
 
   // Turn vXi1 stores of constants into a scalar store.

llvm/lib/Target/X86/X86InstrAVX512.td

def : Pat<(i32 (bitconvert (v32i1 VK32:$src))),
           (COPY_TO_REGCLASS VK32:$src, GR32)>;
 def : Pat<(v64i1 (bitconvert (i64 GR64:$src))),
           (COPY_TO_REGCLASS GR64:$src, VK64)>;
 def : Pat<(i64 (bitconvert (v64i1 VK64:$src))),
           (COPY_TO_REGCLASS VK64:$src, GR64)>;
 
 // Load/store kreg
 let Predicates = [HasDQI] in {
-  def : Pat<(store VK1:$src, addr:$dst),
-            (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK1:$src, VK8))>;
-
   def : Pat<(v1i1 (load addr:$src)),
             (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK1)>;
   def : Pat<(v2i1 (load addr:$src)),
             (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK2)>;
   def : Pat<(v4i1 (load addr:$src)),
             (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK4)>;
 }
 

llvm/test/CodeGen/X86/avx512-extract-subvector-load-store.ll

; AVX512NOTDQ-NEXT:    retq
     store <16 x float> %d2, <16 x float>* %a3
     ret void
 }
 define void @load_v2i1_broadcast_1_v1i1_store(<2 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v2i1_broadcast_1_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovb (%rdi), %k0
 ; AVX512-NEXT:    kshiftrb $1, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v2i1_broadcast_1_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <2 x i1>, <2 x i1>* %a0
     %d1 = shufflevector <2 x i1> %d0,<2 x i1> undef,<1 x i32><i32 1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v3i1_broadcast_1_v1i1_store(<3 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v3i1_broadcast_1_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movb (%rdi), %al
 ; AVX512-NEXT:    shrb %al
 ; AVX512-NEXT:    xorl %ecx, %ecx
 ; AVX512-NEXT:    testb $1, %al
 ; AVX512-NEXT:    movl $255, %eax
 ; AVX512-NEXT:    cmovel %ecx, %eax
 ; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    kshiftrb $1, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v3i1_broadcast_1_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    movb (%rdi), %al
 ; AVX512NOTDQ-NEXT:    shrb %al
 ; AVX512NOTDQ-NEXT:    xorl %ecx, %ecx
 ; AVX512NOTDQ-NEXT:    testb $1, %al
 ; AVX512NOTDQ-NEXT:    movl $255, %eax
 ; AVX512NOTDQ-NEXT:    cmovel %ecx, %eax
 ; AVX512NOTDQ-NEXT:    kmovd %eax, %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <3 x i1>, <3 x i1>* %a0
     %d1 = shufflevector <3 x i1> %d0,<3 x i1> undef,<1 x i32><i32 1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v3i1_broadcast_2_v1i1_store(<3 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v3i1_broadcast_2_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    xorl %eax, %eax
 ; AVX512-NEXT:    testb $4, (%rdi)
 ; AVX512-NEXT:    movl $255, %ecx
 ; AVX512-NEXT:    cmovel %eax, %ecx
 ; AVX512-NEXT:    kmovd %ecx, %k0
 ; AVX512-NEXT:    kshiftrb $2, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v3i1_broadcast_2_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    xorl %eax, %eax
 ; AVX512NOTDQ-NEXT:    testb $4, (%rdi)
 ; AVX512NOTDQ-NEXT:    movl $255, %ecx
 ; AVX512NOTDQ-NEXT:    cmovel %eax, %ecx
 ; AVX512NOTDQ-NEXT:    kmovd %ecx, %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <3 x i1>, <3 x i1>* %a0
     %d1 = shufflevector <3 x i1> %d0,<3 x i1> undef,<1 x i32><i32 2>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v4i1_broadcast_2_v1i1_store(<4 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v4i1_broadcast_2_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovb (%rdi), %k0
 ; AVX512-NEXT:    kshiftrb $2, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v4i1_broadcast_2_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <4 x i1>, <4 x i1>* %a0
     %d1 = shufflevector <4 x i1> %d0,<4 x i1> undef,<1 x i32><i32 2>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v4i1_broadcast_3_v1i1_store(<4 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v4i1_broadcast_3_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovb (%rdi), %k0
 ; AVX512-NEXT:    kshiftrb $3, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v4i1_broadcast_3_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <4 x i1>, <4 x i1>* %a0
     %d1 = shufflevector <4 x i1> %d0,<4 x i1> undef,<1 x i32><i32 3>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v8i1_broadcast_4_v1i1_store(<8 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v8i1_broadcast_4_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovb (%rdi), %k0
 ; AVX512-NEXT:    kshiftrb $4, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<1 x i32><i32 4>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
; AVX512NOTDQ-NEXT:    retq
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v8i1_broadcast_7_v1i1_store(<8 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v8i1_broadcast_7_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovb (%rdi), %k0
 ; AVX512-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <8 x i1>, <8 x i1>* %a0
     %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<1 x i32><i32 7>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
; AVX512NOTDQ-NEXT:    retq
     store <2 x i1> %d1, <2 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_8_v1i1_store(<16 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_8_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovw (%rdi), %k0
 ; AVX512-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<1 x i32><i32 8>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
; AVX512NOTDQ-NEXT:    retq
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v16i1_broadcast_15_v1i1_store(<16 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v16i1_broadcast_15_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovw (%rdi), %k0
 ; AVX512-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovw (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <16 x i1>, <16 x i1>* %a0
     %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<1 x i32><i32 15>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
; AVX512NOTDQ-NEXT:    retq
     store <4 x i1> %d1, <4 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_16_v1i1_store(<32 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_16_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovd (%rdi), %k0
 ; AVX512-NEXT:    kshiftrd $16, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrd $16, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<1 x i32><i32 16>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
; AVX512NOTDQ-NEXT:    retq
     store <8 x i1> %d1, <8 x i1>* %a1
     ret void
 }
 define void @load_v32i1_broadcast_31_v1i1_store(<32 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v32i1_broadcast_31_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovd (%rdi), %k0
 ; AVX512-NEXT:    kshiftrd $31, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovd (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrd $31, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <32 x i1>, <32 x i1>* %a0
     %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<1 x i32><i32 31>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
; AVX512NOTDQ-NEXT:    retq
     store <8 x i1> %d1, <8 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_32_v1i1_store(<64 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_32_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovq (%rdi), %k0
 ; AVX512-NEXT:    kshiftrq $32, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrq $32, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<1 x i32><i32 32>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
; AVX512NOTDQ-NEXT:    retq
     store <16 x i1> %d1, <16 x i1>* %a1
     ret void
 }
 define void @load_v64i1_broadcast_63_v1i1_store(<64 x i1>* %a0,<1 x i1>* %a1) {
 ; AVX512-LABEL: load_v64i1_broadcast_63_v1i1_store:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovq (%rdi), %k0
 ; AVX512-NEXT:    kshiftrq $63, %k0, %k0
+; AVX512-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-NEXT:    retq
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v1i1_store:
 ; AVX512NOTDQ:       # %bb.0:
 ; AVX512NOTDQ-NEXT:    kmovq (%rdi), %k0
 ; AVX512NOTDQ-NEXT:    kshiftrq $63, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512NOTDQ-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512NOTDQ-NEXT:    kmovd %k0, %eax
 ; AVX512NOTDQ-NEXT:    movb %al, (%rsi)
 ; AVX512NOTDQ-NEXT:    retq
     %d0 = load <64 x i1>, <64 x i1>* %a0
     %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<1 x i32><i32 63>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }

llvm/test/CodeGen/X86/avx512-load-trunc-store-i1.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq -O2 | FileCheck %s --check-prefix=AVX512-ALL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O2| FileCheck %s --check-prefix=AVX512-ONLY
 
 define void @load_v1i2_trunc_v1i1_store(<1 x i2>* %a0,<1 x i1>* %a1) {
 ; AVX512-ALL-LABEL: load_v1i2_trunc_v1i1_store:
 ; AVX512-ALL:       # %bb.0:
-; AVX512-ALL-NEXT:    movb (%rdi), %al
-; AVX512-ALL-NEXT:    movb %al, (%rsi)
+; AVX512-ALL-NEXT:    kmovb (%rdi), %k0
+; AVX512-ALL-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-ALL-NEXT:    retq
 ;
 ; AVX512-ONLY-LABEL: load_v1i2_trunc_v1i1_store:
 ; AVX512-ONLY:       # %bb.0:
 ; AVX512-ONLY-NEXT:    movb (%rdi), %al
+; AVX512-ONLY-NEXT:    andl $1, %eax
+; AVX512-ONLY-NEXT:    kmovw %eax, %k0
+; AVX512-ONLY-NEXT:    kmovw %k0, %eax
 ; AVX512-ONLY-NEXT:    movb %al, (%rsi)
 ; AVX512-ONLY-NEXT:    retq
     %d0 = load <1 x i2>, <1 x i2>* %a0
     %d1 = trunc <1 x i2> %d0 to <1 x i1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v1i3_trunc_v1i1_store(<1 x i3>* %a0,<1 x i1>* %a1) {
 ; AVX512-ALL-LABEL: load_v1i3_trunc_v1i1_store:
 ; AVX512-ALL:       # %bb.0:
-; AVX512-ALL-NEXT:    movb (%rdi), %al
-; AVX512-ALL-NEXT:    movb %al, (%rsi)
+; AVX512-ALL-NEXT:    kmovb (%rdi), %k0
+; AVX512-ALL-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-ALL-NEXT:    retq
 ;
 ; AVX512-ONLY-LABEL: load_v1i3_trunc_v1i1_store:
 ; AVX512-ONLY:       # %bb.0:
 ; AVX512-ONLY-NEXT:    movb (%rdi), %al
+; AVX512-ONLY-NEXT:    andl $1, %eax
+; AVX512-ONLY-NEXT:    kmovw %eax, %k0
+; AVX512-ONLY-NEXT:    kmovw %k0, %eax
 ; AVX512-ONLY-NEXT:    movb %al, (%rsi)
 ; AVX512-ONLY-NEXT:    retq
     %d0 = load <1 x i3>, <1 x i3>* %a0
     %d1 = trunc <1 x i3> %d0 to <1 x i1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v1i4_trunc_v1i1_store(<1 x i4>* %a0,<1 x i1>* %a1) {
 ; AVX512-ALL-LABEL: load_v1i4_trunc_v1i1_store:
 ; AVX512-ALL:       # %bb.0:
-; AVX512-ALL-NEXT:    movb (%rdi), %al
-; AVX512-ALL-NEXT:    movb %al, (%rsi)
+; AVX512-ALL-NEXT:    kmovb (%rdi), %k0
+; AVX512-ALL-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-ALL-NEXT:    retq
 ;
 ; AVX512-ONLY-LABEL: load_v1i4_trunc_v1i1_store:
 ; AVX512-ONLY:       # %bb.0:
 ; AVX512-ONLY-NEXT:    movb (%rdi), %al
+; AVX512-ONLY-NEXT:    andl $1, %eax
+; AVX512-ONLY-NEXT:    kmovw %eax, %k0
+; AVX512-ONLY-NEXT:    kmovw %k0, %eax
 ; AVX512-ONLY-NEXT:    movb %al, (%rsi)
 ; AVX512-ONLY-NEXT:    retq
     %d0 = load <1 x i4>, <1 x i4>* %a0
     %d1 = trunc <1 x i4> %d0 to <1 x i1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v1i8_trunc_v1i1_store(<1 x i8>* %a0,<1 x i1>* %a1) {
 ; AVX512-ALL-LABEL: load_v1i8_trunc_v1i1_store:
 ; AVX512-ALL:       # %bb.0:
-; AVX512-ALL-NEXT:    movb (%rdi), %al
-; AVX512-ALL-NEXT:    movb %al, (%rsi)
+; AVX512-ALL-NEXT:    kmovb (%rdi), %k0
+; AVX512-ALL-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-ALL-NEXT:    retq
 ;
 ; AVX512-ONLY-LABEL: load_v1i8_trunc_v1i1_store:
 ; AVX512-ONLY:       # %bb.0:
 ; AVX512-ONLY-NEXT:    movb (%rdi), %al
+; AVX512-ONLY-NEXT:    andl $1, %eax
+; AVX512-ONLY-NEXT:    kmovw %eax, %k0
+; AVX512-ONLY-NEXT:    kmovw %k0, %eax
 ; AVX512-ONLY-NEXT:    movb %al, (%rsi)
 ; AVX512-ONLY-NEXT:    retq
     %d0 = load <1 x i8>, <1 x i8>* %a0
     %d1 = trunc <1 x i8> %d0 to <1 x i1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v1i16_trunc_v1i1_store(<1 x i16>* %a0,<1 x i1>* %a1) {
 ; AVX512-ALL-LABEL: load_v1i16_trunc_v1i1_store:
 ; AVX512-ALL:       # %bb.0:
-; AVX512-ALL-NEXT:    movb (%rdi), %al
-; AVX512-ALL-NEXT:    movb %al, (%rsi)
+; AVX512-ALL-NEXT:    kmovb (%rdi), %k0
+; AVX512-ALL-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-ALL-NEXT:    retq
 ;
 ; AVX512-ONLY-LABEL: load_v1i16_trunc_v1i1_store:
 ; AVX512-ONLY:       # %bb.0:
 ; AVX512-ONLY-NEXT:    movb (%rdi), %al
+; AVX512-ONLY-NEXT:    andl $1, %eax
+; AVX512-ONLY-NEXT:    kmovw %eax, %k0
+; AVX512-ONLY-NEXT:    kmovw %k0, %eax
 ; AVX512-ONLY-NEXT:    movb %al, (%rsi)
 ; AVX512-ONLY-NEXT:    retq
     %d0 = load <1 x i16>, <1 x i16>* %a0
     %d1 = trunc <1 x i16> %d0 to <1 x i1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v1i32_trunc_v1i1_store(<1 x i32>* %a0,<1 x i1>* %a1) {
 ; AVX512-ALL-LABEL: load_v1i32_trunc_v1i1_store:
 ; AVX512-ALL:       # %bb.0:
-; AVX512-ALL-NEXT:    movb (%rdi), %al
-; AVX512-ALL-NEXT:    movb %al, (%rsi)
+; AVX512-ALL-NEXT:    kmovb (%rdi), %k0
+; AVX512-ALL-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-ALL-NEXT:    retq
 ;
 ; AVX512-ONLY-LABEL: load_v1i32_trunc_v1i1_store:
 ; AVX512-ONLY:       # %bb.0:
 ; AVX512-ONLY-NEXT:    movb (%rdi), %al
+; AVX512-ONLY-NEXT:    andl $1, %eax
+; AVX512-ONLY-NEXT:    kmovw %eax, %k0
+; AVX512-ONLY-NEXT:    kmovw %k0, %eax
 ; AVX512-ONLY-NEXT:    movb %al, (%rsi)
 ; AVX512-ONLY-NEXT:    retq
     %d0 = load <1 x i32>, <1 x i32>* %a0
     %d1 = trunc <1 x i32> %d0 to <1 x i1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 define void @load_v1i64_trunc_v1i1_store(<1 x i64>* %a0,<1 x i1>* %a1) {
 ; AVX512-ALL-LABEL: load_v1i64_trunc_v1i1_store:
 ; AVX512-ALL:       # %bb.0:
-; AVX512-ALL-NEXT:    movb (%rdi), %al
-; AVX512-ALL-NEXT:    movb %al, (%rsi)
+; AVX512-ALL-NEXT:    kmovb (%rdi), %k0
+; AVX512-ALL-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512-ALL-NEXT:    kmovb %k0, (%rsi)
 ; AVX512-ALL-NEXT:    retq
 ;
 ; AVX512-ONLY-LABEL: load_v1i64_trunc_v1i1_store:
 ; AVX512-ONLY:       # %bb.0:
 ; AVX512-ONLY-NEXT:    movb (%rdi), %al
+; AVX512-ONLY-NEXT:    andl $1, %eax
+; AVX512-ONLY-NEXT:    kmovw %eax, %k0
+; AVX512-ONLY-NEXT:    kmovw %k0, %eax
 ; AVX512-ONLY-NEXT:    movb %al, (%rsi)
 ; AVX512-ONLY-NEXT:    retq
     %d0 = load <1 x i64>, <1 x i64>* %a0
     %d1 = trunc <1 x i64> %d0 to <1 x i1>
     store <1 x i1> %d1, <1 x i1>* %a1
     ret void
 }
 

llvm/test/CodeGen/X86/avx512-mask-op.ll

; X86-NEXT:    retl
   ret <32 x i16> %ret
 }
 
 define void @test22(<4 x i1> %a, <4 x i1>* %addr) {
 ; KNL-LABEL: test22:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpslld $31, %xmm0, %xmm0
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test22:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    vpslld $31, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovd2m %xmm0, %k0
 ; SKX-NEXT:    kmovb %k0, (%rdi)
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test22:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test22:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    kshiftlb $4, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $4, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test22:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    vpslld $31, %xmm0, %xmm0
 ; X86-NEXT:    vpmovd2m %xmm0, %k0
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovb %k0, (%eax)
 ; X86-NEXT:    retl
   store <4 x i1> %a, <4 x i1>* %addr
   ret void
 }
 
 define void @test23(<2 x i1> %a, <2 x i1>* %addr) {
 ; KNL-LABEL: test23:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test23:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovq2m %xmm0, %k0
 ; SKX-NEXT:    kmovb %k0, (%rdi)
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test23:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $14, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test23:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovq2m %zmm0, %k0
+; AVX512DQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $6, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test23:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; X86-NEXT:    vpmovq2m %xmm0, %k0
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovb %k0, (%eax)
 ; X86-NEXT:    retl
   store <2 x i1> %a, <2 x i1>* %addr
   ret void
 }
 
 define void @store_v1i1(<1 x i1> %c , <1 x i1>* %ptr) {
 ; KNL-LABEL: store_v1i1:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    kmovw %edi, %k0
 ; KNL-NEXT:    knotw %k0, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, (%rsi)
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: store_v1i1:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovd %edi, %k0
 ; SKX-NEXT:    knotw %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k0, %k0
+; SKX-NEXT:    kshiftrb $7, %k0, %k0
 ; SKX-NEXT:    kmovb %k0, (%rsi)
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_v1i1:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    kmovd %edi, %k0
 ; AVX512BW-NEXT:    knotw %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, (%rsi)
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_v1i1:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    kmovw %edi, %k0
 ; AVX512DQ-NEXT:    knotw %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, (%rsi)
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: store_v1i1:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    knotw %k0, %k0
+; X86-NEXT:    kshiftlb $7, %k0, %k0
+; X86-NEXT:    kshiftrb $7, %k0, %k0
 ; X86-NEXT:    kmovb %k0, (%eax)
 ; X86-NEXT:    retl
   %x = xor <1 x i1> %c, <i1 1>
   store <1 x i1> %x, <1 x i1>*  %ptr, align 4
   ret void
 }
 
 define void @store_v2i1(<2 x i1> %c , <2 x i1>* %ptr) {
 ; KNL-LABEL: store_v2i1:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: store_v2i1:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovq2m %xmm0, %k0
 ; SKX-NEXT:    knotw %k0, %k0
+; SKX-NEXT:    kshiftlb $6, %k0, %k0
+; SKX-NEXT:    kshiftrb $6, %k0, %k0
 ; SKX-NEXT:    kmovb %k0, (%rdi)
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_v2i1:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $14, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_v2i1:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovq2m %zmm0, %k0
 ; AVX512DQ-NEXT:    knotw %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $6, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: store_v2i1:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; X86-NEXT:    vpmovq2m %xmm0, %k0
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    knotw %k0, %k0
+; X86-NEXT:    kshiftlb $6, %k0, %k0
+; X86-NEXT:    kshiftrb $6, %k0, %k0
 ; X86-NEXT:    kmovb %k0, (%eax)
 ; X86-NEXT:    retl
   %x = xor <2 x i1> %c, <i1 1, i1 1>
   store <2 x i1> %x, <2 x i1>*  %ptr, align 4
   ret void
 }
 
 define void @store_v4i1(<4 x i1> %c , <4 x i1>* %ptr) {
 ; KNL-LABEL: store_v4i1:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    vpslld $31, %xmm0, %xmm0
 ; KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, (%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: store_v4i1:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    vpslld $31, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovd2m %xmm0, %k0
 ; SKX-NEXT:    knotw %k0, %k0
+; SKX-NEXT:    kshiftlb $4, %k0, %k0
+; SKX-NEXT:    kshiftrb $4, %k0, %k0
 ; SKX-NEXT:    kmovb %k0, (%rdi)
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_v4i1:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, (%rdi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_v4i1:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
 ; AVX512DQ-NEXT:    knotw %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $4, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $4, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: store_v4i1:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    vpslld $31, %xmm0, %xmm0
 ; X86-NEXT:    vpmovd2m %xmm0, %k0
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    knotw %k0, %k0
+; X86-NEXT:    kshiftlb $4, %k0, %k0
+; X86-NEXT:    kshiftrb $4, %k0, %k0
 ; X86-NEXT:    kmovb %k0, (%eax)
 ; X86-NEXT:    retl
   %x = xor <4 x i1> %c, <i1 1, i1 1, i1 1, i1 1>
   store <4 x i1> %x, <4 x i1>*  %ptr, align 4
   ret void
 }
 
 define void @store_v8i1(<8 x i1> %c , <8 x i1>* %ptr) {

 }
 
 define i1 @test_v1i1_add(i1 %x, i1 %y) {
 ; KNL-LABEL: test_v1i1_add:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    kmovw %edi, %k0
 ; KNL-NEXT:    kmovw %esi, %k1
 ; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_v1i1_add:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    andl $1, %edi
 ; SKX-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    andl $1, %esi
 ; SKX-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k0
 ; SKX-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k1
 ; SKX-NEXT:    kxorw %k1, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k0, %k0
+; SKX-NEXT:    kshiftrb $7, %k0, %k0
 ; SKX-NEXT:    kmovb %k0, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_v1i1_add:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    kmovd %edi, %k0
 ; AVX512BW-NEXT:    kmovd %esi, %k1
 ; AVX512BW-NEXT:    kxorw %k1, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
 ; AVX512BW-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_v1i1_add:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    andl $1, %edi
 ; AVX512DQ-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    andl $1, %esi
 ; AVX512DQ-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k0
 ; AVX512DQ-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k1
 ; AVX512DQ-NEXT:    kxorw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_v1i1_add:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movb %al, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movb %al, {{[0-9]+}}(%esp)
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k0
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    kxorw %k1, %k0, %k0
+; X86-NEXT:    kshiftlb $7, %k0, %k0
+; X86-NEXT:    kshiftrb $7, %k0, %k0
 ; X86-NEXT:    kmovb %k0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    popl %ecx
 ; X86-NEXT:    retl
   %m0 = bitcast i1 %x to <1 x i1>
   %m1 = bitcast i1 %y to <1 x i1>
   %m2 = add <1 x i1> %m0,  %m1
   %ret = bitcast <1 x i1> %m2 to i1
   ret i1 %ret
 }
 
 define i1 @test_v1i1_sub(i1 %x, i1 %y) {
 ; KNL-LABEL: test_v1i1_sub:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    kmovw %edi, %k0
 ; KNL-NEXT:    kmovw %esi, %k1
 ; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_v1i1_sub:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    andl $1, %edi
 ; SKX-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    andl $1, %esi
 ; SKX-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k0
 ; SKX-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k1
 ; SKX-NEXT:    kxorw %k1, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k0, %k0
+; SKX-NEXT:    kshiftrb $7, %k0, %k0
 ; SKX-NEXT:    kmovb %k0, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_v1i1_sub:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    kmovd %edi, %k0
 ; AVX512BW-NEXT:    kmovd %esi, %k1
 ; AVX512BW-NEXT:    kxorw %k1, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
 ; AVX512BW-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_v1i1_sub:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    andl $1, %edi
 ; AVX512DQ-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    andl $1, %esi
 ; AVX512DQ-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k0
 ; AVX512DQ-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k1
 ; AVX512DQ-NEXT:    kxorw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_v1i1_sub:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movb %al, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movb %al, {{[0-9]+}}(%esp)
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k0
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    kxorw %k1, %k0, %k0
+; X86-NEXT:    kshiftlb $7, %k0, %k0
+; X86-NEXT:    kshiftrb $7, %k0, %k0
 ; X86-NEXT:    kmovb %k0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    popl %ecx
 ; X86-NEXT:    retl
   %m0 = bitcast i1 %x to <1 x i1>
   %m1 = bitcast i1 %y to <1 x i1>
   %m2 = sub <1 x i1> %m0,  %m1
   %ret = bitcast <1 x i1> %m2 to i1
   ret i1 %ret
 }
 
 define i1 @test_v1i1_mul(i1 %x, i1 %y) {
 ; KNL-LABEL: test_v1i1_mul:
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    kmovw %edi, %k0
 ; KNL-NEXT:    kmovw %esi, %k1
 ; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_v1i1_mul:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    andl $1, %edi
 ; SKX-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    andl $1, %esi
 ; SKX-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k0
 ; SKX-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k1
 ; SKX-NEXT:    kandw %k1, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k0, %k0
+; SKX-NEXT:    kshiftrb $7, %k0, %k0
 ; SKX-NEXT:    kmovb %k0, -{{[0-9]+}}(%rsp)
 ; SKX-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_v1i1_mul:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    kmovd %edi, %k0
 ; AVX512BW-NEXT:    kmovd %esi, %k1
 ; AVX512BW-NEXT:    kandw %k1, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
 ; AVX512BW-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_v1i1_mul:
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    andl $1, %edi
 ; AVX512DQ-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    andl $1, %esi
 ; AVX512DQ-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k0
 ; AVX512DQ-NEXT:    kmovb -{{[0-9]+}}(%rsp), %k1
 ; AVX512DQ-NEXT:    kandw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $7, %k0, %k0
 ; AVX512DQ-NEXT:    kmovb %k0, -{{[0-9]+}}(%rsp)
 ; AVX512DQ-NEXT:    movb -{{[0-9]+}}(%rsp), %al
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: test_v1i1_mul:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movb %al, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    andb $1, %al
 ; X86-NEXT:    movb %al, {{[0-9]+}}(%esp)
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k0
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    kandw %k1, %k0, %k0
+; X86-NEXT:    kshiftlb $7, %k0, %k0
+; X86-NEXT:    kshiftrb $7, %k0, %k0
 ; X86-NEXT:    kmovb %k0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    popl %ecx
 ; X86-NEXT:    retl
   %m0 = bitcast i1 %x to <1 x i1>
   %m1 = bitcast i1 %y to <1 x i1>
   %m2 = mul <1 x i1> %m0,  %m1
   %ret = bitcast <1 x i1> %m2 to i1

llvm/test/CodeGen/X86/avx512-select.ll

 ; X86-AVX512F-NEXT:    kmovw %esi, %k0
 ; X86-AVX512F-NEXT:    movzbl (%ecx), %ecx
 ; X86-AVX512F-NEXT:    kmovw %ecx, %k1
 ; X86-AVX512F-NEXT:    movzbl (%eax), %eax
 ; X86-AVX512F-NEXT:    kmovw %eax, %k2
 ; X86-AVX512F-NEXT:    kandnw %k1, %k2, %k1
 ; X86-AVX512F-NEXT:    kandw %k2, %k0, %k0
 ; X86-AVX512F-NEXT:    korw %k1, %k0, %k0
+; X86-AVX512F-NEXT:    kshiftlw $15, %k0, %k0
+; X86-AVX512F-NEXT:    kshiftrw $15, %k0, %k0
 ; X86-AVX512F-NEXT:    kmovw %k0, %eax
 ; X86-AVX512F-NEXT:    movb %al, (%edx)
 ; X86-AVX512F-NEXT:    popl %esi
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: vselect_v1i1:
 ; X64-AVX512F:       # %bb.0:
 ; X64-AVX512F-NEXT:    movzbl (%rsi), %eax
 ; X64-AVX512F-NEXT:    kmovw %eax, %k0
 ; X64-AVX512F-NEXT:    movzbl (%rdx), %eax
 ; X64-AVX512F-NEXT:    kmovw %eax, %k1
 ; X64-AVX512F-NEXT:    movzbl (%rdi), %eax
 ; X64-AVX512F-NEXT:    kmovw %eax, %k2
 ; X64-AVX512F-NEXT:    kandnw %k1, %k2, %k1
 ; X64-AVX512F-NEXT:    kandw %k2, %k0, %k0
 ; X64-AVX512F-NEXT:    korw %k1, %k0, %k0
+; X64-AVX512F-NEXT:    kshiftlw $15, %k0, %k0
+; X64-AVX512F-NEXT:    kshiftrw $15, %k0, %k0
 ; X64-AVX512F-NEXT:    kmovw %k0, %eax
 ; X64-AVX512F-NEXT:    movb %al, (%rsi)
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X86-AVX512BW-LABEL: vselect_v1i1:
 ; X86-AVX512BW:       # %bb.0:
 ; X86-AVX512BW-NEXT:    pushl %esi
 ; X86-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX512BW-NEXT:    movzbl (%edx), %esi
 ; X86-AVX512BW-NEXT:    kmovd %esi, %k0
 ; X86-AVX512BW-NEXT:    movzbl (%ecx), %ecx
 ; X86-AVX512BW-NEXT:    kmovd %ecx, %k1
 ; X86-AVX512BW-NEXT:    movzbl (%eax), %eax
 ; X86-AVX512BW-NEXT:    kmovd %eax, %k2
 ; X86-AVX512BW-NEXT:    kandnw %k1, %k2, %k1
 ; X86-AVX512BW-NEXT:    kandw %k2, %k0, %k0
 ; X86-AVX512BW-NEXT:    korw %k1, %k0, %k0
+; X86-AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; X86-AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; X86-AVX512BW-NEXT:    kmovd %k0, %eax
 ; X86-AVX512BW-NEXT:    movb %al, (%edx)
 ; X86-AVX512BW-NEXT:    popl %esi
 ; X86-AVX512BW-NEXT:    retl
 ;
 ; X64-AVX512BW-LABEL: vselect_v1i1:
 ; X64-AVX512BW:       # %bb.0:
 ; X64-AVX512BW-NEXT:    movzbl (%rsi), %eax
 ; X64-AVX512BW-NEXT:    kmovd %eax, %k0
 ; X64-AVX512BW-NEXT:    movzbl (%rdx), %eax
 ; X64-AVX512BW-NEXT:    kmovd %eax, %k1
 ; X64-AVX512BW-NEXT:    movzbl (%rdi), %eax
 ; X64-AVX512BW-NEXT:    kmovd %eax, %k2
 ; X64-AVX512BW-NEXT:    kandnw %k1, %k2, %k1
 ; X64-AVX512BW-NEXT:    kandw %k2, %k0, %k0
 ; X64-AVX512BW-NEXT:    korw %k1, %k0, %k0
+; X64-AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; X64-AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; X64-AVX512BW-NEXT:    kmovd %k0, %eax
 ; X64-AVX512BW-NEXT:    movb %al, (%rsi)
 ; X64-AVX512BW-NEXT:    retq
   %a = load <1 x i1>, <1 x i1>* %x
   %b = load <1 x i1>, <1 x i1>* %y
   %b2 = load <1 x i1>, <1 x i1>* %w
   %c = select <1 x i1> %b2, <1 x i1> %a, <1 x i1> %b
   store <1 x i1> %c, <1 x i1>* %x

 ; X86-AVX512F-NEXT:    movzbl (%ecx), %ecx
 ; X86-AVX512F-NEXT:    kmovw %ecx, %k1
 ; X86-AVX512F-NEXT:    kxorw %k1, %k0, %k0
 ; X86-AVX512F-NEXT:    jmp .LBB18_3
 ; X86-AVX512F-NEXT:  .LBB18_1:
 ; X86-AVX512F-NEXT:    movzbl (%eax), %ecx
 ; X86-AVX512F-NEXT:    kmovw %ecx, %k0
 ; X86-AVX512F-NEXT:  .LBB18_3:
+; X86-AVX512F-NEXT:    kshiftlw $15, %k0, %k0
+; X86-AVX512F-NEXT:    kshiftrw $15, %k0, %k0
 ; X86-AVX512F-NEXT:    kmovw %k0, %ecx
 ; X86-AVX512F-NEXT:    movb %cl, (%eax)
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X64-AVX512F-LABEL: select_v1i1:
 ; X64-AVX512F:       # %bb.0:
 ; X64-AVX512F-NEXT:    testb $1, %cl
 ; X64-AVX512F-NEXT:    jne .LBB18_1
 ; X64-AVX512F-NEXT:  # %bb.2:
 ; X64-AVX512F-NEXT:    movzbl (%rdx), %eax
 ; X64-AVX512F-NEXT:    kmovw %eax, %k0
 ; X64-AVX512F-NEXT:    movzbl (%rdi), %eax
 ; X64-AVX512F-NEXT:    kmovw %eax, %k1
 ; X64-AVX512F-NEXT:    kxorw %k1, %k0, %k0
 ; X64-AVX512F-NEXT:    jmp .LBB18_3
 ; X64-AVX512F-NEXT:  .LBB18_1:
 ; X64-AVX512F-NEXT:    movzbl (%rsi), %eax
 ; X64-AVX512F-NEXT:    kmovw %eax, %k0
 ; X64-AVX512F-NEXT:  .LBB18_3:
+; X64-AVX512F-NEXT:    kshiftlw $15, %k0, %k0
+; X64-AVX512F-NEXT:    kshiftrw $15, %k0, %k0
 ; X64-AVX512F-NEXT:    kmovw %k0, %eax
 ; X64-AVX512F-NEXT:    movb %al, (%rsi)
 ; X64-AVX512F-NEXT:    retq
 ;
 ; X86-AVX512BW-LABEL: select_v1i1:
 ; X86-AVX512BW:       # %bb.0:
 ; X86-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512BW-NEXT:    testb $1, {{[0-9]+}}(%esp)
 ; X86-AVX512BW-NEXT:    jne .LBB18_1
 ; X86-AVX512BW-NEXT:  # %bb.2:
 ; X86-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX512BW-NEXT:    movzbl (%edx), %edx
 ; X86-AVX512BW-NEXT:    kmovd %edx, %k0
 ; X86-AVX512BW-NEXT:    movzbl (%ecx), %ecx
 ; X86-AVX512BW-NEXT:    kmovd %ecx, %k1
 ; X86-AVX512BW-NEXT:    kxorw %k1, %k0, %k0
 ; X86-AVX512BW-NEXT:    jmp .LBB18_3
 ; X86-AVX512BW-NEXT:  .LBB18_1:
 ; X86-AVX512BW-NEXT:    movzbl (%eax), %ecx
 ; X86-AVX512BW-NEXT:    kmovd %ecx, %k0
 ; X86-AVX512BW-NEXT:  .LBB18_3:
+; X86-AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; X86-AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; X86-AVX512BW-NEXT:    kmovd %k0, %ecx
 ; X86-AVX512BW-NEXT:    movb %cl, (%eax)
 ; X86-AVX512BW-NEXT:    retl
 ;
 ; X64-AVX512BW-LABEL: select_v1i1:
 ; X64-AVX512BW:       # %bb.0:
 ; X64-AVX512BW-NEXT:    testb $1, %cl
 ; X64-AVX512BW-NEXT:    jne .LBB18_1
 ; X64-AVX512BW-NEXT:  # %bb.2:
 ; X64-AVX512BW-NEXT:    movzbl (%rdx), %eax
 ; X64-AVX512BW-NEXT:    kmovd %eax, %k0
 ; X64-AVX512BW-NEXT:    movzbl (%rdi), %eax
 ; X64-AVX512BW-NEXT:    kmovd %eax, %k1
 ; X64-AVX512BW-NEXT:    kxorw %k1, %k0, %k0
 ; X64-AVX512BW-NEXT:    jmp .LBB18_3
 ; X64-AVX512BW-NEXT:  .LBB18_1:
 ; X64-AVX512BW-NEXT:    movzbl (%rsi), %eax
 ; X64-AVX512BW-NEXT:    kmovd %eax, %k0
 ; X64-AVX512BW-NEXT:  .LBB18_3:
+; X64-AVX512BW-NEXT:    kshiftlw $15, %k0, %k0
+; X64-AVX512BW-NEXT:    kshiftrw $15, %k0, %k0
 ; X64-AVX512BW-NEXT:    kmovd %k0, %eax
 ; X64-AVX512BW-NEXT:    movb %al, (%rsi)
 ; X64-AVX512BW-NEXT:    retq
   %a = load <1 x i1>, <1 x i1>* %x
   %b = load <1 x i1>, <1 x i1>* %y
   %b2 = load <1 x i1>, <1 x i1>* %w
   %b3 = xor <1 x i1> %b, %b2
   %c = select i1 %z, <1 x i1> %a, <1 x i1> %b3

llvm/test/CodeGen/X86/stack-folding-avx512vp2intersect.ll

 ; CHECK-LABEL: stack_fold_vp2intersectq_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vmovaps (%rdi), %ymm0
 ; CHECK-NEXT:    vp2intersectq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %k0 # 32-byte Folded Reload
-; CHECK-NEXT:    kmovw %k1, %eax
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    kshiftlw $12, %k0, %k2
+; CHECK-NEXT:    kshiftrw $12, %k2, %k2
+; CHECK-NEXT:    kmovw %k2, %eax
+; CHECK-NEXT:    movb %al, (%rsi)
+; CHECK-NEXT:    kshiftlw $12, %k1, %k0
+; CHECK-NEXT:    kshiftrw $12, %k0, %k0
+; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    movb %al, (%rdx)
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <4 x i64>, <4 x i64>* %a
   %3 = tail call { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.q.256(<4 x i64> %2, <4 x i64> %b)
   %4 = extractvalue { <4 x i1>, <4 x i1> } %3, 0
   store <4 x i1> %4, <4 x i1>* %m0
   %5 = extractvalue { <4 x i1>, <4 x i1> } %3, 1
   store <4 x i1> %5, <4 x i1>* %m1
   ret void
 }
 declare { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.q.256(<4 x i64>, <4 x i64>)
 
 define void @stack_fold_vp2intersectd_128(<4 x i32>* %a, <4 x i32> %b, <4 x i1>* nocapture %m0, <4 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectd_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vmovaps (%rdi), %xmm0
 ; CHECK-NEXT:    vp2intersectd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
-; CHECK-NEXT:    kmovw %k1, %eax
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    kshiftlw $12, %k0, %k2
+; CHECK-NEXT:    kshiftrw $12, %k2, %k2
+; CHECK-NEXT:    kmovw %k2, %eax
+; CHECK-NEXT:    movb %al, (%rsi)
+; CHECK-NEXT:    kshiftlw $12, %k1, %k0
+; CHECK-NEXT:    kshiftrw $12, %k0, %k0
+; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    movb %al, (%rdx)
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <4 x i32>, <4 x i32>* %a
   %3 = tail call { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.d.128(<4 x i32> %2, <4 x i32> %b)
   %4 = extractvalue { <4 x i1>, <4 x i1> } %3, 0
   store <4 x i1> %4, <4 x i1>* %m0
   %5 = extractvalue { <4 x i1>, <4 x i1> } %3, 1
   store <4 x i1> %5, <4 x i1>* %m1
   ret void
 }
 declare { <4 x i1>, <4 x i1> } @llvm.x86.avx512.vp2intersect.d.128(<4 x i32>, <4 x i32>)
 
 define void @stack_fold_vp2intersectq_128(<2 x i64>* %a, <2 x i64> %b, <2 x i1>* nocapture %m0, <2 x i1>* nocapture %m1) {
 ; CHECK-LABEL: stack_fold_vp2intersectq_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vmovaps (%rdi), %xmm0
 ; CHECK-NEXT:    vp2intersectq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %k0 # 16-byte Folded Reload
-; CHECK-NEXT:    kmovw %k1, %eax
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    kshiftlw $14, %k0, %k2
+; CHECK-NEXT:    kshiftrw $14, %k2, %k2
+; CHECK-NEXT:    kmovw %k2, %eax
+; CHECK-NEXT:    movb %al, (%rsi)
+; CHECK-NEXT:    kshiftlw $14, %k1, %k0
+; CHECK-NEXT:    kshiftrw $14, %k0, %k0
+; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    movb %al, (%rdx)
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = load <2 x i64>, <2 x i64>* %a
   %3 = tail call { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64> %2, <2 x i64> %b)
   %4 = extractvalue { <2 x i1>, <2 x i1> } %3, 0
   store <2 x i1> %4, <2 x i1>* %m0
   %5 = extractvalue { <2 x i1>, <2 x i1> } %3, 1
   store <2 x i1> %5, <2 x i1>* %m1
   ret void
 }
 declare { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64>, <2 x i64>)

llvm/test/CodeGen/X86/vec_saddo.ll

 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm2
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k1
 ; AVX512-NEXT:    vptestmd %xmm2, %xmm2, %k2
 ; AVX512-NEXT:    kxorw %k1, %k0, %k0
 ; AVX512-NEXT:    kxorw %k2, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<4 x i1>, <4 x i1>} @llvm.sadd.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
   %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
   %res = sext <4 x i1> %obit to <4 x i32>
   store <4 x i1> %val, <4 x i1>* %p2

llvm/test/CodeGen/X86/vec_smulo.ll

 ; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovmskps %xmm3, %eax
 ; AVX2-NEXT:    movb %al, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: smulo_v4i1:
 ; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %rbp
 ; AVX512-NEXT:    pushq %rbx
 ; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
 ; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k0
 ; AVX512-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512-NEXT:    kmovd %k1, %r9d
-; AVX512-NEXT:    andb $1, %r9b
-; AVX512-NEXT:    negb %r9b
+; AVX512-NEXT:    kmovd %k1, %r10d
+; AVX512-NEXT:    andb $1, %r10b
+; AVX512-NEXT:    negb %r10b
 ; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
 ; AVX512-NEXT:    kshiftrw $3, %k1, %k2
-; AVX512-NEXT:    kmovd %k2, %r10d
-; AVX512-NEXT:    andb $1, %r10b
-; AVX512-NEXT:    negb %r10b
+; AVX512-NEXT:    kmovd %k2, %r9d
+; AVX512-NEXT:    andb $1, %r9b
+; AVX512-NEXT:    negb %r9b
 ; AVX512-NEXT:    kshiftrw $2, %k1, %k2
-; AVX512-NEXT:    kmovd %k2, %r11d
-; AVX512-NEXT:    andb $1, %r11b
-; AVX512-NEXT:    negb %r11b
+; AVX512-NEXT:    kmovd %k2, %ebp
+; AVX512-NEXT:    andb $1, %bpl
+; AVX512-NEXT:    negb %bpl
 ; AVX512-NEXT:    kshiftrw $2, %k0, %k2
-; AVX512-NEXT:    kmovd %k2, %ebx
-; AVX512-NEXT:    andb $1, %bl
-; AVX512-NEXT:    negb %bl
+; AVX512-NEXT:    kmovd %k2, %edx
+; AVX512-NEXT:    andb $1, %dl
+; AVX512-NEXT:    negb %dl
 ; AVX512-NEXT:    kshiftrw $1, %k0, %k2
 ; AVX512-NEXT:    kmovd %k2, %esi
 ; AVX512-NEXT:    andb $1, %sil
 ; AVX512-NEXT:    negb %sil
 ; AVX512-NEXT:    kshiftrw $1, %k1, %k2
-; AVX512-NEXT:    kmovd %k2, %edx
-; AVX512-NEXT:    andb $1, %dl
-; AVX512-NEXT:    negb %dl
+; AVX512-NEXT:    kmovd %k2, %ecx
+; AVX512-NEXT:    andb $1, %cl
+; AVX512-NEXT:    negb %cl
 ; AVX512-NEXT:    kmovd %k1, %eax
 ; AVX512-NEXT:    andb $1, %al
 ; AVX512-NEXT:    negb %al
-; AVX512-NEXT:    kmovd %k0, %ecx
-; AVX512-NEXT:    andb $1, %cl
-; AVX512-NEXT:    negb %cl
+; AVX512-NEXT:    kmovd %k0, %ebx
+; AVX512-NEXT:    andb $1, %bl
+; AVX512-NEXT:    negb %bl
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    imulb %cl
-; AVX512-NEXT:    movl %eax, %r8d
+; AVX512-NEXT:    imulb %bl
+; AVX512-NEXT:    movl %eax, %r11d
 ; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    movl %r8d, %ecx
-; AVX512-NEXT:    andb $1, %cl
-; AVX512-NEXT:    negb %cl
-; AVX512-NEXT:    cmpb %r8b, %cl
-; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    orb %al, %cl
+; AVX512-NEXT:    movl %r11d, %ebx
+; AVX512-NEXT:    andb $1, %bl
+; AVX512-NEXT:    negb %bl
+; AVX512-NEXT:    cmpb %r11b, %bl
+; AVX512-NEXT:    setne %bl
+; AVX512-NEXT:    orb %al, %bl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    movw $-3, %ax
 ; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    kandw %k0, %k1, %k1
-; AVX512-NEXT:    movl %edx, %eax
+; AVX512-NEXT:    movl %ecx, %eax
 ; AVX512-NEXT:    imulb %sil
-; AVX512-NEXT:    movl %eax, %edx
+; AVX512-NEXT:    movl %eax, %r8d
 ; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    movl %edx, %ecx
+; AVX512-NEXT:    movl %r8d, %ecx
 ; AVX512-NEXT:    andb $1, %cl
 ; AVX512-NEXT:    negb %cl
-; AVX512-NEXT:    cmpb %dl, %cl
+; AVX512-NEXT:    cmpb %r8b, %cl
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k2
 ; AVX512-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512-NEXT:    kshiftrw $14, %k2, %k2
 ; AVX512-NEXT:    korw %k2, %k1, %k2
 ; AVX512-NEXT:    movw $-5, %ax
 ; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kandw %k1, %k2, %k2
-; AVX512-NEXT:    movl %r11d, %eax
-; AVX512-NEXT:    imulb %bl
+; AVX512-NEXT:    movl %ebp, %eax
+; AVX512-NEXT:    imulb %dl
 ; AVX512-NEXT:    movl %eax, %esi
 ; AVX512-NEXT:    seto %al
 ; AVX512-NEXT:    movl %esi, %ecx
 ; AVX512-NEXT:    andb $1, %cl
 ; AVX512-NEXT:    negb %cl
 ; AVX512-NEXT:    cmpb %sil, %cl
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k3
 ; AVX512-NEXT:    kshiftlw $2, %k3, %k3
 ; AVX512-NEXT:    korw %k3, %k2, %k2
 ; AVX512-NEXT:    kshiftlw $13, %k2, %k2
 ; AVX512-NEXT:    kshiftrw $13, %k2, %k2
-; AVX512-NEXT:    movl %r10d, %eax
-; AVX512-NEXT:    imulb %r9b
+; AVX512-NEXT:    movl %r9d, %eax
+; AVX512-NEXT:    imulb %r10b
 ; AVX512-NEXT:    # kill: def $al killed $al def $eax
 ; AVX512-NEXT:    seto %cl
-; AVX512-NEXT:    movl %eax, %ebx
-; AVX512-NEXT:    andb $1, %bl
-; AVX512-NEXT:    negb %bl
-; AVX512-NEXT:    cmpb %al, %bl
-; AVX512-NEXT:    setne %bl
-; AVX512-NEXT:    orb %cl, %bl
+; AVX512-NEXT:    movl %eax, %edx
+; AVX512-NEXT:    andb $1, %dl
+; AVX512-NEXT:    negb %dl
+; AVX512-NEXT:    cmpb %al, %dl
+; AVX512-NEXT:    setne %dl
+; AVX512-NEXT:    orb %cl, %dl
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    kmovd %ecx, %k3
 ; AVX512-NEXT:    kshiftlw $3, %k3, %k3
 ; AVX512-NEXT:    korw %k3, %k2, %k2
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
-; AVX512-NEXT:    kmovd %r8d, %k2
+; AVX512-NEXT:    andl $1, %r11d
+; AVX512-NEXT:    kmovw %r11d, %k2
 ; AVX512-NEXT:    kandw %k0, %k2, %k0
-; AVX512-NEXT:    kmovd %edx, %k2
+; AVX512-NEXT:    kmovd %r8d, %k2
 ; AVX512-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512-NEXT:    kshiftrw $14, %k2, %k2
 ; AVX512-NEXT:    korw %k2, %k0, %k0
 ; AVX512-NEXT:    kandw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %esi, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $13, %k1, %k1
 ; AVX512-NEXT:    korw %k1, %k0, %k0
 ; AVX512-NEXT:    movw $-9, %cx
 ; AVX512-NEXT:    kmovd %ecx, %k1
 ; AVX512-NEXT:    kandw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $12, %k1, %k1
 ; AVX512-NEXT:    korw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    popq %rbp
 ; AVX512-NEXT:    retq
   %t = call {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
   %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
   %res = sext <4 x i1> %obit to <4 x i32>
   store <4 x i1> %val, <4 x i1>* %p2
   ret <4 x i32> %res
 }

llvm/test/CodeGen/X86/vec_ssubo.ll

 ; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k0
 ; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
 ; AVX512-NEXT:    vptestnmd %xmm1, %xmm1, %k2 {%k1}
 ; AVX512-NEXT:    kxorw %k0, %k1, %k0
 ; AVX512-NEXT:    kxorw %k2, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<4 x i1>, <4 x i1>} @llvm.ssub.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
   %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
   %res = sext <4 x i1> %obit to <4 x i32>
   store <4 x i1> %val, <4 x i1>* %p2

llvm/test/CodeGen/X86/vec_uaddo.ll

 ; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
 ; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    kandnw %k0, %k1, %k2
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
-; AVX512-NEXT:    kmovd %k1, %eax
+; AVX512-NEXT:    kshiftlw $12, %k1, %k0
+; AVX512-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<4 x i1>, <4 x i1>} @llvm.uadd.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
   %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
   %res = sext <4 x i1> %obit to <4 x i32>
   store <4 x i1> %val, <4 x i1>* %p2
   ret <4 x i32> %res

llvm/test/CodeGen/X86/vec_umulo.ll

 ; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
 ; AVX2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX2-NEXT:    movb %al, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: umulo_v4i1:
 ; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %rbp
 ; AVX512-NEXT:    pushq %rbx
 ; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512-NEXT:    kmovd %k1, %r9d
-; AVX512-NEXT:    andb $1, %r9b
+; AVX512-NEXT:    kmovd %k1, %r8d
+; AVX512-NEXT:    andb $1, %r8b
 ; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
 ; AVX512-NEXT:    kshiftrw $3, %k1, %k2
+; AVX512-NEXT:    kmovd %k2, %r9d
+; AVX512-NEXT:    andb $1, %r9b
+; AVX512-NEXT:    kshiftrw $2, %k0, %k2
 ; AVX512-NEXT:    kmovd %k2, %r10d
 ; AVX512-NEXT:    andb $1, %r10b
-; AVX512-NEXT:    kshiftrw $2, %k0, %k2
+; AVX512-NEXT:    kshiftrw $2, %k1, %k2
 ; AVX512-NEXT:    kmovd %k2, %r11d
 ; AVX512-NEXT:    andb $1, %r11b
-; AVX512-NEXT:    kshiftrw $2, %k1, %k2
-; AVX512-NEXT:    kmovd %k2, %ebx
-; AVX512-NEXT:    andb $1, %bl
 ; AVX512-NEXT:    kshiftrw $1, %k0, %k2
-; AVX512-NEXT:    kmovd %k2, %edx
-; AVX512-NEXT:    andb $1, %dl
+; AVX512-NEXT:    kmovd %k2, %ecx
+; AVX512-NEXT:    andb $1, %cl
 ; AVX512-NEXT:    kshiftrw $1, %k1, %k2
 ; AVX512-NEXT:    kmovd %k2, %esi
 ; AVX512-NEXT:    andb $1, %sil
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    andb $1, %al
-; AVX512-NEXT:    kmovd %k1, %ecx
-; AVX512-NEXT:    andb $1, %cl
+; AVX512-NEXT:    kmovd %k1, %edx
+; AVX512-NEXT:    andb $1, %dl
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    mulb %cl
-; AVX512-NEXT:    movl %eax, %r8d
+; AVX512-NEXT:    mulb %dl
+; AVX512-NEXT:    movl %eax, %edx
 ; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    testb $-2, %r8b
-; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    orb %al, %cl
+; AVX512-NEXT:    testb $-2, %dl
+; AVX512-NEXT:    setne %bl
+; AVX512-NEXT:    orb %al, %bl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    movw $-3, %ax
 ; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    kandw %k0, %k1, %k1
-; AVX512-NEXT:    movl %edx, %eax
+; AVX512-NEXT:    movl %ecx, %eax
 ; AVX512-NEXT:    mulb %sil
-; AVX512-NEXT:    movl %eax, %edx
+; AVX512-NEXT:    movl %eax, %ebp
 ; AVX512-NEXT:    seto %al
-; AVX512-NEXT:    testb $-2, %dl
-; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    orb %al, %cl
+; AVX512-NEXT:    testb $-2, %bpl
+; AVX512-NEXT:    setne %bl
+; AVX512-NEXT:    orb %al, %bl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k2
 ; AVX512-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512-NEXT:    kshiftrw $14, %k2, %k2
 ; AVX512-NEXT:    korw %k2, %k1, %k2
 ; AVX512-NEXT:    movw $-5, %ax
 ; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kandw %k1, %k2, %k2
-; AVX512-NEXT:    movl %r11d, %eax
-; AVX512-NEXT:    mulb %bl
+; AVX512-NEXT:    movl %r10d, %eax
+; AVX512-NEXT:    mulb %r11b
 ; AVX512-NEXT:    movl %eax, %esi
 ; AVX512-NEXT:    seto %al
 ; AVX512-NEXT:    testb $-2, %sil
-; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    orb %al, %cl
+; AVX512-NEXT:    setne %bl
+; AVX512-NEXT:    orb %al, %bl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k3
 ; AVX512-NEXT:    kshiftlw $2, %k3, %k3
 ; AVX512-NEXT:    korw %k3, %k2, %k2
 ; AVX512-NEXT:    kshiftlw $13, %k2, %k2
 ; AVX512-NEXT:    kshiftrw $13, %k2, %k2
-; AVX512-NEXT:    movl %r9d, %eax
-; AVX512-NEXT:    mulb %r10b
+; AVX512-NEXT:    movl %r8d, %eax
+; AVX512-NEXT:    mulb %r9b
 ; AVX512-NEXT:    # kill: def $al killed $al def $eax
-; AVX512-NEXT:    seto %cl
+; AVX512-NEXT:    seto %bl
 ; AVX512-NEXT:    testb $-2, %al
-; AVX512-NEXT:    setne %bl
-; AVX512-NEXT:    orb %cl, %bl
+; AVX512-NEXT:    setne %cl
+; AVX512-NEXT:    orb %bl, %cl
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    kmovd %ecx, %k3
 ; AVX512-NEXT:    kshiftlw $3, %k3, %k3
 ; AVX512-NEXT:    korw %k3, %k2, %k2
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
-; AVX512-NEXT:    kmovd %r8d, %k2
+; AVX512-NEXT:    andl $1, %edx
+; AVX512-NEXT:    kmovw %edx, %k2
 ; AVX512-NEXT:    kandw %k0, %k2, %k0
-; AVX512-NEXT:    kmovd %edx, %k2
+; AVX512-NEXT:    kmovd %ebp, %k2
 ; AVX512-NEXT:    kshiftlw $15, %k2, %k2
 ; AVX512-NEXT:    kshiftrw $14, %k2, %k2
 ; AVX512-NEXT:    korw %k2, %k0, %k0
 ; AVX512-NEXT:    kandw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %esi, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $13, %k1, %k1
 ; AVX512-NEXT:    korw %k1, %k0, %k0
 ; AVX512-NEXT:    movw $-9, %cx
 ; AVX512-NEXT:    kmovd %ecx, %k1
 ; AVX512-NEXT:    kandw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $12, %k1, %k1
 ; AVX512-NEXT:    korw %k1, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    popq %rbp
 ; AVX512-NEXT:    retq
   %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
   %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
   %res = sext <4 x i1> %obit to <4 x i32>
   store <4 x i1> %val, <4 x i1>* %p2
   ret <4 x i32> %res
 }

llvm/test/CodeGen/X86/vec_usubo.ll

 ; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
 ; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k1
 ; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k2 {%k1}
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
-; AVX512-NEXT:    kmovd %k1, %eax
+; AVX512-NEXT:    kshiftlw $12, %k1, %k0
+; AVX512-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    retq
   %t = call {<4 x i1>, <4 x i1>} @llvm.usub.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
   %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
   %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
   %res = sext <4 x i1> %obit to <4 x i32>
   store <4 x i1> %val, <4 x i1>* %p2
   ret <4 x i32> %res