[X86] Teach combineBitcastvxi1 to prefer movmsk on avx512 in more cases

[craig.topper]

Summary

If the input to the bitcast is a sign bit test, it makes sense to
directly use vpmovmskb or vmovmskps/pd. This removes the need to
copy the sign bits to a k-register and then to a GPR.

Fixes PR46200.

[RKSimon]

llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll

6340

I don't recognise what (c/c++) intrinsic this would have come from - is this likely to be a problem? Same for the other *-intrinsics-upgrade.ll cases

[craig.topper]

llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll

6340

I believe this is _mm256_movepi8_mask which is the intrinsic for vpmovb2m. The intrinsic itself is defined to return unsigned short which introduced the bitcast. If it got used with another intrinsic that new how to bitcast back those bitcasts would cancel.

But I guess we have a bunch of intrinsics that take scalar integer as arguments for mask and introduce a bitcast to vXi1 in intrinsic lowering. Which is too late to cancel the bitcast that would form MOVMSK. Maybe we need a combine to reverse MOVMSK? I guess I need to do some experiments.

[craig.topper]

Add a reversing transform for bitcasts that appear during intrinsic lowering. This
required a new test file. The output for that file shown here is the current output
before this patch. No existing tests in tree demonstrated the need for the reversing
combine.

[RKSimon]

llvm/test/CodeGen/X86/bitcast-vector-bool.ll

6

Add a common --check-prefixes=AVX prefix

llvm/test/CodeGen/X86/movmsk-cmp.ll

6

Add --check-prefixes=AVX,AVX512 common prefixes to KNL/SKX now that they lower to the same code.

[RKSimon]

LGTM - cheers

llvm/lib/Target/X86/X86ISelLowering.cpp

if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
       return DAG.getZExtOrTrunc(V, DL, VT);
     }
   }
 
   // If the input is a truncate from v16i8 or v32i8 go ahead and use a
   // movmskb even with avx512. This will be better than truncating to vXi1 and
   // using a kmov. This can especially help KNL if the input is a v16i8/v32i8
   // vpcmpeqb/vpcmpgtb.
-  bool IsTruncated = Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
+  bool PreferMovMsk = Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
                       (Src.getOperand(0).getValueType() == MVT::v16i8 ||
                        Src.getOperand(0).getValueType() == MVT::v32i8 ||
                        Src.getOperand(0).getValueType() == MVT::v64i8);
 
+  // Prefer movmsk for AVX512 for (bitcast (setlt X, 0)) which can be handled
+  // directly with vpmovmskb/vmovmskps/vmovmskpd.
+  if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse() &&
+      cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT &&
+      ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode())) {
+    EVT CmpVT = Src.getOperand(0).getValueType();
+    EVT EltVT = CmpVT.getVectorElementType();
+    if (CmpVT.getSizeInBits() <= 256 &&
+        (EltVT == MVT::i8 || EltVT == MVT::i32 || EltVT == MVT::i64))
+      PreferMovMsk = true;
+  }
+
   // With AVX512 vxi1 types are legal and we prefer using k-regs.
   // MOVMSK is supported in SSE2 or later.
-  if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !IsTruncated))
+  if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !PreferMovMsk))
     return SDValue();
 
   // There are MOVMSK flavors for types v16i8, v32i8, v4f32, v8f32, v4f64 and
   // v8f64. So all legal 128-bit and 256-bit vectors are covered except for
   // v8i16 and v16i16.
   // For these two cases, we can shuffle the upper element bytes to a
   // consecutive sequence at the start of the vector and treat the results as
   // v16i8 or v32i8, and for v16i8 this is the preferable solution. However,
if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
       isa<ConstantSDNode>(N0)) {
     auto *C = cast<ConstantSDNode>(N0);
     if (C->isAllOnesValue())
       return DAG.getConstant(1, SDLoc(N0), VT);
     if (C->isNullValue())
       return DAG.getConstant(0, SDLoc(N0), VT);
   }
 
+  // Look for MOVMSK that is maybe truncated and then bitcasted to vXi1.
+  // Turn it into a sign bit compare that produces a k-register. This avoids
+  // a trip through a GPR.
+  if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
+      VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
+      isPowerOf2_32(VT.getVectorNumElements())) {
+    unsigned NumElts = VT.getVectorNumElements();
+    SDValue Src = N0;
+
+    // Peek through truncate.
+    if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse())
+      Src = N0.getOperand(0);
+
+    if (Src.getOpcode() == X86ISD::MOVMSK && Src.hasOneUse()) {
+      SDValue MovmskIn = Src.getOperand(0);
+      MVT MovmskVT = MovmskIn.getSimpleValueType();
+      unsigned MovMskElts = MovmskVT.getVectorNumElements();
+
+      // We allow extra bits of the movmsk to be used since they are known zero.
+      // We can't convert a VPMOVMSKB without avx512bw.
+      if (MovMskElts <= NumElts &&
+          (Subtarget.hasBWI() || MovmskVT.getVectorElementType() != MVT::i8)) {
+        EVT IntVT = EVT(MovmskVT).changeVectorElementTypeToInteger();
+        MovmskIn = DAG.getBitcast(IntVT, MovmskIn);
+        SDLoc dl(N);
+        MVT CmpVT = MVT::getVectorVT(MVT::i1, MovMskElts);
+        SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn,
+                                   DAG.getConstant(0, dl, IntVT), ISD::SETLT);
+        if (EVT(CmpVT) == VT)
+          return Cmp;
+
+        // Pad with zeroes up to original VT to replace the zeroes that were
+        // being used from the MOVMSK.
+        unsigned NumConcats = NumElts / MovMskElts;
+        SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT));
+        Ops[0] = Cmp;
+        return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops);
+      }
+    }
+  }
+
   // Try to remove bitcasts from input and output of mask arithmetic to
   // remove GPR<->K-register crossings.
   if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
     return V;
 
   // Convert a bitcasted integer logic operation that has one bitcasted
   // floating-point operand into a floating-point logic operation. This may
   // create a load of a constant, but that is cheaper than materializing the

llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll

; X64-NEXT:    retq # encoding: [0xc3]
   ret i16 %res2
 }
 
 declare i16 @llvm.x86.avx512.cvtb2mask.128(<16 x i8>)
 
 define i16@test_int_x86_avx512_cvtb2mask_128(<16 x i8> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_cvtb2mask_128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovb2m %xmm0, %k0 # encoding: [0x62,0xf2,0x7e,0x08,0x29,0xc0]
-; CHECK-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    vpmovmskb %xmm0, %eax # encoding: [0xc5,0xf9,0xd7,0xc0]
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
     %res = call i16 @llvm.x86.avx512.cvtb2mask.128(<16 x i8> %x0)
     ret i16 %res
 }
 
 declare i32 @llvm.x86.avx512.cvtb2mask.256(<32 x i8>)
 
 define i32@test_int_x86_avx512_cvtb2mask_256(<32 x i8> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_cvtb2mask_256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovb2m %ymm0, %k0 # encoding: [0x62,0xf2,0x7e,0x28,0x29,0xc0]
-; CHECK-NEXT:    kmovd %k0, %eax # encoding: [0xc5,0xfb,0x93,0xc0]
+; CHECK-NEXT:    vpmovmskb %ymm0, %eax # encoding: [0xc5,0xfd,0xd7,0xc0]
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
     %res = call i32 @llvm.x86.avx512.cvtb2mask.256(<32 x i8> %x0)
     ret i32 %res
 }
 
 declare i8 @llvm.x86.avx512.cvtw2mask.128(<8 x i16>)
 

llvm/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll

; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
     ret i8 %res
 }
 
 declare i8 @llvm.x86.avx512.cvtd2mask.256(<8 x i32>)
 
 define i8@test_int_x86_avx512_cvtd2mask_256(<8 x i32> %x0) {
 ; CHECK-LABEL: test_int_x86_avx512_cvtd2mask_256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovd2m %ymm0, %k0 # encoding: [0x62,0xf2,0x7e,0x28,0x39,0xc0]
-; CHECK-NEXT:    kmovw %k0, %eax # encoding: [0xc5,0xf8,0x93,0xc0]
+; CHECK-NEXT:    vmovmskps %ymm0, %eax # encoding: [0xc5,0xfc,0x50,0xc0]
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
     %res = call i8 @llvm.x86.avx512.cvtd2mask.256(<8 x i32> %x0)
     ret i8 %res
 }
 
 declare i8 @llvm.x86.avx512.cvtq2mask.128(<2 x i64>)

llvm/test/CodeGen/X86/bitcast-setcc-256.ll

 ;
 ; AVX12-LABEL: bitcast_8i32_store:
 ; AVX12:       # %bb.0:
 ; AVX12-NEXT:    vmovmskps %ymm0, %eax
 ; AVX12-NEXT:    movb %al, (%rdi)
 ; AVX12-NEXT:    vzeroupper
 ; AVX12-NEXT:    retq
 ;
-; AVX512F-LABEL: bitcast_8i32_store:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpcmpgtd %ymm0, %ymm1, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    movb %al, (%rdi)
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: bitcast_8i32_store:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpcmpgtd %ymm0, %ymm1, %k0
-; AVX512BW-NEXT:    kmovd %k0, %eax
-; AVX512BW-NEXT:    movb %al, (%rdi)
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: bitcast_8i32_store:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovmskps %ymm0, %eax
+; AVX512-NEXT:    movb %al, (%rdi)
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %a1 = icmp slt <8 x i32> %a0, zeroinitializer
   %a2 = bitcast <8 x i1> %a1 to i8
   store i8 %a2, i8* %p
   ret void
 }
 
 define void @bitcast_4i64_store(i4* %p, <4 x i64> %a0) {
 ; SSE2-SSSE3-LABEL: bitcast_4i64_store:

llvm/test/CodeGen/X86/bitcast-vector-bool.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2-SSSE3,SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE2-SSSE3,SSSE3
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX12,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX12,AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512
 
 ;
 ; 128-bit vectors
 ;
 
 define i1 @bitcast_v2i64_to_v2i1(<2 x i64> %a0) nounwind {
 ; SSE2-SSSE3-LABEL: bitcast_v2i64_to_v2i1:
 ; SSE2-SSSE3:       # %bb.0:

 ; AVX12-NEXT:    shrb $2, %cl
 ; AVX12-NEXT:    andb $3, %al
 ; AVX12-NEXT:    addb %cl, %al
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    retq
 ;
 ; AVX512-LABEL: bitcast_v4i32_to_v2i2:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm1, %k0
-; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    vmovmskps %xmm0, %eax
 ; AVX512-NEXT:    movl %eax, %ecx
 ; AVX512-NEXT:    shrb $2, %cl
 ; AVX512-NEXT:    andb $3, %al
 ; AVX512-NEXT:    addb %cl, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %1 = icmp slt <4 x i32> %a0, zeroinitializer
   %2 = bitcast <4 x i1> %1 to <2 x i2>

 ; AVX12-NEXT:    movl %ecx, %eax
 ; AVX12-NEXT:    shrl $8, %eax
 ; AVX12-NEXT:    addb %cl, %al
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    retq
 ;
 ; AVX512-LABEL: bitcast_v16i8_to_v2i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovb2m %xmm0, %k0
-; AVX512-NEXT:    kmovw %k0, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %ecx
-; AVX512-NEXT:    vpextrb $1, %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %xmm0, %ecx
+; AVX512-NEXT:    movl %ecx, %eax
+; AVX512-NEXT:    shrl $8, %eax
 ; AVX512-NEXT:    addb %cl, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %1 = icmp slt <16 x i8> %a0, zeroinitializer
   %2 = bitcast <16 x i1> %1 to <2 x i8>
   %3 = extractelement <2 x i8> %2, i32 0
   %4 = extractelement <2 x i8> %2, i32 1
   %5 = add i8 %3, %4

 ; AVX12-NEXT:    andb $3, %al
 ; AVX12-NEXT:    addb %cl, %al
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    vzeroupper
 ; AVX12-NEXT:    retq
 ;
 ; AVX512-LABEL: bitcast_v4i64_to_v2i2:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm1, %k0
-; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    vmovmskpd %ymm0, %eax
 ; AVX512-NEXT:    movl %eax, %ecx
 ; AVX512-NEXT:    shrb $2, %cl
 ; AVX512-NEXT:    andb $3, %al
 ; AVX512-NEXT:    addb %cl, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %1 = icmp slt <4 x i64> %a0, zeroinitializer

 ; AVX12-NEXT:    andb $15, %al
 ; AVX12-NEXT:    addb %cl, %al
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX12-NEXT:    vzeroupper
 ; AVX12-NEXT:    retq
 ;
 ; AVX512-LABEL: bitcast_v8i32_to_v2i4:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vpcmpgtd %ymm0, %ymm1, %k0
-; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    vmovmskps %ymm0, %eax
 ; AVX512-NEXT:    movl %eax, %ecx
 ; AVX512-NEXT:    shrb $4, %cl
 ; AVX512-NEXT:    andb $15, %al
 ; AVX512-NEXT:    addb %cl, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %1 = icmp slt <8 x i32> %a0, zeroinitializer

 ; AVX2-NEXT:    shrl $16, %eax
 ; AVX2-NEXT:    addl %ecx, %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: bitcast_v32i8_to_v2i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    pushq %rbp
-; AVX512-NEXT:    movq %rsp, %rbp
-; AVX512-NEXT:    andq $-32, %rsp
-; AVX512-NEXT:    subq $32, %rsp
-; AVX512-NEXT:    vpmovb2m %ymm0, %k0
-; AVX512-NEXT:    kmovd %k0, (%rsp)
-; AVX512-NEXT:    vmovdqa (%rsp), %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %ecx
-; AVX512-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %ymm0, %ecx
+; AVX512-NEXT:    movl %ecx, %eax
+; AVX512-NEXT:    shrl $16, %eax
 ; AVX512-NEXT:    addl %ecx, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    movq %rbp, %rsp
-; AVX512-NEXT:    popq %rbp
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %1 = icmp slt <32 x i8> %a0, zeroinitializer
   %2 = bitcast <32 x i1> %1 to <2 x i16>
   %3 = extractelement <2 x i16> %2, i32 0
   %4 = extractelement <2 x i16> %2, i32 1
   %5 = add i16 %3, %4
   ret i16 %5

llvm/test/CodeGen/X86/movmsk-cmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s  --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s  --check-prefix=AVX --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s  --check-prefix=KNL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s  --check-prefix=SKX
 
 define i1 @allones_v16i8_sign(<16 x i8> %arg) {
 ; SSE2-LABEL: allones_v16i8_sign:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    cmpw $-1, %ax
 ; SSE2-NEXT:    sete %al
 ; SSE2-NEXT:    retq

 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpmovmskb %xmm0, %eax
 ; KNL-NEXT:    cmpw $-1, %ax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v16i8_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovb2m %xmm0, %k0
-; SKX-NEXT:    kortestw %k0, %k0
-; SKX-NEXT:    setb %al
+; SKX-NEXT:    vpmovmskb %xmm0, %eax
+; SKX-NEXT:    cmpw $-1, %ax
+; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = icmp slt <16 x i8> %arg, zeroinitializer
   %tmp1 = bitcast <16 x i1> %tmp to i16
   %tmp2 = icmp eq i16 %tmp1, -1
   ret i1 %tmp2
 }
 
 define i1 @allzeros_v16i8_sign(<16 x i8> %arg) {

 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpmovmskb %xmm0, %eax
 ; KNL-NEXT:    testw %ax, %ax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v16i8_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovb2m %xmm0, %k0
-; SKX-NEXT:    kortestw %k0, %k0
+; SKX-NEXT:    vpmovmskb %xmm0, %eax
+; SKX-NEXT:    testw %ax, %ax
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = icmp slt <16 x i8> %arg, zeroinitializer
   %tmp1 = bitcast <16 x i1> %tmp to i16
   %tmp2 = icmp eq i16 %tmp1, 0
   ret i1 %tmp2
 }
 

 ; KNL-NEXT:    vpmovmskb %ymm0, %eax
 ; KNL-NEXT:    cmpl $-1, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v32i8_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovb2m %ymm0, %k0
-; SKX-NEXT:    kortestd %k0, %k0
-; SKX-NEXT:    setb %al
+; SKX-NEXT:    vpmovmskb %ymm0, %eax
+; SKX-NEXT:    cmpl $-1, %eax
+; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %tmp = icmp slt <32 x i8> %arg, zeroinitializer
   %tmp1 = bitcast <32 x i1> %tmp to i32
   %tmp2 = icmp eq i32 %tmp1, -1
   ret i1 %tmp2
 }
 

 ; KNL-NEXT:    vpmovmskb %ymm0, %eax
 ; KNL-NEXT:    testl %eax, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v32i8_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovb2m %ymm0, %k0
-; SKX-NEXT:    kortestd %k0, %k0
+; SKX-NEXT:    vpmovmskb %ymm0, %eax
+; SKX-NEXT:    testl %eax, %eax
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %tmp = icmp slt <32 x i8> %arg, zeroinitializer
   %tmp1 = bitcast <32 x i1> %tmp to i32
   %tmp2 = icmp eq i32 %tmp1, 0
   ret i1 %tmp2
 }

 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovmskps %xmm0, %eax
 ; AVX-NEXT:    cmpb $15, %al
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: allones_v4i32_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    andb $15, %al
+; KNL-NEXT:    vmovmskps %xmm0, %eax
 ; KNL-NEXT:    cmpb $15, %al
 ; KNL-NEXT:    sete %al
-; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v4i32_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
+; SKX-NEXT:    vmovmskps %xmm0, %eax
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = icmp slt <4 x i32> %arg, zeroinitializer
   %tmp1 = bitcast <4 x i1> %tmp to i4
   %tmp2 = icmp eq i4 %tmp1, -1
   ret i1 %tmp2
 }

 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovmskps %xmm0, %eax
 ; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: allzeros_v4i32_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    testb $15, %al
+; KNL-NEXT:    vmovmskps %xmm0, %eax
+; KNL-NEXT:    testb %al, %al
 ; KNL-NEXT:    sete %al
-; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v4i32_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %xmm0, %k0
-; SKX-NEXT:    kortestb %k0, %k0
+; SKX-NEXT:    vmovmskps %xmm0, %eax
+; SKX-NEXT:    testb %al, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    retq
   %tmp = icmp slt <4 x i32> %arg, zeroinitializer
   %tmp1 = bitcast <4 x i1> %tmp to i4
   %tmp2 = icmp eq i4 %tmp1, 0
   ret i1 %tmp2
 }
 

 ; AVX-NEXT:    vmovmskps %ymm0, %eax
 ; AVX-NEXT:    cmpb $-1, %al
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: allones_v8i32_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    vmovmskps %ymm0, %eax
 ; KNL-NEXT:    cmpb $-1, %al
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v8i32_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %ymm0, %k0
-; SKX-NEXT:    kortestb %k0, %k0
-; SKX-NEXT:    setb %al
+; SKX-NEXT:    vmovmskps %ymm0, %eax
+; SKX-NEXT:    cmpb $-1, %al
+; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %tmp = icmp slt <8 x i32> %arg, zeroinitializer
   %tmp1 = bitcast <8 x i1> %tmp to i8
   %tmp2 = icmp eq i8 %tmp1, -1
   ret i1 %tmp2
 }
 

 ; AVX-NEXT:    vmovmskps %ymm0, %eax
 ; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: allzeros_v8i32_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    vmovmskps %ymm0, %eax
 ; KNL-NEXT:    testb %al, %al
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v8i32_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %ymm0, %k0
-; SKX-NEXT:    kortestb %k0, %k0
+; SKX-NEXT:    vmovmskps %ymm0, %eax
+; SKX-NEXT:    testb %al, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %tmp = icmp slt <8 x i32> %arg, zeroinitializer
   %tmp1 = bitcast <8 x i1> %tmp to i8
   %tmp2 = icmp eq i8 %tmp1, 0
   ret i1 %tmp2
 }

 ; AVX-NEXT:    vmovmskpd %ymm0, %eax
 ; AVX-NEXT:    cmpb $15, %al
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: allones_v4i64_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    andb $15, %al
+; KNL-NEXT:    vmovmskpd %ymm0, %eax
 ; KNL-NEXT:    cmpb $15, %al
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v4i64_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovq2m %ymm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
+; SKX-NEXT:    vmovmskpd %ymm0, %eax
 ; SKX-NEXT:    cmpb $15, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %tmp = icmp slt <4 x i64> %arg, zeroinitializer
   %tmp1 = bitcast <4 x i1> %tmp to i4
   %tmp2 = icmp eq i4 %tmp1, -1
   ret i1 %tmp2

 ; AVX-NEXT:    vmovmskpd %ymm0, %eax
 ; AVX-NEXT:    testb %al, %al
 ; AVX-NEXT:    sete %al
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: allzeros_v4i64_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    testb $15, %al
+; KNL-NEXT:    vmovmskpd %ymm0, %eax
+; KNL-NEXT:    testb %al, %al
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v4i64_sign:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovq2m %ymm0, %k0
-; SKX-NEXT:    kortestb %k0, %k0
+; SKX-NEXT:    vmovmskpd %ymm0, %eax
+; SKX-NEXT:    testb %al, %al
 ; SKX-NEXT:    sete %al
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %tmp = icmp slt <4 x i64> %arg, zeroinitializer
   %tmp1 = bitcast <4 x i1> %tmp to i4
   %tmp2 = icmp eq i4 %tmp1, 0
   ret i1 %tmp2
 }

 ;
 ; AVX-LABEL: movmskpd:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovmskpd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: movmskpd:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    andl $3, %eax
-; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    vmovmskpd %xmm0, %eax
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: movmskpd:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovq2m %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andl $3, %eax
+; SKX-NEXT:    vmovmskpd %xmm0, %eax
 ; SKX-NEXT:    retq
   %a = bitcast <2 x double> %x to <2 x i64>
   %b = icmp slt <2 x i64> %a, zeroinitializer
   %c = bitcast <2 x i1> %b to i2
   %d = zext i2 %c to i32
   ret i32 %d
 }
 
 define i32 @movmskps(<4 x float> %x) {
 ; SSE2-LABEL: movmskps:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: movmskps:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovmskps %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: movmskps:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    andl $15, %eax
-; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    vmovmskps %xmm0, %eax
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: movmskps:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andl $15, %eax
+; SKX-NEXT:    vmovmskps %xmm0, %eax
 ; SKX-NEXT:    retq
   %a = bitcast <4 x float> %x to <4 x i32>
   %b = icmp slt <4 x i32> %a, zeroinitializer
   %c = bitcast <4 x i1> %b to i4
   %d = zext i4 %c to i32
   ret i32 %d
 }
 
 define i32 @movmskpd256(<4 x double> %x) {
 ; SSE2-LABEL: movmskpd256:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: movmskpd256:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovmskpd %ymm0, %eax
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: movmskpd256:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    andl $15, %eax
+; KNL-NEXT:    vmovmskpd %ymm0, %eax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: movmskpd256:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovq2m %ymm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    andl $15, %eax
+; SKX-NEXT:    vmovmskpd %ymm0, %eax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %a = bitcast <4 x double> %x to <4 x i64>
   %b = icmp slt <4 x i64> %a, zeroinitializer
   %c = bitcast <4 x i1> %b to i4
   %d = zext i4 %c to i32
   ret i32 %d
 }

 ; AVX-LABEL: movmskps256:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovmskps %ymm0, %eax
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
 ; KNL-LABEL: movmskps256:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    movzbl %al, %eax
+; KNL-NEXT:    vmovmskps %ymm0, %eax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: movmskps256:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %ymm0, %k0
-; SKX-NEXT:    kmovb %k0, %eax
+; SKX-NEXT:    vmovmskps %ymm0, %eax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %a = bitcast <8 x float> %x to <8 x i32>
   %b = icmp slt <8 x i32> %a, zeroinitializer
   %c = bitcast <8 x i1> %b to i8
   %d = zext i8 %c to i32
   ret i32 %d
 }

 ;
 ; KNL-LABEL: movmskb:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpmovmskb %xmm0, %eax
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: movmskb:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovb2m %xmm0, %k0
-; SKX-NEXT:    kmovw %k0, %eax
+; SKX-NEXT:    vpmovmskb %xmm0, %eax
 ; SKX-NEXT:    retq
   %a = icmp slt <16 x i8> %x, zeroinitializer
   %b = bitcast <16 x i1> %a to i16
   %c = zext i16 %b to i32
   ret i32 %c
 }
 
 define i32 @movmskb256(<32 x i8> %x) {

 ; KNL-LABEL: movmskb256:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpmovmskb %ymm0, %eax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: movmskb256:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovb2m %ymm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
+; SKX-NEXT:    vpmovmskb %ymm0, %eax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
   %a = icmp slt <32 x i8> %x, zeroinitializer
   %b = bitcast <32 x i1> %a to i32
   ret i32 %b
 }
 
 ; Multiple extract elements from a vector compare.