[DAGCombiner][X86][PowerPC] Teach visitSIGN_EXTEND_INREG to fold (sext_in_reg (aext/sext x)) -> (sext x) when x has more than 1 sign bit and the sext_inreg is from one of them.

[craig.topper]

Summary

If x has multiple sign bits than it doesn't matter which one we extend from so we can sext from x's msb instead.

The X86 setcc-combine.ll changes are a little weird, but probably not much of a performance difference. It appears we ended up with a (sext_inreg (aext (trunc (extractelt)))) after type legalization. The sext_inreg+aext now gets optimized by this combine to leave (sext (trunc (extractelt))). Then we visit the trunc before we visit the sext. This ends up changing the truncate to an extractvectorelt from a bitcasted vector. I can see a couple ways to fix this. Early exit from visitTRUNCATE if its used by a sext similar to what is done for aext. This is tricky because we aren't always able to optimize a sext+trunc. The other option is to change type legalization to promote extractvectorelt inputs and result type simultaneously. If we promote the result first we can temporarily have an extractvectorelt with a result type we never really wanted.

[craig.topper]

Rebase

[hfinkel]

test/CodeGen/PowerPC/f128-truncateNconv.ll

358

This test-case change looks technically correct (note that this will change behavior of programs with UB (if the value being converted to an integer wasn't really representable in the smaller integer size)).

[RKSimon]

test/CodeGen/X86/setcc-combine.ll

2–3

Please can you replace the -mcpu with -mattr=+sse2/sse41 variants to prove that the codegen changes are different if we have suitable pextrd support?

[craig.topper]

Rebase. Looks like this is a regression on sse4.1 on setcc-combine.ll. I'll work on a follow up patch for that.

[RKSimon]

LGTM thanks

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
   if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
       EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
                        N0.getOperand(0), N1);
 
   // fold (sext_in_reg (sext x)) -> (sext x)
   // fold (sext_in_reg (aext x)) -> (sext x)
-  // if x is small enough.
+  // if x is small enough or if we know that x has more than 1 sign bit and the
+  // sign_extend_inreg is extending from one of them.
   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
     SDValue N00 = N0.getOperand(0);
-    if (N00.getScalarValueSizeInBits() <= EVTBits &&
+    unsigned N00Bits = N00.getScalarValueSizeInBits();
+    if ((N00Bits <= EVTBits ||
+         (N00Bits - DAG.ComputeNumSignBits(N00)) < EVTBits) &&
         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00);
   }
 
   // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
   if ((N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
        N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
        N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&

test/CodeGen/PowerPC/f128-truncateNconv.ll

 
 ; Function Attrs: norecurse nounwind readonly
 define signext i16 @qpConv2shw(fp128* nocapture readonly %a) {
 ; CHECK-LABEL: qpConv2shw:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsh r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
   %conv = fptosi fp128 %0 to i16
   ret i16 %conv
 }
 
 ; Function Attrs: norecurse nounwind

 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
 ; CHECK-NEXT:    lxv v3, 16(r4)
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsh r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
   %1 = load fp128, fp128* getelementptr inbounds
                             ([4 x fp128], [4 x fp128]* @f128Array,
                              i64 0, i64 1), align 16
   %add = fadd fp128 %0, %1
   %conv = fptosi fp128 %add to i16

 
 ; Function Attrs: norecurse nounwind readonly
 define signext i8 @qpConv2sb(fp128* nocapture readonly %a) {
 ; CHECK-LABEL: qpConv2sb:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsb r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
   %conv = fptosi fp128 %0 to i8
   ret i8 %conv
 }
 
 ; Function Attrs: norecurse nounwind

 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
 ; CHECK-NEXT:    lxv v3, 16(r4)
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsb r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
   %1 = load fp128, fp128* getelementptr inbounds
                             ([4 x fp128], [4 x fp128]* @f128Array,
                              i64 0, i64 1), align 16
   %add = fadd fp128 %0, %1
   %conv = fptosi fp128 %add to i8

test/CodeGen/PowerPC/ppc64-P9-setb.ll

 ; CHECK-LABEL: setb28:
 ; CHECK-NOT: xor
 ; CHECK-NOT: li
 ; CHECK: cmpd {{c?r?(0, )?}}r3, r4
 ; CHECK-NEXT: setb r3, cr0
 ; CHECK-NOT: addic
 ; CHECK-NOT: subfe
 ; CHECK-NOT: isel
-; CHECK: extsh
+; CHECK: extsw
 ; CHECK: blr
 ; CHECK-PWR8-LABEL: setb28
 ; CHECK-PWR8-DAG: cmpd
 ; CHECK-PWR8-DAG: addic
 ; CHECK-PWR8-DAG: subfe
 ; CHECK-PWR8: isel
-; CHECK-PWR8: extsh
+; CHECK-PWR8: extsw
 ; CHECK-PWR8: blr
 }
 
 ; select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setlt
 define i64 @setb29(i64 %a, i64 %b) {
   %t1 = icmp slt i64 %a, %b
   %t2 = icmp sgt i64 %a, %b
   %t3 = zext i1 %t2 to i8

test/CodeGen/X86/setcc-combine.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefixes=CHECK,SSE2
 ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s --check-prefixes=CHECK,SSE41
 
 define i32 @test_eq_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_eq_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_eq_1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp eq <4 x i32> %sext, zeroinitializer
   %t0 = extractelement <4 x i1> %cmp1, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }
 
 define i32 @test_ne_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_ne_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_ne_1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    pextrb $4, %xmm1, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp ne <4 x i32> %sext, zeroinitializer
   %t0 = extractelement <4 x i1> %cmp1, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }

 }
 
 define i32 @test_ge_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_ge_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_ge_1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp sge <4 x i32> %sext, zeroinitializer
   %t0 = extractelement <4 x i1> %cmp1, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }
 
 define i32 @test_lt_1(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_lt_1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_lt_1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    pextrb $4, %xmm1, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp slt <4 x i32> %sext, zeroinitializer
   %t0 = extractelement <4 x i1> %cmp, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }

 }
 
 define i32 @test_eq_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_eq_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_eq_2:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    pextrb $4, %xmm1, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp eq <4 x i32> %sext, zeroinitializer
   %t0 = extractelement <4 x i1> %cmp1, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }
 
 define i32 @test_ne_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_ne_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_ne_2:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp ne <4 x i32> %sext, zeroinitializer
   %t0 = extractelement <4 x i1> %cmp1, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }
 
 define i32 @test_le_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_le_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_le_2:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    pextrb $4, %xmm1, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp sle <4 x i32> zeroinitializer, %sext
   %t0 = extractelement <4 x i1> %cmp1, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }
; CHECK-NEXT:    retq
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }
 
 define i32 @test_lt_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_lt_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_lt_2:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp slt <4 x i32> zeroinitializer, %sext
   %t0 = extractelement <4 x i1> %cmp, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }
 
 define i32 @test_gt_2(<4 x i32> %A, <4 x i32> %B) {
 ; SSE2-LABEL: test_gt_2:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    pextrw $2, %xmm0, %eax
+; SSE2-NEXT:    movsbl %al, %eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_gt_2:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    pextrb $4, %xmm0, %eax
+; SSE41-NEXT:    movsbl %al, %eax
 ; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp sgt <4 x i32> zeroinitializer, %sext
   %t0 = extractelement <4 x i1> %cmp1, i32 1
   %t1 = sext i1 %t0 to i32
   ret i32 %t1
 }

test/CodeGen/X86/vsel-cmp-load.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx                       | FileCheck %s --check-prefixes=ALL,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2                      | FileCheck %s --check-prefixes=ALL,AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw,avx512vl,avx512f | FileCheck %s --check-prefixes=ALL,AVX512
 
 ; PR37427 - https://bugs.llvm.org/show_bug.cgi?id=37427
 
 define <8 x i32> @eq_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ; AVX1-LABEL: eq_zero:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX1-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX1-NEXT:    vpslld $24, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: eq_zero:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
; AVX512-NEXT:    retq
   %sel = select <16 x i1> %cmp, <16 x i16> %x, <16 x i16> %y
   ret <16 x i16> %sel
 }
 
 define <8 x i32> @slt_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ; AVX1-LABEL: slt_zero:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX1-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX1-NEXT:    vpslld $24, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: slt_zero:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovsxbd (%rdi), %ymm2
 ; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0

 define <8 x float> @ne_zero_fp_select(<8 x i8>* %p, <8 x float> %x, <8 x float> %y) {
 ; AVX1-LABEL: ne_zero_fp_select:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX1-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX1-NEXT:    vpslld $24, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ne_zero_fp_select:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3