[x86, SSE] optimize pcmp results better (PR28484)

[spatel]

Summary

We know that pcmp (SSE/AVX at least; I'm intentionally leaving 512-bit out of this patch because I don't know what happens there) produces all-ones/all-zeros bitmasks, so we can use that behavior to avoid unnecessary constant loading.

FWIW, I see no perf differences in test-suite with this change. I don't expect that a zext of a bitmask is a common pattern. This is a first step towards the better motivating example in PR28486:
https://llvm.org/bugs/show_bug.cgi?id=28486
...which is itself just an extract from a case where we seemingly get everything wrong:
https://godbolt.org/g/Ez2bDW

One could argue that load+and is actually a better solution for some CPUs (Intel big cores) because shifts don't have the same throughput potential as load+and on those cores, but I think that should be handled as a CPU-specific later transformation if it ever comes up. Removing the load is the more general x86 optimization. Note that the uneven usage of vpbroadcast in the test cases is filed as PR28505:
https://llvm.org/bugs/show_bug.cgi?id=28505

[delena]

lib/Target/X86/X86ISelLowering.cpp

28139 ↗

(On Diff #63519)

We cam mark nodes like PCMP with AssertSext. And use this marker to simplify AND.

28148 ↗

(On Diff #63519)

VT0 is always equal to VT1.

28152 ↗

(On Diff #63519)

On AVX-512 (skylake-avx512) the result is in a mask reg, also for 256 and 128 vector inputs.

[spatel]

lib/Target/X86/X86ISelLowering.cpp

28139 ↗

(On Diff #63519)

Ah - I didn't know about AssertSext. To use it, we would add one of those nodes any time we create a PCMPEQ/PCMPGT? And then we would check for an AssertSext at this point rather than PCMPEQ/PCMPGT?

Ok if I add a 'TODO' comment in this patch?

28148 ↗

(On Diff #63519)

I was paranoid that something like this:

      t18: v4i32 = setcc t2, t4, seteq:ch
      t17: v4i32 = BUILD_VECTOR Constant:i32<1>, Constant:i32<1>, Constant:i32<1>, Constant:i32<1>
    t19: v4i32 = and t18, t17
  t8: v2i64 = bitcast t19
  t10: v2i64 = BUILD_VECTOR Constant:i64<4294967297>, Constant:i64<4294967297>
t11: v2i64 = and t8, t10

might cause the original (v4i32) constant to get folded away; ie, we might have a bitcast on one side of the 'and' but not the other. If this can't possibly happen, then certainly we can remove the check.

28152 ↗

(On Diff #63519)

If it is the mask form, then wouldn't the node be PCMPEQM rather than PCMPEQ?

[delena]

lib/Target/X86/X86ISelLowering.cpp

28139 ↗

(On Diff #63519)

yes

28152 ↗

(On Diff #63519)

you are right. I think you can remove "todo AVX-512" from the comments.

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST &&
     if (N00Type.isFloatingPoint() && N10Type.isFloatingPoint()) {
       SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
       return DAG.getBitcast(VT, FPLogic);
     }
   }
   return SDValue();
 }
 
+/// If this is a PCMPEQ or PCMPGT result that is bitwise-anded with 1 (this is
+/// the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to
+/// eliminate loading the vector constant mask value. This relies on the fact
+/// that a PCMP always creates an all-ones or all-zeros bitmask per element.
+static SDValue combinePCMPAnd1(SDNode *N, SelectionDAG &DAG) {
+  SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
+  SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
+
+  // TODO: Use AssertSext to mark any nodes that have the property of producing
+  // all-ones or all-zeros. Then check for that node rather than particular
+  // opcodes.
+  if (Op0.getOpcode() != X86ISD::PCMPEQ && Op0.getOpcode() != X86ISD::PCMPGT)
+    return SDValue();
+
+  // The existence of the PCMP node guarantees that we have the required SSE2 or
+  // AVX2 for a shift of this vector type, but there is no vector shift by
+  // immediate for a vector with byte elements (PSRLB). 512-bit vectors use the
+  // masked compare nodes, so they should not make it here.
+  EVT VT0 = Op0.getValueType();
+  EVT VT1 = Op1.getValueType();
+  unsigned EltBitWidth = VT0.getScalarType().getSizeInBits();
+  if (VT0 != VT1 || EltBitWidth == 8)
+    return SDValue();
+
+  assert(VT0.getSizeInBits() == 128 || VT0.getSizeInBits() == 256);
+
+  APInt SplatVal;
+  if (!ISD::isConstantSplatVector(Op1.getNode(), SplatVal) || SplatVal != 1)
+    return SDValue();
+
+  SDLoc DL(N);
+  SDValue ShAmt = DAG.getConstant(EltBitWidth - 1, DL, MVT::i8);
+  SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
+  return DAG.getBitcast(N->getValueType(0), Shift);
+}
+
 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
                           TargetLowering::DAGCombinerInfo &DCI,
                           const X86Subtarget &Subtarget) {
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
   if (SDValue Zext = combineVectorZext(N, DAG, DCI, Subtarget))
     return Zext;
 
   if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
     return R;
 
   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
     return FPLogic;
 
   if (SDValue R = combineANDXORWithAllOnesIntoANDNP(N, DAG))
     return R;
 
+  if (SDValue ShiftRight = combinePCMPAnd1(N, DAG))
+    return ShiftRight;
+
   EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDLoc DL(N);
 
   // Create BEXTR instructions
   // BEXTR is ((X >> imm) & (2**size-1))
   if (VT != MVT::i32 && VT != MVT::i64)

llvm/trunk/test/CodeGen/X86/avx512-ext.ll

; SKX-NEXT:    retq
   %1 = zext <64 x i1> %mask to <64 x i8>
   ret <64 x i8> %1
 }
 
 define <32 x i16> @zext_32xi1_to_32xi16(<32 x i16> %x, <32 x i16> %y) #0 {
 ; KNL-LABEL: zext_32xi1_to_32xi16:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vpsrlw $15, %ymm0, %ymm0
 ; KNL-NEXT:    vpcmpeqw %ymm3, %ymm1, %ymm1
-; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vpsrlw $15, %ymm1, %ymm1
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_32xi1_to_32xi16:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpcmpeqw %zmm1, %zmm0, %k1
 ; SKX-NEXT:    vmovdqu16 {{.*}}(%rip), %zmm0 {%k1} {z}
 ; SKX-NEXT:    retq
   %mask = icmp eq <32 x i16> %x, %y
   %1 = zext <32 x i1> %mask to <32 x i16>
   ret <32 x i16> %1
 }
 
 define <16 x i16> @zext_16xi1_to_16xi16(<16 x i16> %x, <16 x i16> %y) #0 {
 ; KNL-LABEL: zext_16xi1_to_16xi16:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    vpsrlw $15, %ymm0, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_16xi1_to_16xi16:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpcmpeqw %ymm1, %ymm0, %k1
 ; SKX-NEXT:    vmovdqu16 {{.*}}(%rip), %ymm0 {%k1} {z}
 ; SKX-NEXT:    retq
   %mask = icmp eq <16 x i16> %x, %y

 
 define <4 x i32> @zext_4xi1_to_4x32(<4 x i8> %x, <4 x i8> %y) #0 {
 ; KNL-LABEL: zext_4xi1_to_4x32:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
 ; KNL-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; KNL-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; KNL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1
-; KNL-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; KNL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_4xi1_to_4x32:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vmovdqa64 {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
 ; SKX-NEXT:    vpandq %xmm2, %xmm1, %xmm1
 ; SKX-NEXT:    vpandq %xmm2, %xmm0, %xmm0
 ; SKX-NEXT:    vpcmpeqd %xmm1, %xmm0, %k1
 ; SKX-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k1} {z}
 ; SKX-NEXT:    retq
   %mask = icmp eq <4 x i8> %x, %y
   %1 = zext <4 x i1> %mask to <4 x i32>
   ret <4 x i32> %1
 }
 
 define <2 x i64> @zext_2xi1_to_2xi64(<2 x i8> %x, <2 x i8> %y) #0 {
 ; KNL-LABEL: zext_2xi1_to_2xi64:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
 ; KNL-NEXT:    vpand %xmm2, %xmm1, %xmm1
 ; KNL-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; KNL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpsrlq $63, %xmm0, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: zext_2xi1_to_2xi64:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vmovdqa64 {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
 ; SKX-NEXT:    vpandq %xmm2, %xmm1, %xmm1
 ; SKX-NEXT:    vpandq %xmm2, %xmm0, %xmm0
 ; SKX-NEXT:    vpcmpeqq %xmm1, %xmm0, %k1
 ; SKX-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm0 {%k1} {z}
 ; SKX-NEXT:    retq
   %mask = icmp eq <2 x i8> %x, %y
   %1 = zext <2 x i1> %mask to <2 x i64>
   ret <2 x i64> %1
 }

llvm/trunk/test/CodeGen/X86/avx512-vec-cmp.ll

 
 define <2 x i64> @test45(<2 x i16> %x, <2 x i16> %y) #0 {
 ; KNL-LABEL: test45:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; KNL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
 ; KNL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
 ; KNL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    vpsrlq $63, %xmm0, %xmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test45:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpxord %xmm2, %xmm2, %xmm2
 ; SKX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7]
 ; SKX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
 ; SKX-NEXT:    vpcmpeqq %xmm1, %xmm0, %k1

llvm/trunk/test/CodeGen/X86/shift-pcmp.ll

;
   ret <8 x i16> %shl
 }
 
 ; Don't fail with an assert due to an undef in the buildvector
 define <8 x i16> @bar(<8 x i16> %a, <8 x i16> %b) {
 ; SSE-LABEL: bar:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
-; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    psrlw $15, %xmm0
 ; SSE-NEXT:    psllw $5, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: bar:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $15, %xmm0, %xmm0
 ; AVX-NEXT:    vpsllw $5, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
   %icmp = icmp eq <8 x i16> %a, %b
   %zext = zext <8 x i1> %icmp to <8 x i16>
   %shl = shl nuw nsw <8 x i16> %zext, <i16 5, i16 undef, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
   ret <8 x i16> %shl
 }
 

llvm/trunk/test/CodeGen/X86/vector-pcmp.ll

;
   %zext = zext <16 x i1> %cmp to <16 x i8>
   ret <16 x i8> %zext
 }
 
 define <16 x i16> @cmpeq_zext_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: cmpeq_zext_v16i16:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pcmpeqw %xmm2, %xmm0
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
-; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    psrlw $15, %xmm0
 ; SSE-NEXT:    pcmpeqw %xmm3, %xmm1
-; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    psrlw $15, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: cmpeq_zext_v16i16:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: cmpeq_zext_v16i16:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $15, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
   %cmp = icmp eq <16 x i16> %a, %b
   %zext = zext <16 x i1> %cmp to <16 x i16>
   ret <16 x i16> %zext
 }
 
 define <4 x i32> @cmpeq_zext_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; SSE-LABEL: cmpeq_zext_v4i32:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    psrld $31, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: cmpeq_zext_v4i32:
-; AVX1:       # BB#0:
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: cmpeq_zext_v4i32:
-; AVX2:       # BB#0:
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: cmpeq_zext_v4i32:
+; AVX:       # BB#0:
+; AVX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $31, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
   %cmp = icmp eq <4 x i32> %a, %b
   %zext = zext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %zext
 }
 
 define <4 x i64> @cmpeq_zext_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE2-LABEL: cmpeq_zext_v4i64:
 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,0,3,2]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
 ; SSE2-NEXT:    pand %xmm4, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm4, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: cmpeq_zext_v4i64:
 ; SSE42:       # BB#0:
 ; SSE42-NEXT:    pcmpeqq %xmm2, %xmm0
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [1,1]
-; SSE42-NEXT:    pand %xmm2, %xmm0
+; SSE42-NEXT:    psrlq $63, %xmm0
 ; SSE42-NEXT:    pcmpeqq %xmm3, %xmm1
-; SSE42-NEXT:    pand %xmm2, %xmm1
+; SSE42-NEXT:    psrlq $63, %xmm1
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: cmpeq_zext_v4i64:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: cmpeq_zext_v4i64:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlq $63, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
   %cmp = icmp eq <4 x i64> %a, %b
   %zext = zext <4 x i1> %cmp to <4 x i64>
   ret <4 x i64> %zext
 }
 
 define <32 x i8> @cmpgt_zext_v32i8(<32 x i8> %a, <32 x i8> %b) {
;
   %zext = zext <32 x i1> %cmp to <32 x i8>
   ret <32 x i8> %zext
 }
 
 define <8 x i16> @cmpgt_zext_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; SSE-LABEL: cmpgt_zext_v8i16:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pcmpgtw %xmm1, %xmm0
-; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    psrlw $15, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: cmpgt_zext_v8i16:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $15, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
   %cmp = icmp sgt <8 x i16> %a, %b
   %zext = zext <8 x i1> %cmp to <8 x i16>
   ret <8 x i16> %zext
 }
 
 define <8 x i32> @cmpgt_zext_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: cmpgt_zext_v8i32:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
-; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    psrld $31, %xmm0
 ; SSE-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    psrld $31, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: cmpgt_zext_v8i32:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: cmpgt_zext_v8i32:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $31, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
   %cmp = icmp sgt <8 x i32> %a, %b
   %zext = zext <8 x i1> %cmp to <8 x i32>
   ret <8 x i32> %zext
 }
 
 define <2 x i64> @cmpgt_zext_v2i64(<2 x i64> %a, <2 x i64> %b) {

 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: cmpgt_zext_v2i64:
 ; SSE42:       # BB#0:
 ; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE42-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE42-NEXT:    psrlq $63, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX-LABEL: cmpgt_zext_v2i64:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpsrlq $63, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
   %cmp = icmp sgt <2 x i64> %a, %b
   %zext = zext <2 x i1> %cmp to <2 x i64>
   ret <2 x i64> %zext
 }