[DAG] SimplifyDemandedBits - peek through SHL if we only demand sign bits.

[RKSimon]

Summary

If we're only demanding the (shifted) sign bits of the shift source value, then we can use the value directly.

This handles SimplifyDemandedBits/SimplifyMultipleUseDemandedBits for both ISD::SHL and X86ISD::VSHLI.

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

case ISD::XOR: {
     // If all of the demanded bits are known zero on one side, return the
     // other.
     if (DemandedBits.isSubsetOf(RHSKnown.Zero))
       return Op.getOperand(0);
     if (DemandedBits.isSubsetOf(LHSKnown.Zero))
       return Op.getOperand(1);
     break;
   }
+  case ISD::SHL: {
+    // If we are only demanding sign bits then we can use the shift source
+    // directly.
+    if (const APInt *MaxSA =
+            DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
+      SDValue Op0 = Op.getOperand(0);
+      unsigned ShAmt = MaxSA->getZExtValue();
+      unsigned BitWidth = DemandedBits.getBitWidth();
+      unsigned NumSignBits =
+          DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
+      unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
+      if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
+        return Op0;
+    }
+    break;
+  }
   case ISD::SETCC: {
     SDValue Op0 = Op.getOperand(0);
     SDValue Op1 = Op.getOperand(1);
     ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
     // If (1) we only need the sign-bit, (2) the setcc operands are the same
     // width as the setcc result, and (3) the result of a setcc conforms to 0 or
     // -1, we may be able to bypass the setcc.
     if (DemandedBits.isSignMask() &&
if (const APInt *SA =
       Known.Zero.setLowBits(ShAmt);
 
       // Try shrinking the operation as long as the shift amount will still be
       // in range.
       if ((ShAmt < DemandedBits.getActiveBits()) &&
           ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
         return true;
     }
+
+    // If we are only demanding sign bits then we can use the shift source
+    // directly.
+    if (const APInt *MaxSA =
+            TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
+      unsigned ShAmt = MaxSA->getZExtValue();
+      unsigned NumSignBits =
+          TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
+      unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
+      if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
+        return TLO.CombineTo(Op, Op0);
+    }
     break;
   }
   case ISD::SRL: {
     SDValue Op0 = Op.getOperand(0);
     SDValue Op1 = Op.getOperand(1);
     EVT ShiftVT = Op1.getValueType();
 
     if (const APInt *SA =

llvm/lib/Target/X86/X86ISelLowering.cpp

if (Op0.getOpcode() == X86ISD::VSRLI &&
         unsigned NewOpc = Diff < 0 ? X86ISD::VSRLI : X86ISD::VSHLI;
         SDValue NewShift = TLO.DAG.getNode(
             NewOpc, SDLoc(Op), VT, Op0.getOperand(0),
             TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
         return TLO.CombineTo(Op, NewShift);
       }
     }
 
+    // If we are only demanding sign bits then we can use the shift source directly.
+    unsigned NumSignBits =
+        TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1);
+    unsigned UpperDemandedBits =
+        BitWidth - OriginalDemandedBits.countTrailingZeros();
+    if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
+      return TLO.CombineTo(Op, Op0);
+
     if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
                              TLO, Depth + 1))
       return true;
 
     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
     Known.Zero <<= ShAmt;
     Known.One <<= ShAmt;
 
SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
   case X86ISD::PINSRW: {
     // If we don't demand the inserted element, return the base vector.
     SDValue Vec = Op.getOperand(0);
     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
     MVT VecVT = Vec.getSimpleValueType();
     if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
         !DemandedElts[CIdx->getZExtValue()])
       return Vec;
-     break;
   }
+  case X86ISD::VSHLI: {
+    // If we are only demanding sign bits then we can use the shift source
+    // directly.
+    SDValue Op0 = Op.getOperand(0);
+    unsigned ShAmt = Op.getConstantOperandVal(1);
+    unsigned BitWidth = DemandedBits.getBitWidth();
+    unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
+    unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
+    if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
+      return Op0;
+    break;
+  }
   case X86ISD::VSRAI:
     // iff we only need the sign bit then we can use the source directly.
     // TODO: generalize where we only demand extended signbits.
     if (DemandedBits.isSignMask())
       return Op.getOperand(0);
     break;
   case X86ISD::PCMPGT:
     // icmp sgt(0, R) == ashr(R, BitWidth-1).

llvm/test/CodeGen/AMDGPU/sdiv.ll

 ; GCN-NEXT:    s_mov_b32 s4, s6
 ; GCN-NEXT:    s_mov_b32 s5, s7
 ; GCN-NEXT:    s_mov_b32 s6, s2
 ; GCN-NEXT:    s_mov_b32 s7, s3
 ; GCN-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
 ; GCN-NEXT:    buffer_load_sbyte v1, off, s[4:7], 0 offset:2
 ; GCN-NEXT:    buffer_load_ushort v2, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    buffer_load_sbyte v3, off, s[4:7], 0 offset:6
-; GCN-NEXT:    s_waitcnt vmcnt(2)
-; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v4
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v4
 ; GCN-NEXT:    v_cvt_f32_i32_e32 v0, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GCN-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GCN-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
-; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GCN-NEXT:    v_or_b32_e32 v1, 1, v1
 ; GCN-NEXT:    v_mul_f32_e32 v3, v0, v4
 ; GCN-NEXT:    v_trunc_f32_e32 v3, v3
 ; GCN-NEXT:    v_mad_f32 v0, -v3, v2, v0
 ; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3

 ; TONGA-NEXT:    s_mov_b32 s4, s6
 ; TONGA-NEXT:    s_mov_b32 s5, s7
 ; TONGA-NEXT:    s_mov_b32 s6, s2
 ; TONGA-NEXT:    s_mov_b32 s7, s3
 ; TONGA-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
 ; TONGA-NEXT:    buffer_load_sbyte v1, off, s[4:7], 0 offset:2
 ; TONGA-NEXT:    buffer_load_ushort v2, off, s[4:7], 0 offset:4
 ; TONGA-NEXT:    buffer_load_sbyte v3, off, s[4:7], 0 offset:6
-; TONGA-NEXT:    s_waitcnt vmcnt(2)
-; TONGA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; TONGA-NEXT:    v_or_b32_e32 v0, v0, v1
 ; TONGA-NEXT:    s_waitcnt vmcnt(0)
-; TONGA-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; TONGA-NEXT:    v_or_b32_e32 v2, v2, v3
+; TONGA-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; TONGA-NEXT:    v_or_b32_e32 v2, v2, v4
 ; TONGA-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; TONGA-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; TONGA-NEXT:    v_or_b32_e32 v0, v0, v4
 ; TONGA-NEXT:    v_cvt_f32_i32_e32 v0, v0
+; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; TONGA-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; TONGA-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
-; TONGA-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; TONGA-NEXT:    v_or_b32_e32 v1, 1, v1
 ; TONGA-NEXT:    v_mul_f32_e32 v3, v0, v4
 ; TONGA-NEXT:    v_trunc_f32_e32 v3, v3
 ; TONGA-NEXT:    v_mad_f32 v0, -v3, v2, v0
 ; TONGA-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; TONGA-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
 ; TONGA-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
 ; TONGA-NEXT:    v_add_u32_e32 v0, vcc, v0, v3

 ; GFX9-NEXT:    s_mov_b32 s4, s6
 ; GFX9-NEXT:    s_mov_b32 s5, s7
 ; GFX9-NEXT:    s_mov_b32 s6, s2
 ; GFX9-NEXT:    s_mov_b32 s7, s3
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
 ; GFX9-NEXT:    buffer_load_sbyte v1, off, s[4:7], 0 offset:2
 ; GFX9-NEXT:    buffer_load_ushort v2, off, s[4:7], 0 offset:4
 ; GFX9-NEXT:    buffer_load_sbyte v3, off, s[4:7], 0 offset:6
-; GFX9-NEXT:    s_waitcnt vmcnt(2)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 16, v3
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v4
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v4
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v3
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GFX9-NEXT:    v_or_b32_e32 v1, 1, v1
 ; GFX9-NEXT:    v_mul_f32_e32 v3, v0, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v3
 ; GFX9-NEXT:    v_mad_f32 v0, -v3, v2, v0
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, |v2|
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v0, v4, v0

llvm/test/CodeGen/X86/bitcast-setcc-128.ll

 ; SSE2-SSSE3-NEXT:    orl $-65536, %ecx # imm = 0xFFFF0000
 ; SSE2-SSSE3-NEXT:    movabsq $-4294967296, %rax # imm = 0xFFFFFFFF00000000
 ; SSE2-SSSE3-NEXT:    orq %rcx, %rax
 ; SSE2-SSSE3-NEXT:    retq
 ;
 ; AVX1-LABEL: v16i8_widened_with_ones:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    orl $-65536, %ecx # imm = 0xFFFF0000
 ; AVX1-NEXT:    movabsq $-4294967296, %rax # imm = 0xFFFFFFFF00000000
 ; AVX1-NEXT:    orq %rcx, %rax
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v16i8_widened_with_ones:
 ; AVX2:       # %bb.0: # %entry

llvm/test/CodeGen/X86/funnel-shift.ll

; X64-AVX2-NEXT:    retq
   %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 32, i32 32, i32 32, i32 32>)
   ret <4 x i32> %f
 }
 
 %struct.S = type { [11 x i8], i8 }
 define void @PR45265(i32 %0, %struct.S* nocapture readonly %1) nounwind {
 ; X32-SSE2-LABEL: PR45265:
 ; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    pushl %ebx
 ; X32-SSE2-NEXT:    pushl %edi
 ; X32-SSE2-NEXT:    pushl %esi
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-SSE2-NEXT:    leal (%eax,%eax,2), %edx
 ; X32-SSE2-NEXT:    movzwl 8(%ecx,%edx,4), %esi
 ; X32-SSE2-NEXT:    movsbl 10(%ecx,%edx,4), %edi
-; X32-SSE2-NEXT:    shll $16, %edi
-; X32-SSE2-NEXT:    orl %edi, %esi
+; X32-SSE2-NEXT:    movl %edi, %ebx
+; X32-SSE2-NEXT:    shll $16, %ebx
+; X32-SSE2-NEXT:    orl %esi, %ebx
 ; X32-SSE2-NEXT:    movl 4(%ecx,%edx,4), %ecx
-; X32-SSE2-NEXT:    shrdl $8, %esi, %ecx
+; X32-SSE2-NEXT:    shrdl $8, %ebx, %ecx
 ; X32-SSE2-NEXT:    xorl %eax, %ecx
 ; X32-SSE2-NEXT:    sarl $31, %eax
 ; X32-SSE2-NEXT:    sarl $31, %edi
-; X32-SSE2-NEXT:    shldl $24, %esi, %edi
+; X32-SSE2-NEXT:    shldl $24, %ebx, %edi
 ; X32-SSE2-NEXT:    xorl %eax, %edi
 ; X32-SSE2-NEXT:    orl %edi, %ecx
 ; X32-SSE2-NEXT:    jne .LBB44_1
 ; X32-SSE2-NEXT:  # %bb.2:
 ; X32-SSE2-NEXT:    popl %esi
 ; X32-SSE2-NEXT:    popl %edi
+; X32-SSE2-NEXT:    popl %ebx
 ; X32-SSE2-NEXT:    jmp _Z3foov # TAILCALL
 ; X32-SSE2-NEXT:  .LBB44_1:
 ; X32-SSE2-NEXT:    popl %esi
 ; X32-SSE2-NEXT:    popl %edi
+; X32-SSE2-NEXT:    popl %ebx
 ; X32-SSE2-NEXT:    retl
 ;
 ; X64-AVX2-LABEL: PR45265:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    movslq %edi, %rax
 ; X64-AVX2-NEXT:    leaq (%rax,%rax,2), %rcx
 ; X64-AVX2-NEXT:    movsbq 10(%rsi,%rcx,4), %rdx
 ; X64-AVX2-NEXT:    shlq $16, %rdx

llvm/test/CodeGen/X86/promote-cmp.ll

 ; AVX1-LABEL: PR45808:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm3
 ; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllq $63, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllq $63, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: PR45808:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vpxor {{.*}}(%rip), %ymm2, %ymm2
 ; AVX2-NEXT:    vpsllq $63, %ymm2, %ymm2
 ; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
   %3 = icmp sgt <4 x i64> %0, %1
   %4 = xor <4 x i1> %3, <i1 true, i1 true, i1 false, i1 false>
   %5 = select <4 x i1> %4, <4 x i64> %0, <4 x i64> %1
   ret <4 x i64> %5
 }

llvm/test/CodeGen/X86/sdiv_fix_sat.ll

 ; X64-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    pxor %xmm2, %xmm2
 ; X64-NEXT:    pcmpgtd %xmm0, %xmm2
 ; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; X64-NEXT:    paddq %xmm0, %xmm0
 ; X64-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    movq %xmm0, %rbx
-; X64-NEXT:    movq %rbx, %r15
-; X64-NEXT:    sarq $63, %r15
-; X64-NEXT:    movq %r15, %r12
-; X64-NEXT:    shldq $31, %rbx, %r12
+; X64-NEXT:    movq %rbx, %rbp
+; X64-NEXT:    sarq $63, %rbp
+; X64-NEXT:    shldq $31, %rbx, %rbp
 ; X64-NEXT:    pxor %xmm0, %xmm0
 ; X64-NEXT:    pcmpgtd %xmm1, %xmm0
 ; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X64-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    movq %xmm1, %rdx
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %r14
-; X64-NEXT:    sarq $63, %r14
-; X64-NEXT:    shlq $31, %rbx
-; X64-NEXT:    movq %rbx, %rdi
-; X64-NEXT:    movq %r12, %rsi
-; X64-NEXT:    movq %r14, %rcx
+; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    sarq $63, %r15
+; X64-NEXT:    movq %rbx, %r12
+; X64-NEXT:    shlq $31, %r12
+; X64-NEXT:    movq %r12, %rdi
+; X64-NEXT:    movq %rbp, %rsi
+; X64-NEXT:    movq %r15, %rcx
 ; X64-NEXT:    callq __divti3
 ; X64-NEXT:    movq %rax, %r13
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    movq %rdx, %r14
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    subq $1, %r13
-; X64-NEXT:    sbbq $0, %rbp
-; X64-NEXT:    movq %rbx, %rdi
-; X64-NEXT:    movq %r12, %rsi
+; X64-NEXT:    sbbq $0, %r14
+; X64-NEXT:    shrq $63, %rbx
+; X64-NEXT:    xorl %r15d, %ebx
+; X64-NEXT:    movq %r12, %rdi
+; X64-NEXT:    movq %rbp, %rsi
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; X64-NEXT:    movq %r14, %rcx
+; X64-NEXT:    movq %r15, %rcx
 ; X64-NEXT:    callq __modti3
 ; X64-NEXT:    orq %rax, %rdx
 ; X64-NEXT:    setne %al
-; X64-NEXT:    shlq $31, %r15
-; X64-NEXT:    shrq $63, %r15
-; X64-NEXT:    xorl %r14d, %r15d
-; X64-NEXT:    testb %r15b, %al
-; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
+; X64-NEXT:    testb %bl, %al
+; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
 ; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
 ; X64-NEXT:    movl $4294967295, %edx # imm = 0xFFFFFFFF
 ; X64-NEXT:    cmpq %rdx, %r13
 ; X64-NEXT:    movl $4294967295, %eax # imm = 0xFFFFFFFF
 ; X64-NEXT:    cmovbq %r13, %rax
 ; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    testq %rbp, %rbp
+; X64-NEXT:    testq %r14, %r14
 ; X64-NEXT:    cmovnsq %rdx, %r13
 ; X64-NEXT:    cmoveq %rax, %r13
-; X64-NEXT:    cmovnsq %rcx, %rbp
+; X64-NEXT:    cmovnsq %rcx, %r14
 ; X64-NEXT:    movabsq $-4294967296, %rcx # imm = 0xFFFFFFFF00000000
 ; X64-NEXT:    cmpq %rcx, %r13
 ; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    cmovaq %r13, %rax
-; X64-NEXT:    testq %rbp, %rbp
+; X64-NEXT:    testq %r14, %r14
 ; X64-NEXT:    cmovsq %rcx, %r13
-; X64-NEXT:    cmpq $-1, %rbp
+; X64-NEXT:    cmpq $-1, %r14
 ; X64-NEXT:    cmoveq %rax, %r13
 ; X64-NEXT:    movq %r13, %xmm0
 ; X64-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    # xmm0 = mem[2,3,0,1]
-; X64-NEXT:    movq %xmm0, %rbp
-; X64-NEXT:    movq %rbp, %rbx
-; X64-NEXT:    sarq $63, %rbx
+; X64-NEXT:    movq %xmm0, %rbx
 ; X64-NEXT:    movq %rbx, %r13
-; X64-NEXT:    shldq $31, %rbp, %r13
+; X64-NEXT:    sarq $63, %r13
+; X64-NEXT:    shldq $31, %rbx, %r13
 ; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    # xmm0 = mem[2,3,0,1]
 ; X64-NEXT:    movq %xmm0, %rdx
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %r14
-; X64-NEXT:    sarq $63, %r14
-; X64-NEXT:    shlq $31, %rbp
-; X64-NEXT:    movq %rbp, %rdi
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    sarq $63, %rbp
+; X64-NEXT:    movq %rbx, %r15
+; X64-NEXT:    shlq $31, %r15
+; X64-NEXT:    movq %r15, %rdi
 ; X64-NEXT:    movq %r13, %rsi
-; X64-NEXT:    movq %r14, %rcx
+; X64-NEXT:    movq %rbp, %rcx
 ; X64-NEXT:    callq __divti3
 ; X64-NEXT:    movq %rax, %r12
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rdx, %r14
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    subq $1, %r12
-; X64-NEXT:    sbbq $0, %r15
-; X64-NEXT:    movq %rbp, %rdi
+; X64-NEXT:    sbbq $0, %r14
+; X64-NEXT:    shrq $63, %rbx
+; X64-NEXT:    xorl %ebp, %ebx
+; X64-NEXT:    movq %r15, %rdi
 ; X64-NEXT:    movq %r13, %rsi
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; X64-NEXT:    movq %r14, %rcx
+; X64-NEXT:    movq %rbp, %rcx
 ; X64-NEXT:    callq __modti3
 ; X64-NEXT:    orq %rax, %rdx
 ; X64-NEXT:    setne %al
-; X64-NEXT:    shlq $31, %rbx
-; X64-NEXT:    shrq $63, %rbx
-; X64-NEXT:    xorl %r14d, %ebx
 ; X64-NEXT:    testb %bl, %al
-; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
+; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
 ; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
 ; X64-NEXT:    movl $4294967295, %ecx # imm = 0xFFFFFFFF
 ; X64-NEXT:    cmpq %rcx, %r12
 ; X64-NEXT:    movl $4294967295, %eax # imm = 0xFFFFFFFF
 ; X64-NEXT:    cmovbq %r12, %rax
-; X64-NEXT:    testq %r15, %r15
+; X64-NEXT:    testq %r14, %r14
 ; X64-NEXT:    cmovnsq %rcx, %r12
 ; X64-NEXT:    cmoveq %rax, %r12
 ; X64-NEXT:    movl $0, %eax
-; X64-NEXT:    cmovnsq %rax, %r15
+; X64-NEXT:    cmovnsq %rax, %r14
 ; X64-NEXT:    movabsq $-4294967296, %rcx # imm = 0xFFFFFFFF00000000
 ; X64-NEXT:    cmpq %rcx, %r12
 ; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    cmovaq %r12, %rax
-; X64-NEXT:    testq %r15, %r15
+; X64-NEXT:    testq %r14, %r14
 ; X64-NEXT:    cmovsq %rcx, %r12
-; X64-NEXT:    cmpq $-1, %r15
+; X64-NEXT:    cmpq $-1, %r14
 ; X64-NEXT:    cmoveq %rax, %r12
 ; X64-NEXT:    movq %r12, %xmm0
 ; X64-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; X64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; X64-NEXT:    psrlq $1, %xmm1
 ; X64-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
 ; X64-NEXT:    # xmm1 = mem[2,3,0,1]
 ; X64-NEXT:    pxor %xmm0, %xmm0
 ; X64-NEXT:    pcmpgtd %xmm1, %xmm0
 ; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X64-NEXT:    paddq %xmm1, %xmm1
 ; X64-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; X64-NEXT:    movq %xmm1, %rbp
-; X64-NEXT:    movq %rbp, %r14
-; X64-NEXT:    sarq $63, %r14
-; X64-NEXT:    movq %r14, %r13
-; X64-NEXT:    shldq $31, %rbp, %r13
+; X64-NEXT:    movq %xmm1, %rbx
+; X64-NEXT:    movq %rbx, %r12
+; X64-NEXT:    sarq $63, %r12
+; X64-NEXT:    shldq $31, %rbx, %r12
 ; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
 ; X64-NEXT:    # xmm1 = mem[2,3,0,1]
 ; X64-NEXT:    pxor %xmm0, %xmm0
 ; X64-NEXT:    pcmpgtd %xmm1, %xmm0
 ; X64-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; X64-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    movq %xmm1, %rdx
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %rbx
-; X64-NEXT:    sarq $63, %rbx
-; X64-NEXT:    shlq $31, %rbp
-; X64-NEXT:    movq %rbp, %rdi
-; X64-NEXT:    movq %r13, %rsi
-; X64-NEXT:    movq %rbx, %rcx
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    sarq $63, %rbp
+; X64-NEXT:    movq %rbx, %r15
+; X64-NEXT:    shlq $31, %r15
+; X64-NEXT:    movq %r15, %rdi
+; X64-NEXT:    movq %r12, %rsi
+; X64-NEXT:    movq %rbp, %rcx
 ; X64-NEXT:    callq __divti3
-; X64-NEXT:    movq %rax, %r12
+; X64-NEXT:    movq %rax, %r13
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rdx, %r14
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    subq $1, %r12
-; X64-NEXT:    sbbq $0, %r15
-; X64-NEXT:    movq %rbp, %rdi
-; X64-NEXT:    movq %r13, %rsi
+; X64-NEXT:    subq $1, %r13
+; X64-NEXT:    sbbq $0, %r14
+; X64-NEXT:    shrq $63, %rbx
+; X64-NEXT:    xorl %ebp, %ebx
+; X64-NEXT:    movq %r15, %rdi
+; X64-NEXT:    movq %r12, %rsi
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; X64-NEXT:    movq %rbx, %rcx
+; X64-NEXT:    movq %rbp, %rcx
 ; X64-NEXT:    callq __modti3
 ; X64-NEXT:    orq %rax, %rdx
 ; X64-NEXT:    setne %al
-; X64-NEXT:    shlq $31, %r14
-; X64-NEXT:    shrq $63, %r14
-; X64-NEXT:    xorl %ebx, %r14d
-; X64-NEXT:    testb %r14b, %al
-; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
-; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
+; X64-NEXT:    testb %bl, %al
+; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
+; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
 ; X64-NEXT:    movl $4294967295, %ecx # imm = 0xFFFFFFFF
-; X64-NEXT:    cmpq %rcx, %r12
+; X64-NEXT:    cmpq %rcx, %r13
 ; X64-NEXT:    movl $4294967295, %eax # imm = 0xFFFFFFFF
-; X64-NEXT:    cmovbq %r12, %rax
-; X64-NEXT:    testq %r15, %r15
-; X64-NEXT:    cmovnsq %rcx, %r12
-; X64-NEXT:    cmoveq %rax, %r12
+; X64-NEXT:    cmovbq %r13, %rax
+; X64-NEXT:    testq %r14, %r14
+; X64-NEXT:    cmovnsq %rcx, %r13
+; X64-NEXT:    cmoveq %rax, %r13
 ; X64-NEXT:    movl $0, %eax
-; X64-NEXT:    cmovnsq %rax, %r15
+; X64-NEXT:    cmovnsq %rax, %r14
 ; X64-NEXT:    movabsq $-4294967296, %rcx # imm = 0xFFFFFFFF00000000
-; X64-NEXT:    cmpq %rcx, %r12
+; X64-NEXT:    cmpq %rcx, %r13
 ; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    cmovaq %r12, %rax
-; X64-NEXT:    testq %r15, %r15
-; X64-NEXT:    cmovsq %rcx, %r12
-; X64-NEXT:    cmpq $-1, %r15
-; X64-NEXT:    cmoveq %rax, %r12
-; X64-NEXT:    movq %r12, %xmm0
+; X64-NEXT:    cmovaq %r13, %rax
+; X64-NEXT:    testq %r14, %r14
+; X64-NEXT:    cmovsq %rcx, %r13
+; X64-NEXT:    cmpq $-1, %r14
+; X64-NEXT:    cmoveq %rax, %r13
+; X64-NEXT:    movq %r13, %xmm0
 ; X64-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    # xmm0 = mem[2,3,0,1]
-; X64-NEXT:    movq %xmm0, %rbp
-; X64-NEXT:    movq %rbp, %rbx
-; X64-NEXT:    sarq $63, %rbx
+; X64-NEXT:    movq %xmm0, %rbx
 ; X64-NEXT:    movq %rbx, %r13
-; X64-NEXT:    shldq $31, %rbp, %r13
+; X64-NEXT:    sarq $63, %r13
+; X64-NEXT:    shldq $31, %rbx, %r13
 ; X64-NEXT:    pshufd $78, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    # xmm0 = mem[2,3,0,1]
 ; X64-NEXT:    movq %xmm0, %rdx
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %r14
-; X64-NEXT:    sarq $63, %r14
-; X64-NEXT:    shlq $31, %rbp
-; X64-NEXT:    movq %rbp, %rdi
+; X64-NEXT:    movq %rdx, %rbp
+; X64-NEXT:    sarq $63, %rbp
+; X64-NEXT:    movq %rbx, %r15
+; X64-NEXT:    shlq $31, %r15
+; X64-NEXT:    movq %r15, %rdi
 ; X64-NEXT:    movq %r13, %rsi
-; X64-NEXT:    movq %r14, %rcx
+; X64-NEXT:    movq %rbp, %rcx
 ; X64-NEXT:    callq __divti3
 ; X64-NEXT:    movq %rax, %r12
 ; X64-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; X64-NEXT:    movq %rdx, %r15
+; X64-NEXT:    movq %rdx, %r14
 ; X64-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    subq $1, %r12
-; X64-NEXT:    sbbq $0, %r15
-; X64-NEXT:    movq %rbp, %rdi
+; X64-NEXT:    sbbq $0, %r14
+; X64-NEXT:    shrq $63, %rbx
+; X64-NEXT:    xorl %ebp, %ebx
+; X64-NEXT:    movq %r15, %rdi
 ; X64-NEXT:    movq %r13, %rsi
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; X64-NEXT:    movq %r14, %rcx
+; X64-NEXT:    movq %rbp, %rcx
 ; X64-NEXT:    callq __modti3
 ; X64-NEXT:    orq %rax, %rdx
 ; X64-NEXT:    setne %al
-; X64-NEXT:    shlq $31, %rbx
-; X64-NEXT:    shrq $63, %rbx
-; X64-NEXT:    xorl %r14d, %ebx
 ; X64-NEXT:    testb %bl, %al
-; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
+; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
 ; X64-NEXT:    cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
 ; X64-NEXT:    movl $4294967295, %ecx # imm = 0xFFFFFFFF
 ; X64-NEXT:    cmpq %rcx, %r12
 ; X64-NEXT:    movl $4294967295, %eax # imm = 0xFFFFFFFF
 ; X64-NEXT:    cmovbq %r12, %rax
-; X64-NEXT:    testq %r15, %r15
+; X64-NEXT:    testq %r14, %r14
 ; X64-NEXT:    cmovnsq %rcx, %r12
 ; X64-NEXT:    cmoveq %rax, %r12
 ; X64-NEXT:    movl $0, %eax
-; X64-NEXT:    cmovnsq %rax, %r15
+; X64-NEXT:    cmovnsq %rax, %r14
 ; X64-NEXT:    movabsq $-4294967296, %rcx # imm = 0xFFFFFFFF00000000
 ; X64-NEXT:    cmpq %rcx, %r12
 ; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    cmovaq %r12, %rax
-; X64-NEXT:    testq %r15, %r15
+; X64-NEXT:    testq %r14, %r14
 ; X64-NEXT:    cmovsq %rcx, %r12
-; X64-NEXT:    cmpq $-1, %r15
+; X64-NEXT:    cmpq $-1, %r14
 ; X64-NEXT:    cmoveq %rax, %r12
 ; X64-NEXT:    movq %r12, %xmm0
 ; X64-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; X64-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; X64-NEXT:    psrlq $1, %xmm1
 ; X64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; X64-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; X64-NEXT:    addq $104, %rsp

llvm/test/CodeGen/X86/vector-mulfix-legalize.ll

 ; CHECK-NEXT:    movl $32767, %r8d # imm = 0x7FFF
 ; CHECK-NEXT:    cmovgl %r8d, %edx
 ; CHECK-NEXT:    cmpl $-16384, %ecx # imm = 0xC000
 ; CHECK-NEXT:    movl $32768, %ecx # imm = 0x8000
 ; CHECK-NEXT:    cmovll %ecx, %edx
 ; CHECK-NEXT:    pextrw $1, %xmm0, %esi
 ; CHECK-NEXT:    movswl %si, %edi
 ; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    shrl $15, %eax
+; CHECK-NEXT:    shrl $16, %eax
 ; CHECK-NEXT:    leal (%rdi,%rdi), %esi
 ; CHECK-NEXT:    shrdw $15, %ax, %si
 ; CHECK-NEXT:    sarl $15, %edi
 ; CHECK-NEXT:    cmpl $16383, %edi # imm = 0x3FFF
 ; CHECK-NEXT:    cmovgl %r8d, %esi
 ; CHECK-NEXT:    cmpl $-16384, %edi # imm = 0xC000
 ; CHECK-NEXT:    cmovll %ecx, %esi
 ; CHECK-NEXT:    movd %xmm0, %eax

llvm/test/CodeGen/X86/vector-reduce-and-bool.ll

 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    cmpw $-1, %ax
 ; AVX1-NEXT:    sete %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: icmp_v32i16_v32i1:
 ; AVX2:       # %bb.0:

llvm/test/CodeGen/X86/vector-reduce-or-bool.ll

 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpacksswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    testw %ax, %ax
 ; AVX1-NEXT:    setne %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: icmp_v32i16_v32i1:
 ; AVX2:       # %bb.0:

 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpor %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    testw %ax, %ax
 ; AVX1-NEXT:    setne %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: icmp_v64i8_v64i1:
 ; AVX2:       # %bb.0: