[x86] eliminate more sign-bit tests with vector select

[spatel]

Summary

vselect (pcmpgt 0, X), Y, Z --> shrunkblend X, Y, Z

This shortcoming was noted in D47330, and the test diffs show we already had other examples where we failed to fold to a SHRUNKBLEND:

/// Dynamic (non-constant condition) vector blend where only the sign bits
/// of the condition elements are used. This is used to enforce that the
/// condition mask is not valid for generic VSELECT optimizations.

[RKSimon]

test/CodeGen/X86/vsel-cmp-load.ll

252

This can go

260

How come this folds but the AVX1 case in slt_zero above doesn't?

[spatel]

Patch updated:
Remove a stale FIXME comment from a test.

[spatel]

test/CodeGen/X86/vsel-cmp-load.ll

260

AVX1 is more complicated due to ISA limitations, so I was planning to catch that one next. There, we've split the PCMPGT into halves, so I'll need to match a pattern with a concat:

    t41: v4i32 = X86ISD::PCMPGT t37, t32
        t31: v8i16 = vector_shuffle<4,5,6,7,u,u,u,u> t28, undef:v8i16
      t33: v4i32 = sign_extend_vector_inreg t31
    t42: v4i32 = X86ISD::PCMPGT t37, t33
  t40: v8i32 = concat_vectors t41, t42
  t4: v8i32,ch = CopyFromReg t0, Register:v8i32 %1
  t6: v8i32,ch = CopyFromReg t0, Register:v8i32 %2
t23: v8i32 = vselect t40, t4, t6

[spatel]

test/CodeGen/X86/vsel-cmp-load.ll

260

Or probably easier - we match the pattern after type legalization, but before vector op legalization:

    t21: v8i32 = BUILD_VECTOR Constant:i32<0>, Constant:i32<0>, Constant:i32<0>, Constant:i32<0>, Constant:i32<0>, Constant:i32<0>, Constant:i32<0>, Constant:i32<0>
  t22: v8i32 = setcc t24, t21, setlt:ch
  t4: v8i32,ch = CopyFromReg t0, Register:v8i32 %1
  t6: v8i32,ch = CopyFromReg t0, Register:v8i32 %2
t23: v8i32 = vselect t22, t4, t6

[spatel]

test/CodeGen/X86/vsel-cmp-load.ll

260

See D48078 for an implementation of that suggestion.

[spatel]

Abandoning - we do better by matching setcc+vselect earlier.

lib/Target/X86/X86ISelLowering.cpp

if (!FalseC->isNullValue())
       R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
 
     return R;
   }
 
   return SDValue();
 }
 
+/// Try to convert a vector select into a SHRUNKBLEND and eliminate a compare
+/// that it only testing the sign-bits of its operand.
+static SDValue combineVSelectSignBitCmp(SDNode *N, SelectionDAG &DAG) {
+  SDValue Cond = N->getOperand(0);
+  if (Cond.getOpcode() != X86ISD::PCMPGT)
+    return SDValue();
+
+  assert(N->getOpcode() == ISD::VSELECT && "PCMPGT with scalar select?");
+  SDValue Cond0 = Cond.getOperand(0);
+  if (!ISD::isBuildVectorAllZeros(Cond0.getNode()))
+    return SDValue();
+
+  EVT CondOpVT = Cond0.getValueType();
+  EVT VT = N->getValueType(0);
+  if (CondOpVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
+    return SDValue();
+
+  // vselect (pcmpgt 0, X), Y, Z --> shrunkblend X, Y, Z
+  SDValue Cond1 = Cond.getOperand(1);
+  return DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(N), VT, Cond1, N->getOperand(1),
+                     N->getOperand(2));
+}
+
 /// Do target-specific dag combines on SELECT and VSELECT nodes.
 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
                              TargetLowering::DAGCombinerInfo &DCI,
                              const X86Subtarget &Subtarget) {
   SDLoc DL(N);
   SDValue Cond = N->getOperand(0);
   // Get the LHS/RHS of the select.
   SDValue LHS = N->getOperand(1);
static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
   // Custom action for SELECT MMX
   if (VT == MVT::x86mmx) {
     LHS = DAG.getBitcast(MVT::i64, LHS);
     RHS = DAG.getBitcast(MVT::i64, RHS);
     SDValue newSelect = DAG.getNode(ISD::SELECT, DL, MVT::i64, Cond, LHS, RHS);
     return DAG.getBitcast(VT, newSelect);
   }
 
+  if (SDValue V = combineVSelectSignBitCmp(N, DAG))
+    return V;
+
   return SDValue();
 }
 
 /// Combine:
 ///   (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S)
 /// to:
 ///   (brcond/cmov/setcc .., (LADD x, 1), COND_LE)
 /// i.e., reusing the EFLAGS produced by the LOCKed instruction.

test/CodeGen/X86/vsel-cmp-load.ll

 ; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: slt_zero:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovsxbd (%rdi), %ymm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: slt_zero:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmovsxbw (%rdi), %xmm2
 ; AVX512-NEXT:    vpmovw2m %xmm2, %k1
 ; AVX512-NEXT:    vpblendmd %ymm0, %ymm1, %ymm0 {%k1}

 ; AVX512-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <4 x i8>, <4 x i8>* %p
   %cmp = icmp sgt <4 x i8> %load, zeroinitializer
   %sel = select <4 x i1> %cmp, <4 x double> %x, <4 x double> %y
   ret <4 x double> %sel
 }
 
 ; FIXME: The compare with 0 for AVX2 should be eliminated.
 
 define <8 x float> @slt_zero_fp_select(<8 x i16>* %p, <8 x float> %x, <8 x float> %y) {
 ; AVX1-LABEL: slt_zero_fp_select:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxwd 8(%rdi), %xmm2
 ; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: slt_zero_fp_select:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovsxwd (%rdi), %ymm2
-; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: slt_zero_fp_select:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vpcmpgtw (%rdi), %xmm2, %k1
 ; AVX512-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1}
 ; AVX512-NEXT:    retq
   %load = load <8 x i16>, <8 x i16>* %p
   %cmp = icmp slt <8 x i16> %load, zeroinitializer
   %sel = select <8 x i1> %cmp, <8 x float> %x, <8 x float> %y
   ret <8 x float> %sel
 }
 

test/CodeGen/X86/vselect-pcmp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx       | FileCheck %s --check-prefix=AVX --check-prefix=AVX12F --check-prefix=AVX12 --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2      | FileCheck %s --check-prefix=AVX --check-prefix=AVX12F --check-prefix=AVX12 --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f   | FileCheck %s --check-prefix=AVX --check-prefix=AVX12F --check-prefix=AVX512 --check-prefix=AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl  | FileCheck %s --check-prefix=AVX                       --check-prefix=AVX512 --check-prefix=AVX512VL
 
 ; The condition vector for BLENDV* only cares about the sign bit of each element.
 ; So in these tests, if we generate BLENDV*, we should be able to remove the redundant cmp op.
 
 ; Test 128-bit vectors for all legal element types.
 
-; FIXME: Why didn't AVX-512 optimize too?
-
 define <16 x i8> @signbit_sel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %mask) {
-; AVX12-LABEL: signbit_sel_v16i8:
-; AVX12:       # %bb.0:
-; AVX12-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
-; AVX12-NEXT:    retq
-;
-; AVX512-LABEL: signbit_sel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm2
-; AVX512-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: signbit_sel_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
   %tr = icmp slt <16 x i8> %mask, zeroinitializer
   %z = select <16 x i1> %tr, <16 x i8> %x, <16 x i8> %y
   ret <16 x i8> %z
 }
 
 ; Sorry 16-bit, you're not important enough to support?
 
 define <8 x i16> @signbit_sel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %mask) {

 ;
 ; AVX2-LABEL: signbit_sel_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: signbit_sel_v32i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
 ; AVX512-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %tr = icmp slt <32 x i8> %mask, zeroinitializer
   %z = select <32 x i1> %tr, <32 x i8> %x, <32 x i8> %y
   ret <32 x i8> %z
 }
 
 ; Sorry 16-bit, you'll never be important enough to support?

test/CodeGen/X86/vselect.ll

 ; AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %x = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
   %y = select <2 x i1> %cond, <2 x i64> %c, <2 x i64> %d
   %z = add <2 x i64> %x, %y
   ret <2 x i64> %z
 }
 
-; Similar to above, but condition has a use that isn't a condition of a vselect so we can't optimize.
+; Similar to above, but condition has a use that isn't a condition of a vselect.
+; The blend does not require a cmp, but we may produce one anyway for the math op.
 define <2 x i64> @shrunkblend_nonvselectuse(<2 x i1> %cond, <2 x i64> %a, <2 x i64> %b, <2 x i64> %c, <2 x i64> %d) {
 ; SSE2-LABEL: shrunkblend_nonvselectuse:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllq $63, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
 ; SSE2-NEXT:    pandn %xmm2, %xmm0

 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    paddq %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: shrunkblend_nonvselectuse:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsllq $63, %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
 ; AVX-NEXT:    vblendvpd %xmm0, %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %x = select <2 x i1> %cond, <2 x i64> %a, <2 x i64> %b
   %y = sext <2 x i1> %cond to <2 x i64>
   %z = add <2 x i64> %x, %y
   ret <2 x i64> %z
 }