[ValueTracking] recognize obfuscated variants of umin/umax

[spatel]

Summary

The smallest tests that I could find to expose this are codegen tests (because SelectionDAGBuilder::visitSelect() uses matchSelectPattern to create UMAX/UMIN nodes), so I've semi-arbitrarily chosen x86 as the target for the regression tests.

If these were written as unsigned compares in IR, InstCombine canonicalizes the unsigned compares to signed compares. Ie, running the optimizer pessimizes the codegen for this case on trunk:

define <4 x i32> @umax_vec(<4 x i32> %x) {
  %cmp = icmp ugt <4 x i32> %x, <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
  %sel = select <4 x i1> %cmp, <4 x i32> %x, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
  ret <4 x i32> %sel
}

$ ./opt umax.ll -S | ./llc -o - -mattr=avx

vpmaxud	LCPI0_0(%rip), %xmm0, %xmm0

$ ./opt -instcombine umax.ll -S | ./llc -o - -mattr=avx

vpxor	%xmm1, %xmm1, %xmm1
vpcmpgtd	%xmm0, %xmm1, %xmm1
vmovaps	LCPI0_0(%rip), %xmm2    ## xmm2 = [2147483647,2147483647,2147483647,2147483647]
vblendvps	%xmm1, %xmm0, %xmm2, %xmm0

[spatel]

Patch updated - rebase after rL285499 , but no difference in intent/functionality:

1. Insert the unsigned min/max pattern matching block next to the updated signed min/max block.
2. Changed name of test file since we have signed and unsigned tests now.

[spatel]

Ping.

[efriedma]

LGTM.

llvm/trunk/lib/Analysis/ValueTracking.cpp

if ((CmpLHS == TrueVal && match(FalseVal, m_Neg(m_Specific(CmpLHS)))) ||
 
       // ABS(X) ==> (X <s 0) ? -X : X and (X <s 1) ? -X : X
       // NABS(X) ==> (X <s 0) ? X : -X and (X <s 1) ? X : -X
       if (Pred == ICmpInst::ICMP_SLT && (*C1 == 0 || *C1 == 1)) {
         return {(CmpLHS == FalseVal) ? SPF_ABS : SPF_NABS, SPNB_NA, false};
       }
     }
 
+    // An unsigned min/max can be written with a signed compare.
+    const APInt *C2;
+    if ((CmpLHS == TrueVal && match(FalseVal, m_APInt(C2))) ||
+        (CmpLHS == FalseVal && match(TrueVal, m_APInt(C2)))) {
+      // Is the sign bit set?
+      // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
+      // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
+      if (Pred == CmpInst::ICMP_SLT && *C1 == 0 && C2->isMaxSignedValue()) {
+        LHS = TrueVal;
+        RHS = FalseVal;
+        return {CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false};
+      }
+
+      // Is the sign bit clear?
+      // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
+      // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
+      if (Pred == CmpInst::ICMP_SGT && C1->isAllOnesValue() &&
+          C2->isMinSignedValue()) {
+        LHS = TrueVal;
+        RHS = FalseVal;
+        return {CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false};
+      }
+    }
+
+    // Look through 'not' ops to find disguised signed min/max.
     // (X >s C) ? ~X : ~C ==> (~X <s ~C) ? ~X : ~C ==> SMIN(~X, ~C)
     // (X <s C) ? ~X : ~C ==> (~X >s ~C) ? ~X : ~C ==> SMAX(~X, ~C)
-    const APInt *C2;
     if (match(TrueVal, m_Not(m_Specific(CmpLHS))) &&
         match(FalseVal, m_APInt(C2)) && ~(*C1) == *C2 &&
         (Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SLT)) {
       LHS = TrueVal;
       RHS = FalseVal;
       return {Pred == CmpInst::ICMP_SGT ? SPF_SMIN : SPF_SMAX, SPNB_NA, false};
     }
 

llvm/trunk/test/CodeGen/X86/vec_minmax_match.ll

 ; CHECK-NEXT:    retq
 ;
   %not_x = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
   %cmp = icmp sgt <4 x i32> %x, zeroinitializer
   %sel = select <4 x i1> %cmp, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %not_x
   ret <4 x i32> %sel
 }
 
-; FIXME: These are unsigned min/max ops.
-
 define <4 x i32> @umax_vec1(<4 x i32> %x) {
 ; CHECK-LABEL: umax_vec1:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm1
-; CHECK-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
-; CHECK-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    vpmaxud {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
 ;
   %cmp = icmp slt <4 x i32> %x, zeroinitializer
   %sel = select <4 x i1> %cmp, <4 x i32> %x, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
   ret <4 x i32> %sel
 }
 
 define <4 x i32> @umax_vec2(<4 x i32> %x) {
 ; CHECK-LABEL: umax_vec2:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1
-; CHECK-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    vpmaxud {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
 ;
   %cmp = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
   %sel = select <4 x i1> %cmp, <4 x i32> <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>, <4 x i32> %x
   ret <4 x i32> %sel
 }
 
 define <4 x i32> @umin_vec1(<4 x i32> %x) {
 ; CHECK-LABEL: umin_vec1:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm1
-; CHECK-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm0, %xmm0
+; CHECK-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
 ;
   %cmp = icmp slt <4 x i32> %x, zeroinitializer
   %sel = select <4 x i1> %cmp, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, <4 x i32> %x
   ret <4 x i32> %sel
 }
 
 define <4 x i32> @umin_vec2(<4 x i32> %x) {
 ; CHECK-LABEL: umin_vec2:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1
-; CHECK-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
-; CHECK-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
 ;
   %cmp = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
   %sel = select <4 x i1> %cmp, <4 x i32> %x, <4 x i32> <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
   ret <4 x i32> %sel
 }