[InstCombine] Do not exercise nested max/min pattern on abs

[anna]

Summary

This is a fix for assertion failure in
getInverseMinMaxSelectPattern when ABS is passed in as a select pattern.

We should not be invoking the simplification rule for
ABS(MIN(~ x,y))) or ABS(MAX(~x,y)) combinations.

Added a test case which would cause an assertion failure without the patch.

[sanjoy]

LGTM

lib/Transforms/InstCombine/InstCombineSelect.cpp

Instruction *InstCombiner::foldSPFofSPF(Instruction *Inner,
   // MIN(MIN(~A, ~B), ~C) == ~MAX(MAX(A, B), C)
   // MIN(MAX(~A, ~B), ~C) == ~MAX(MIN(A, B), C)
   // MAX(MIN(~A, ~B), ~C) == ~MIN(MAX(A, B), C)
   // MAX(MAX(~A, ~B), ~C) == ~MIN(MIN(A, B), C)
   //
   // This transform is performance neutral if we can elide at least one xor from
   // the set of three operands, since we'll be tacking on an xor at the very
   // end.
-  if (IsFreeOrProfitableToInvert(A, NotA, ElidesXor) &&
+  if (SelectPatternResult::isMinOrMax(SPF1) &&
+      SelectPatternResult::isMinOrMax(SPF2) &&
+      IsFreeOrProfitableToInvert(A, NotA, ElidesXor) &&
       IsFreeOrProfitableToInvert(B, NotB, ElidesXor) &&
       IsFreeOrProfitableToInvert(C, NotC, ElidesXor) && ElidesXor) {
     if (!NotA)
       NotA = Builder->CreateNot(A);
     if (!NotB)
       NotB = Builder->CreateNot(B);
     if (!NotC)
       NotC = Builder->CreateNot(C);

test/Transforms/InstCombine/max-of-nots.ll

;
   %xor_y = xor i32 %y, -1
   %s0 = select i1 %c0, i32 %xor_y, i32 -1
   %xor_x = xor i32 %x, -1
   %c1 = icmp slt i32 %s0, %xor_x
   %smax96 = select i1 %c1, i32 %xor_x, i32 %s0
   ret i32 %smax96
 }
 
+ ; negative test case (i.e. can not simplify) : ABS(MIN(NOT x,y))
+define i32 @abs_of_min_of_not(i32 %x, i32 %y) {
+; CHECK-LABEL: @abs_of_min_of_not(
+; CHECK-NEXT: xor
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp sge
+; CHECK-NEXT: select
+; CHECK-NEXT: icmp sgt
+; CHECK-NEXT: sub
+; CHECK-NEXT: select
+; CHECK-NEXT: ret
+
+  %xord = xor i32 %x, -1
+  %yadd = add i32 %y, 2
+  %cond.i = icmp sge i32 %yadd, %xord
+  %min = select i1 %cond.i, i32 %xord, i32 %yadd
+  %cmp2 = icmp sgt i32 %min, -1
+  %sub = sub i32 0, %min
+  %abs = select i1 %cmp2, i32 %min, i32 %sub
+  ret i32  %abs
+}
+
 define <2 x i32> @max_of_nots_vec(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @max_of_nots_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i32> %y, zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> %y, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <2 x i32> [[TMP2]], %x
 ; CHECK-NEXT:    [[TMP4:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP2]], <2 x i32> %x
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i32> [[TMP4]], <i32 -1, i32 -1>
 ; CHECK-NEXT:    ret <2 x i32> [[TMP5]]