[SLP] Fix crash when matching associative reduction for integer min/max.

[vdmitrie]

Summary

Associative reduction matcher in SLP begins with select instruction but when
it reached call to llvm.umax (or alike) via def-use chain the latter also matched
as UMax kind. The routine's later code assumes matched instruction to be a select
and thus it merely died on the first encountered cast that did not fit.

[spatel]

LGTM - I'm looking at what it will take to allow matching intrinsics, but we should fix the crash first.

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

static RecurKind getRdxKind(Instruction *I) {
     if (match(I, m_FMul(m_Value(), m_Value())))
       return RecurKind::FMul;
 
     if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
       return RecurKind::FMax;
     if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
       return RecurKind::FMin;
 
+
+    if (auto *Select = dyn_cast<SelectInst>(I)) {
+      // These would also match llvm.{u,s}{min,max} intrinsic call
+      // if were not guarded by the SelectInst check above.
       if (match(I, m_SMax(m_Value(), m_Value())))
         return RecurKind::SMax;
       if (match(I, m_SMin(m_Value(), m_Value())))
         return RecurKind::SMin;
       if (match(I, m_UMax(m_Value(), m_Value())))
         return RecurKind::UMax;
       if (match(I, m_UMin(m_Value(), m_Value())))
         return RecurKind::UMin;
-
-    if (auto *Select = dyn_cast<SelectInst>(I)) {
       // Try harder: look for min/max pattern based on instructions producing
       // same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2).
       // During the intermediate stages of SLP, it's very common to have
       // pattern like this (since optimizeGatherSequence is run only once
       // at the end):
       // %1 = extractelement <2 x i32> %a, i32 0
       // %2 = extractelement <2 x i32> %a, i32 1
       // %cond = icmp sgt i32 %1, %2

llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Given LLVM IR caused associative reduction matching routine crash in SLP.
+; The routines begins with select as integer Umax reduction kind
+; and then follows to llvm.umax intrinsic call which also matched
+; to UMax and thus same reduction kind is returned.
+; The routine's later code merely assumes the instruction to be a select.
+
+define dso_local void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[NEXT:%.*]], label [[THEN:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[UM:%.*]] = call i8 @llvm.umax.i8(i8 0, i8 undef)
+; CHECK-NEXT:    [[SELCMP:%.*]] = icmp ult i8 [[UM]], undef
+; CHECK-NEXT:    [[I0:%.*]] = select i1 [[SELCMP]], i8 undef, i8 [[UM]]
+; CHECK-NEXT:    br label [[NEXT]]
+; CHECK:       next:
+; CHECK-NEXT:    [[T7_0:%.*]] = phi i8 [ undef, [[ENTRY:%.*]] ], [ [[I0]], [[THEN]] ]
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %next, label %then
+
+then:
+  %um = call i8 @llvm.umax.i8(i8 0, i8 undef)
+  %selcmp = icmp ult i8 %um, undef
+  %i0 = select i1 %selcmp, i8 undef, i8 %um
+  br label %next
+
+next:
+  %t7.0 = phi i8 [ undef, %entry ], [ %i0, %then ]
+  ret void
+}
+
+declare i8 @llvm.umax.i8(i8, i8)
+