[SLPVectorize] Add support for fmaximum/fminimum reduction

[anna]

Summary

This patch adds support for vectorized reduction of maximum/minimum intrinsics
which are under the appropriate reduction kind.

Added relevant tests for reduction.

[anna]

Will precommit the tests (regular vectorized code will have no difference, just the reduction ones will be vectorized with this patch).

[ABataev]

llvm/test/Transforms/SLPVectorizer/X86/fmaximum-fminimum.ll

202

1. Fix this messages, remove unused check tags
2. Create a separate patch with the test.

[anna]

llvm/test/Transforms/SLPVectorizer/X86/fmaximum-fminimum.ll

202

These CHECK lines are generated by the auto generator. I've removed these and separate patch is created for the tests.

[anna]

rebased over test in D155096

[ABataev]

LG

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

static bool isVectorizable(RecurKind Kind, Instruction *I) {
 
     if (Kind == RecurKind::FMax || Kind == RecurKind::FMin) {
       // FP min/max are associative except for NaN and -0.0. We do not
       // have to rule out -0.0 here because the intrinsic semantics do not
       // specify a fixed result for it.
       return I->getFastMathFlags().noNaNs();
     }
 
+    if (Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimum)
+      return true;
+
     return I->isAssociative();
   }
 
   static Value *getRdxOperand(Instruction *I, unsigned Index) {
     // Poison-safe 'or' takes the form: select X, true, Y
     // To make that work with the normal operand processing, we skip the
     // true value operand.
     // TODO: Change the code and data structures to handle this without a hack.
case RecurKind::FMax:
                                       cast<ConstantFP>(RHS)->getValueAPF()));
       return Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS);
     case RecurKind::FMin:
       if (IsConstant)
         return ConstantFP::get(LHS->getType(),
                                minnum(cast<ConstantFP>(LHS)->getValueAPF(),
                                       cast<ConstantFP>(RHS)->getValueAPF()));
       return Builder.CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS);
+    case RecurKind::FMaximum:
+      if (IsConstant)
+        return ConstantFP::get(LHS->getType(),
+                               maximum(cast<ConstantFP>(LHS)->getValueAPF(),
+                                      cast<ConstantFP>(RHS)->getValueAPF()));
+      return Builder.CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS);
+    case RecurKind::FMinimum:
+      if (IsConstant)
+        return ConstantFP::get(LHS->getType(),
+                               minimum(cast<ConstantFP>(LHS)->getValueAPF(),
+                                      cast<ConstantFP>(RHS)->getValueAPF()));
+      return Builder.CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS);
     case RecurKind::SMax:
       if (IsConstant || UseSelect) {
         Value *Cmp = Builder.CreateICmpSGT(LHS, RHS, Name);
         return Builder.CreateSelect(Cmp, LHS, RHS, Name);
       }
       return Builder.CreateBinaryIntrinsic(Intrinsic::smax, LHS, RHS);
     case RecurKind::SMin:
       if (IsConstant || UseSelect) {
static RecurKind getRdxKind(Value *V) {
     if (match(I, m_FMul(m_Value(), m_Value())))
       return RecurKind::FMul;
 
     if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
       return RecurKind::FMax;
     if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
       return RecurKind::FMin;
 
+    if (match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value())))
+      return RecurKind::FMaximum;
+    if (match(I, m_Intrinsic<Intrinsic::minimum>(m_Value(), m_Value())))
+      return RecurKind::FMinimum;
     // This matches either cmp+select or intrinsics. SLP is expected to handle
     // either form.
     // TODO: If we are canonicalizing to intrinsics, we can remove several
     //       special-case paths that deal with selects.
     if (match(I, m_SMax(m_Value(), m_Value())))
       return RecurKind::SMax;
     if (match(I, m_SMin(m_Value(), m_Value())))
       return RecurKind::SMin;
case RecurKind::FMul: {
             TTI->getArithmeticReductionCost(RdxOpcode, VectorTy, FMF, CostKind);
       ScalarCost = EvaluateScalarCost([&]() {
         return TTI->getArithmeticInstrCost(RdxOpcode, ScalarTy, CostKind);
       });
       break;
     }
     case RecurKind::FMax:
     case RecurKind::FMin:
+    case RecurKind::FMaximum:
+    case RecurKind::FMinimum:
     case RecurKind::SMax:
     case RecurKind::SMin:
     case RecurKind::UMax:
     case RecurKind::UMin: {
       Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
       if (!AllConsts)
         VectorCost = TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
       ScalarCost = EvaluateScalarCost([&]() {

 
 static bool matchRdxBop(Instruction *I, Value *&V0, Value *&V1) {
   if (match(I, m_BinOp(m_Value(V0), m_Value(V1))))
     return true;
   if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(V0), m_Value(V1))))
     return true;
   if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(V0), m_Value(V1))))
     return true;
+  if (match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(V0), m_Value(V1))))
+    return true;
+  if (match(I, m_Intrinsic<Intrinsic::minimum>(m_Value(V0), m_Value(V1))))
+    return true;
   if (match(I, m_Intrinsic<Intrinsic::smax>(m_Value(V0), m_Value(V1))))
     return true;
   if (match(I, m_Intrinsic<Intrinsic::smin>(m_Value(V0), m_Value(V1))))
     return true;
   if (match(I, m_Intrinsic<Intrinsic::umax>(m_Value(V0), m_Value(V1))))
     return true;
   if (match(I, m_Intrinsic<Intrinsic::umin>(m_Value(V0), m_Value(V1))))
     return true;

llvm/test/Transforms/SLPVectorizer/X86/fmaximum-fminimum.ll

;
   %t1 = load double, ptr %g1, align 4
   %m1 = tail call double @llvm.maximum.f64(double %t1, double %t0)
   ret double %m1
 }
 
 define float @reduction_v4f32(ptr %p) {
 ; SSE-LABEL: define float @reduction_v4f32
 ; SSE-SAME: (ptr [[P:%.*]]) {
-; SSE-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
-; SSE-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
-; SSE-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[P]], i64 3
-; SSE-NEXT:    [[T0:%.*]] = load float, ptr [[P]], align 4
-; SSE-NEXT:    [[T1:%.*]] = load float, ptr [[G1]], align 4
-; SSE-NEXT:    [[T2:%.*]] = load float, ptr [[G2]], align 4
-; SSE-NEXT:    [[T3:%.*]] = load float, ptr [[G3]], align 4
-; SSE-NEXT:    [[M1:%.*]] = tail call float @llvm.maximum.f32(float [[T1]], float [[T0]])
-; SSE-NEXT:    [[M2:%.*]] = tail call float @llvm.maximum.f32(float [[T2]], float [[M1]])
-; SSE-NEXT:    [[M3:%.*]] = tail call float @llvm.maximum.f32(float [[T3]], float [[M2]])
-; SSE-NEXT:    ret float [[M3]]
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 4
+; SSE-NEXT:    [[TMP2:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP1]])
+; SSE-NEXT:    ret float [[TMP2]]
 ;
 ; AVX-LABEL: define float @reduction_v4f32
 ; AVX-SAME: (ptr [[P:%.*]]) #[[ATTR1]] {
-; AVX-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
-; AVX-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
-; AVX-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[P]], i64 3
-; AVX-NEXT:    [[T0:%.*]] = load float, ptr [[P]], align 4
-; AVX-NEXT:    [[T1:%.*]] = load float, ptr [[G1]], align 4
-; AVX-NEXT:    [[T2:%.*]] = load float, ptr [[G2]], align 4
-; AVX-NEXT:    [[T3:%.*]] = load float, ptr [[G3]], align 4
-; AVX-NEXT:    [[M1:%.*]] = tail call float @llvm.maximum.f32(float [[T1]], float [[T0]])
-; AVX-NEXT:    [[M2:%.*]] = tail call float @llvm.maximum.f32(float [[T2]], float [[M1]])
-; AVX-NEXT:    [[M3:%.*]] = tail call float @llvm.maximum.f32(float [[T3]], float [[M2]])
-; AVX-NEXT:    ret float [[M3]]
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP1]])
+; AVX-NEXT:    ret float [[TMP2]]
 ;
   %g1 = getelementptr inbounds float, ptr %p, i64 1
   %g2 = getelementptr inbounds float, ptr %p, i64 2
   %g3 = getelementptr inbounds float, ptr %p, i64 3
   %t0 = load float, ptr %p, align 4
   %t1 = load float, ptr %g1, align 4
   %t2 = load float, ptr %g2, align 4
   %t3 = load float, ptr %g3, align 4
   %m1 = tail call float @llvm.maximum.f32(float %t1, float %t0)
   %m2 = tail call float @llvm.maximum.f32(float %t2, float %m1)
   %m3 = tail call float @llvm.maximum.f32(float %t3, float %m2)
   ret float %m3
 }
 
 define double @reduction_v4f64_fminimum(ptr %p) {
 ; SSE-LABEL: define double @reduction_v4f64_fminimum
 ; SSE-SAME: (ptr [[P:%.*]]) {
-; SSE-NEXT:    [[G1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
-; SSE-NEXT:    [[G2:%.*]] = getelementptr inbounds double, ptr [[P]], i64 2
-; SSE-NEXT:    [[G3:%.*]] = getelementptr inbounds double, ptr [[P]], i64 3
-; SSE-NEXT:    [[T0:%.*]] = load double, ptr [[P]], align 4
-; SSE-NEXT:    [[T1:%.*]] = load double, ptr [[G1]], align 4
-; SSE-NEXT:    [[T2:%.*]] = load double, ptr [[G2]], align 4
-; SSE-NEXT:    [[T3:%.*]] = load double, ptr [[G3]], align 4
-; SSE-NEXT:    [[M1:%.*]] = tail call double @llvm.minimum.f64(double [[T1]], double [[T0]])
-; SSE-NEXT:    [[M2:%.*]] = tail call double @llvm.minimum.f64(double [[T2]], double [[M1]])
-; SSE-NEXT:    [[M3:%.*]] = tail call double @llvm.minimum.f64(double [[T3]], double [[M2]])
-; SSE-NEXT:    ret double [[M3]]
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr [[P]], align 4
+; SSE-NEXT:    [[TMP2:%.*]] = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> [[TMP1]])
+; SSE-NEXT:    ret double [[TMP2]]
 ;
 ; AVX-LABEL: define double @reduction_v4f64_fminimum
 ; AVX-SAME: (ptr [[P:%.*]]) #[[ATTR1]] {
-; AVX-NEXT:    [[G1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
-; AVX-NEXT:    [[G2:%.*]] = getelementptr inbounds double, ptr [[P]], i64 2
-; AVX-NEXT:    [[G3:%.*]] = getelementptr inbounds double, ptr [[P]], i64 3
-; AVX-NEXT:    [[T0:%.*]] = load double, ptr [[P]], align 4
-; AVX-NEXT:    [[T1:%.*]] = load double, ptr [[G1]], align 4
-; AVX-NEXT:    [[T2:%.*]] = load double, ptr [[G2]], align 4
-; AVX-NEXT:    [[T3:%.*]] = load double, ptr [[G3]], align 4
-; AVX-NEXT:    [[M1:%.*]] = tail call double @llvm.minimum.f64(double [[T1]], double [[T0]])
-; AVX-NEXT:    [[M2:%.*]] = tail call double @llvm.minimum.f64(double [[T2]], double [[M1]])
-; AVX-NEXT:    [[M3:%.*]] = tail call double @llvm.minimum.f64(double [[T3]], double [[M2]])
-; AVX-NEXT:    ret double [[M3]]
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr [[P]], align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    ret double [[TMP2]]
 ;
   %g1 = getelementptr inbounds double, ptr %p, i64 1
   %g2 = getelementptr inbounds double, ptr %p, i64 2
   %g3 = getelementptr inbounds double, ptr %p, i64 3
   %t0 = load double, ptr %p, align 4
   %t1 = load double, ptr %g1, align 4
   %t2 = load double, ptr %g2, align 4
   %t3 = load double, ptr %g3, align 4
   %m1 = tail call double @llvm.minimum.f64(double %t1, double %t0)
   %m2 = tail call double @llvm.minimum.f64(double %t2, double %m1)
   %m3 = tail call double @llvm.minimum.f64(double %t3, double %m2)
   ret double %m3
 }
 
 define float @reduction_v8f32_fminimum(ptr %p) {
 ; SSE-LABEL: define float @reduction_v8f32_fminimum
 ; SSE-SAME: (ptr [[P:%.*]]) {
-; SSE-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
-; SSE-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
-; SSE-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[P]], i64 3
-; SSE-NEXT:    [[G4:%.*]] = getelementptr inbounds float, ptr [[P]], i64 4
-; SSE-NEXT:    [[G5:%.*]] = getelementptr inbounds float, ptr [[P]], i64 5
-; SSE-NEXT:    [[G6:%.*]] = getelementptr inbounds float, ptr [[P]], i64 6
-; SSE-NEXT:    [[G7:%.*]] = getelementptr inbounds float, ptr [[P]], i64 7
-; SSE-NEXT:    [[T0:%.*]] = load float, ptr [[P]], align 4
-; SSE-NEXT:    [[T1:%.*]] = load float, ptr [[G1]], align 4
-; SSE-NEXT:    [[T2:%.*]] = load float, ptr [[G2]], align 4
-; SSE-NEXT:    [[T3:%.*]] = load float, ptr [[G3]], align 4
-; SSE-NEXT:    [[T4:%.*]] = load float, ptr [[G4]], align 4
-; SSE-NEXT:    [[T5:%.*]] = load float, ptr [[G5]], align 4
-; SSE-NEXT:    [[T6:%.*]] = load float, ptr [[G6]], align 4
-; SSE-NEXT:    [[T7:%.*]] = load float, ptr [[G7]], align 4
-; SSE-NEXT:    [[M1:%.*]] = tail call float @llvm.minimum.f32(float [[T1]], float [[T0]])
-; SSE-NEXT:    [[M2:%.*]] = tail call float @llvm.minimum.f32(float [[T2]], float [[M1]])
-; SSE-NEXT:    [[M3:%.*]] = tail call float @llvm.minimum.f32(float [[T3]], float [[M2]])
-; SSE-NEXT:    [[M4:%.*]] = tail call float @llvm.minimum.f32(float [[T4]], float [[M3]])
-; SSE-NEXT:    [[M5:%.*]] = tail call float @llvm.minimum.f32(float [[M4]], float [[T6]])
-; SSE-NEXT:    [[M6:%.*]] = tail call float @llvm.minimum.f32(float [[M5]], float [[T5]])
-; SSE-NEXT:    [[M7:%.*]] = tail call float @llvm.minimum.f32(float [[M6]], float [[T7]])
-; SSE-NEXT:    ret float [[M7]]
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x float>, ptr [[P]], align 4
+; SSE-NEXT:    [[TMP2:%.*]] = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> [[TMP1]])
+; SSE-NEXT:    ret float [[TMP2]]
 ;
 ; AVX-LABEL: define float @reduction_v8f32_fminimum
 ; AVX-SAME: (ptr [[P:%.*]]) #[[ATTR1]] {
-; AVX-NEXT:    [[G1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
-; AVX-NEXT:    [[G2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
-; AVX-NEXT:    [[G3:%.*]] = getelementptr inbounds float, ptr [[P]], i64 3
-; AVX-NEXT:    [[G4:%.*]] = getelementptr inbounds float, ptr [[P]], i64 4
-; AVX-NEXT:    [[G5:%.*]] = getelementptr inbounds float, ptr [[P]], i64 5
-; AVX-NEXT:    [[G6:%.*]] = getelementptr inbounds float, ptr [[P]], i64 6
-; AVX-NEXT:    [[G7:%.*]] = getelementptr inbounds float, ptr [[P]], i64 7
-; AVX-NEXT:    [[T0:%.*]] = load float, ptr [[P]], align 4
-; AVX-NEXT:    [[T1:%.*]] = load float, ptr [[G1]], align 4
-; AVX-NEXT:    [[T2:%.*]] = load float, ptr [[G2]], align 4
-; AVX-NEXT:    [[T3:%.*]] = load float, ptr [[G3]], align 4
-; AVX-NEXT:    [[T4:%.*]] = load float, ptr [[G4]], align 4
-; AVX-NEXT:    [[T5:%.*]] = load float, ptr [[G5]], align 4
-; AVX-NEXT:    [[T6:%.*]] = load float, ptr [[G6]], align 4
-; AVX-NEXT:    [[T7:%.*]] = load float, ptr [[G7]], align 4
-; AVX-NEXT:    [[M1:%.*]] = tail call float @llvm.minimum.f32(float [[T1]], float [[T0]])
-; AVX-NEXT:    [[M2:%.*]] = tail call float @llvm.minimum.f32(float [[T2]], float [[M1]])
-; AVX-NEXT:    [[M3:%.*]] = tail call float @llvm.minimum.f32(float [[T3]], float [[M2]])
-; AVX-NEXT:    [[M4:%.*]] = tail call float @llvm.minimum.f32(float [[T4]], float [[M3]])
-; AVX-NEXT:    [[M5:%.*]] = tail call float @llvm.minimum.f32(float [[M4]], float [[T6]])
-; AVX-NEXT:    [[M6:%.*]] = tail call float @llvm.minimum.f32(float [[M5]], float [[T5]])
-; AVX-NEXT:    [[M7:%.*]] = tail call float @llvm.minimum.f32(float [[M6]], float [[T7]])
-; AVX-NEXT:    ret float [[M7]]
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, ptr [[P]], align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    ret float [[TMP2]]
 ;
   %g1 = getelementptr inbounds float, ptr %p, i64 1
   %g2 = getelementptr inbounds float, ptr %p, i64 2
   %g3 = getelementptr inbounds float, ptr %p, i64 3
   %g4 = getelementptr inbounds float, ptr %p, i64 4
   %g5 = getelementptr inbounds float, ptr %p, i64 5
   %g6 = getelementptr inbounds float, ptr %p, i64 6
   %g7 = getelementptr inbounds float, ptr %p, i64 7