[Clang] Add min/max reduction builtins.

[fhahn]

Summary

This patch implements builtin_reduce_max and builtin_reduce_min as
specified in D111529.

The order of operations does not matter for min or max reductions and
they can be directly lowered to the corresponding
llvm.vector.reduce.{fmin,fmax,umin,umax,smin,smax} intrinsic calls.

[craig.topper]

clang/lib/Sema/SemaChecking.cpp

16546

Is this indented incorrectly? There appear to be 2 ifs at the same level

[fhahn]

Remove stray !TyA->getAs<VectorType>() check.

[fhahn]

clang/lib/Sema/SemaChecking.cpp

16546

Yes they were, thanks! The if (!TyA->getAs<VectorType>()) was a left-over from an earlier iteration. Removed.

[fhahn]

rebase on top of recently updated patches.

[fhahn]

Rebased after recent changes in D111985 https://reviews.llvm.org/D111985: changes the return type of SemaBuiltinReduceMath to bool and drop the CallResult argument.

Also added tests with const and variables with address space and moved tests to builtins-reduction-math.c instead of builtins-elementwise-math.c

[aaron.ballman]

clang/lib/Sema/SemaChecking.cpp

16611–16614

What about type sugar?

16616

Should this type undergo the usual conversions?

[fhahn]

rebase on top of recent changes.

[aaron.ballman]

Precommit CI looks to be failing.

/var/lib/buildkite-agent/builds/llvm-project/clang/test/CodeGen/builtins-reduction-math.c:13:9: error: initializing 'float' with an expression of incompatible type 'float4' (vector of 4 'float' values)

  float r1 = __builtin_reduce_max(vf1);

[fhahn]

In D112001#3099965, @aaron.ballman wrote:

Precommit CI looks to be failing.

/var/lib/buildkite-agent/builds/llvm-project/clang/test/CodeGen/builtins-reduction-math.c:13:9: error: initializing 'float' with an expression of incompatible type 'float4' (vector of 4 'float' values)

  float r1 = __builtin_reduce_max(vf1);

The last version of the patch did not correctly set the return type to the element type. That should be fixed now!

[aaron.ballman]

LGTM!

clang/include/clang/Basic/Builtins.def

 BUILTIN(__builtin_convertvector, "v."   , "nct")
 BUILTIN(__builtin_alloca, "v*z"   , "Fn")
 BUILTIN(__builtin_alloca_with_align, "v*zIz", "Fn")
 BUILTIN(__builtin_call_with_static_chain, "v.", "nt")
 
 BUILTIN(__builtin_elementwise_abs, "v.", "nct")
 BUILTIN(__builtin_elementwise_max, "v.", "nct")
 BUILTIN(__builtin_elementwise_min, "v.", "nct")
+BUILTIN(__builtin_reduce_max, "v.", "nct")
+BUILTIN(__builtin_reduce_min, "v.", "nct")
 
 BUILTIN(__builtin_matrix_transpose, "v.", "nFt")
 BUILTIN(__builtin_matrix_column_major_load, "v.", "nFt")
 BUILTIN(__builtin_matrix_column_major_store, "v.", "nFt")
 
 // "Overloaded" Atomic operator builtins.  These are overloaded to support data
 // types of i8, i16, i32, i64, and i128.  The front-end sees calls to the
 // non-suffixed version of these (which has a bogus type) and transforms them to

clang/include/clang/Basic/DiagnosticSemaKinds.td

 def err_builtin_launder_invalid_arg : Error<
   "%select{non-pointer|function pointer|void pointer}0 argument to "
   "'__builtin_launder' is not allowed">;
 
 def err_builtin_invalid_arg_type: Error <
   "%ordinal0 argument must be a "
   "%select{vector, integer or floating point type|matrix|"
    "pointer to a valid matrix element type|"
-   "signed integer or floating point type}1 (was %2)">;
+   "signed integer or floating point type|vector type}1 (was %2)">;
 
 def err_builtin_matrix_disabled: Error<
   "matrix types extension is disabled. Pass -fenable-matrix to enable it">;
 def err_matrix_index_not_integer: Error<
   "matrix %select{row|column}0 index is not an integer">;
 def err_matrix_index_outside_range: Error<
   "matrix %select{row|column}0 index is outside the allowed range [0, %1)">;
 def err_matrix_incomplete_index: Error<

clang/include/clang/Sema/Sema.h

private:
   bool SemaBuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
   bool SemaBuiltinPPCMMACall(CallExpr *TheCall, unsigned BuiltinID,
                              const char *TypeDesc);
 
   bool CheckPPCMMAType(QualType Type, SourceLocation TypeLoc);
 
   bool SemaBuiltinElementwiseMath(CallExpr *TheCall);
   bool SemaBuiltinElementwiseMathOneArg(CallExpr *TheCall);
+  bool SemaBuiltinReduceMath(CallExpr *TheCall);
 
   // Matrix builtin handling.
   ExprResult SemaBuiltinMatrixTranspose(CallExpr *TheCall,
                                         ExprResult CallResult);
   ExprResult SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
                                               ExprResult CallResult);
   ExprResult SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall,
                                                ExprResult CallResult);

clang/lib/CodeGen/CGBuiltin.cpp

if (Op0->getType()->isIntOrIntVectorTy()) {
                                                  ? llvm::Intrinsic::smin
                                                  : llvm::Intrinsic::umin,
                                              Op0, Op1, nullptr, "elt.min");
     } else
       Result = Builder.CreateMinNum(Op0, Op1, "elt.min");
     return RValue::get(Result);
   }
 
+  case Builtin::BI__builtin_reduce_max: {
+    auto GetIntrinsicID = [](QualType QT, llvm::Type *IrTy) {
+      if (IrTy->isIntOrIntVectorTy()) {
+        if (auto *VecTy = QT->getAs<VectorType>())
+          QT = VecTy->getElementType();
+        if (QT->isSignedIntegerType())
+          return llvm::Intrinsic::vector_reduce_smax;
+        else
+          return llvm::Intrinsic::vector_reduce_umax;
+      }
+      return llvm::Intrinsic::vector_reduce_fmax;
+    };
+    Value *Op0 = EmitScalarExpr(E->getArg(0));
+    Value *Result = Builder.CreateUnaryIntrinsic(
+        GetIntrinsicID(E->getArg(0)->getType(), Op0->getType()), Op0, nullptr,
+        "rdx.min");
+    return RValue::get(Result);
+  }
+
+  case Builtin::BI__builtin_reduce_min: {
+    auto GetIntrinsicID = [](QualType QT, llvm::Type *IrTy) {
+      if (IrTy->isIntOrIntVectorTy()) {
+        if (auto *VecTy = QT->getAs<VectorType>())
+          QT = VecTy->getElementType();
+        if (QT->isSignedIntegerType())
+          return llvm::Intrinsic::vector_reduce_smin;
+        else
+          return llvm::Intrinsic::vector_reduce_umin;
+      }
+      return llvm::Intrinsic::vector_reduce_fmin;
+    };
+    Value *Op0 = EmitScalarExpr(E->getArg(0));
+    Value *Result = Builder.CreateUnaryIntrinsic(
+        GetIntrinsicID(E->getArg(0)->getType(), Op0->getType()), Op0, nullptr,
+        "rdx.min");
+    return RValue::get(Result);
+  }
+
   case Builtin::BI__builtin_matrix_transpose: {
     const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
     Value *MatValue = EmitScalarExpr(E->getArg(0));
     MatrixBuilder<CGBuilderTy> MB(Builder);
     Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
                                              MatrixTy->getNumColumns());
     return RValue::get(Result);
   }

clang/lib/Sema/SemaChecking.cpp

case Builtin::BI__builtin_elementwise_abs:
     if (SemaBuiltinElementwiseMathOneArg(TheCall))
       return ExprError();
     break;
   case Builtin::BI__builtin_elementwise_min:
   case Builtin::BI__builtin_elementwise_max:
     if (SemaBuiltinElementwiseMath(TheCall))
       return ExprError();
     break;
+  case Builtin::BI__builtin_reduce_max:
+  case Builtin::BI__builtin_reduce_min:
+    if (SemaBuiltinReduceMath(TheCall))
+      return ExprError();
+    break;
   case Builtin::BI__builtin_matrix_transpose:
     return SemaBuiltinMatrixTranspose(TheCall, TheCallResult);
 
   case Builtin::BI__builtin_matrix_column_major_load:
     return SemaBuiltinMatrixColumnMajorLoad(TheCall, TheCallResult);
 
   case Builtin::BI__builtin_matrix_column_major_store:
     return SemaBuiltinMatrixColumnMajorStore(TheCall, TheCallResult);

 // false.
 static bool checkMathBuiltinElementType(Sema &S, SourceLocation Loc,
                                         QualType Ty) {
   if (!Ty->getAs<VectorType>() && !ConstantMatrixType::isValidElementType(Ty)) {
     S.Diag(Loc, diag::err_builtin_invalid_arg_type)
         << 1 << /* vector, integer or float ty*/ 0 << Ty;
     return true;
   }
   return false;
 }
 
 bool Sema::SemaBuiltinElementwiseMathOneArg(CallExpr *TheCall) {
   if (checkArgCount(*this, TheCall, 1))
     return true;
 
   ExprResult A = UsualUnaryConversions(TheCall->getArg(0));
   SourceLocation ArgLoc = TheCall->getArg(0)->getBeginLoc();
if (checkMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA))
     return true;
 
   TheCall->setArg(0, A.get());
   TheCall->setArg(1, B.get());
   TheCall->setType(Res);
   return false;
 }
 
+bool Sema::SemaBuiltinReduceMath(CallExpr *TheCall) {
+  if (checkArgCount(*this, TheCall, 1))
+    return true;
+
+  ExprResult A = UsualUnaryConversions(TheCall->getArg(0));
+  if (A.isInvalid())
+    return true;
+
+  TheCall->setArg(0, A.get());
+  const VectorType *TyA = A.get()->getType()->getAs<VectorType>();
+  if (!TyA) {
+    SourceLocation ArgLoc = TheCall->getArg(0)->getBeginLoc();
+    return Diag(ArgLoc, diag::err_builtin_invalid_arg_type)
+           << 1 << /* vector ty*/ 4 << A.get()->getType();
+  }
+
+  TheCall->setType(TyA->getElementType());
+  return false;
+}
+
 ExprResult Sema::SemaBuiltinMatrixTranspose(CallExpr *TheCall,
                                             ExprResult CallResult) {
   if (checkArgCount(*this, TheCall, 1))
     return ExprError();
 
   ExprResult MatrixArg = DefaultLvalueConversion(TheCall->getArg(0));
   if (MatrixArg.isInvalid())
     return MatrixArg;

clang/test/CodeGen/builtins-reduction-math.c

+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+typedef float float4 __attribute__((ext_vector_type(4)));
+typedef short int si8 __attribute__((ext_vector_type(8)));
+typedef unsigned int u4 __attribute__((ext_vector_type(4)));
+
+__attribute__((address_space(1))) float4 vf1_as_one;
+
+void test_builtin_reduce_max(float4 vf1, si8 vi1, u4 vu1) {
+  // CHECK-LABEL: define void @test_builtin_reduce_max(
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, <4 x float>* %vf1.addr, align 16
+  // CHECK-NEXT: call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[VF1]])
+  float r1 = __builtin_reduce_max(vf1);
+
+  // CHECK:      [[VI1:%.+]] = load <8 x i16>, <8 x i16>* %vi1.addr, align 16
+  // CHECK-NEXT: call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> [[VI1]])
+  short r2 = __builtin_reduce_max(vi1);
+
+  // CHECK:      [[VU1:%.+]] = load <4 x i32>, <4 x i32>* %vu1.addr, align 16
+  // CHECK-NEXT: call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[VU1]])
+  unsigned r3 = __builtin_reduce_max(vu1);
+
+  // CHECK:      [[VF1_AS1:%.+]] = load <4 x float>, <4 x float> addrspace(1)* @vf1_as_one, align 16
+  // CHECK-NEXT: [[RDX1:%.+]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[VF1_AS1]])
+  // CHECK-NEXT: fpext float [[RDX1]] to double
+  const double r4 = __builtin_reduce_max(vf1_as_one);
+
+  // CHECK:      [[CVI1:%.+]] = load <8 x i16>, <8 x i16>* %cvi1, align 16
+  // CHECK-NEXT: [[RDX2:%.+]] = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> [[CVI1]])
+  // CHECK-NEXT: sext i16 [[RDX2]] to i64
+  const si8 cvi1 = vi1;
+  unsigned long long r5 = __builtin_reduce_max(cvi1);
+}
+
+void test_builtin_reduce_min(float4 vf1, si8 vi1, u4 vu1) {
+  // CHECK-LABEL: define void @test_builtin_reduce_min(
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, <4 x float>* %vf1.addr, align 16
+  // CHECK-NEXT: call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[VF1]])
+  float r1 = __builtin_reduce_min(vf1);
+
+  // CHECK:      [[VI1:%.+]] = load <8 x i16>, <8 x i16>* %vi1.addr, align 16
+  // CHECK-NEXT: call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> [[VI1]])
+  short r2 = __builtin_reduce_min(vi1);
+
+  // CHECK:      [[VU1:%.+]] = load <4 x i32>, <4 x i32>* %vu1.addr, align 16
+  // CHECK-NEXT: call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[VU1]])
+  unsigned r3 = __builtin_reduce_min(vu1);
+
+  // CHECK:      [[VF1_AS1:%.+]] = load <4 x float>, <4 x float> addrspace(1)* @vf1_as_one, align 16
+  // CHECK-NEXT: [[RDX1:%.+]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[VF1_AS1]])
+  // CHECK-NEXT: fpext float [[RDX1]] to double
+  const double r4 = __builtin_reduce_min(vf1_as_one);
+
+  // CHECK:      [[CVI1:%.+]] = load <8 x i16>, <8 x i16>* %cvi1, align 16
+  // CHECK-NEXT: [[RDX2:%.+]] = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> [[CVI1]])
+  // CHECK-NEXT: sext i16 [[RDX2]] to i64
+  const si8 cvi1 = vi1;
+  unsigned long long r5 = __builtin_reduce_min(cvi1);
+}

clang/test/Sema/builtins-reduction-math.c

+// RUN: %clang_cc1 %s -pedantic -verify -triple=x86_64-apple-darwin9
+
+typedef float float4 __attribute__((ext_vector_type(4)));
+typedef int int3 __attribute__((ext_vector_type(3)));
+typedef unsigned unsigned4 __attribute__((ext_vector_type(4)));
+
+struct Foo {
+  char *p;
+};
+
+void test_builtin_reduce_max(int i, float4 v, int3 iv) {
+  struct Foo s = __builtin_reduce_max(iv);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'int'}}
+
+  i = __builtin_reduce_max(v, v);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+
+  i = __builtin_reduce_max();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  i = __builtin_reduce_max(i);
+  // expected-error@-1 {{1st argument must be a vector type (was 'int')}}
+}
+
+void test_builtin_reduce_min(int i, float4 v, int3 iv) {
+  struct Foo s = __builtin_reduce_min(iv);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'int'}}
+
+  i = __builtin_reduce_min(v, v);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+
+  i = __builtin_reduce_min();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  i = __builtin_reduce_min(i);
+  // expected-error@-1 {{1st argument must be a vector type (was 'int')}}
+}