[Flang] Map `ieee_fma` intrinsic to `llvm.fma`

[sunshaoce]

Summary

Map ieee_fma intrinsic to LLVM IR as llvm.fma.

[klausler]

Most target architectures support FMA and related instructions, and there's a standard IEEE_FMA generic function in the intrinsic module IEEE_ARITHMETIC in Fortran 2018. Please implement FMA in a way that is not just for RISC-V.

[sunshaoce]

Address @klausler's comment. And, I’m not sure if the fmadd intrinsic is in the right place in the intrinsic.md.

[klausler]

Don't add a new nonstandard intrinsic when the standard already has a standard name for FMA, namely the procedure IEEE_FMA in the intrinsic module IEEE_ARITHMETIC. See subclause 17.11.4 in Fortran 2018.

The technique that has worked for us in the implementation of other procedures in intrinsic modules is to first declare a builtin intrinsic procedure, say "builtin_fma", in the module fortran_builtins, and then rename it in the intrinsic module via "use fortran_builtins, only: ieee_fma => builtin_fma". Then implement "builtin_fma" like any other standard intrinsic function. If you look in fortran_builtins for declarations of other INTRINSIC procedures, you'll be able to find useful examples.

[sunshaoce]

@klausler, thanks for your detailed comments. I have tried to follow them.

[klausler]

flang/lib/Evaluate/intrinsics.cpp

884

IEEE_FMA doesn't work for COMPLEX. Use "SameReal" here, not "SameFloating".

[sunshaoce]

Replace SameFloating with SameReal

flang/lib/Evaluate/fold-real.cpp

#endif
   } else if (name == "sqrt") {
     return FoldElementalIntrinsic<T, T>(context, std::move(funcRef),
         ScalarFunc<T, T>(
             [](const Scalar<T> &x) -> Scalar<T> { return x.SQRT().value; }));
   } else if (name == "sum") {
     return FoldSum<T>(context, std::move(funcRef));
   } else if (name == "tiny") {
     return Expr<T>{Scalar<T>::TINY()};
+  } else if (name == "__builtin_fma") {
+    CHECK(args.size() == 3);
   } else if (name == "__builtin_ieee_next_after") {
     if (const auto *yExpr{UnwrapExpr<Expr<SomeReal>>(args[1])}) {
       return common::visit(
           [&](const auto &yVal) {
             using TY = ResultType<decltype(yVal)>;
             return FoldElementalIntrinsic<T, T, TY>(context, std::move(funcRef),
                 ScalarFunc<T, T, TY>([&](const Scalar<T> &x,
                                          const Scalar<TY> &y) -> Scalar<T> {

flang/lib/Evaluate/intrinsics.cpp

// NULL() is a special case handled in Probe() below
         {{"vector", SameType, Rank::vector}, {"mask", AnyLogical, Rank::array},
             {"field", SameType, Rank::conformable}},
         SameType, Rank::conformable, IntrinsicClass::transformationalFunction},
     {"verify",
         {{"string", SameCharNoLen}, {"set", SameCharNoLen},
             {"back", AnyLogical, Rank::elemental, Optionality::optional},
             DefaultingKIND},
         KINDInt},
+    {"__builtin_fma", {{"f1", SameReal}, {"f2", SameReal}, {"f3", SameReal}},
+        SameReal},
     {"__builtin_ieee_is_nan", {{"a", AnyFloating}}, DefaultLogical},
     {"__builtin_ieee_is_negative", {{"a", AnyFloating}}, DefaultLogical},
     {"__builtin_ieee_is_normal", {{"a", AnyFloating}}, DefaultLogical},
     {"__builtin_ieee_next_after", {{"x", SameReal}, {"y", AnyReal}}, SameReal},
     {"__builtin_ieee_next_down", {{"x", SameReal}}, SameReal},
     {"__builtin_ieee_next_up", {{"x", SameReal}}, SameReal},
     {"__builtin_ieee_support_datatype",
         {{"x", AnyReal, Rank::elemental, Optionality::optional}},

flang/lib/Optimizer/Builder/IntrinsicCall.cpp

static constexpr MathOperation mathOperations[] = {
     {"exp", "exp", genF64F64FuncType, genMathOp<mlir::math::ExpOp>},
     {"exp", "cexpf", genComplexComplexFuncType<4>,
      genComplexMathOp<mlir::complex::ExpOp>},
     {"exp", "cexp", genComplexComplexFuncType<8>,
      genComplexMathOp<mlir::complex::ExpOp>},
     // math::FloorOp returns a real, while Fortran FLOOR returns integer.
     {"floor", "floorf", genF32F32FuncType, genMathOp<mlir::math::FloorOp>},
     {"floor", "floor", genF64F64FuncType, genMathOp<mlir::math::FloorOp>},
+    {"fma", "llvm.fma.f32", genF32F32F32F32FuncType,
+     genMathOp<mlir::math::FmaOp>},
+    {"fma", "llvm.fma.f64", genF64F64F64F64FuncType,
+     genMathOp<mlir::math::FmaOp>},
     {"gamma", "tgammaf", genF32F32FuncType, genLibCall},
     {"gamma", "tgamma", genF64F64FuncType, genLibCall},
     {"hypot", "hypotf", genF32F32F32FuncType, genLibCall},
     {"hypot", "hypot", genF64F64F64FuncType, genLibCall},
     {"log", "logf", genF32F32FuncType, genMathOp<mlir::math::LogOp>},
     {"log", "log", genF64F64FuncType, genMathOp<mlir::math::LogOp>},
     {"log", "clogf", genComplexComplexFuncType<4>,
      genComplexMathOp<mlir::complex::LogOp>},

flang/module/__fortran_builtins.f90

module __Fortran_builtins
 
   type :: __builtin_dim3
     integer :: x=1, y=1, z=1
   end type
   type(__builtin_dim3) :: &
     __builtin_threadIdx, __builtin_blockDim, __builtin_blockIdx, __builtin_gridDim
   integer, parameter :: __builtin_warpsize = 32
 
+  intrinsic :: __builtin_fma
   intrinsic :: __builtin_ieee_is_nan, __builtin_ieee_is_negative, &
     __builtin_ieee_is_normal
   intrinsic :: __builtin_ieee_next_after, __builtin_ieee_next_down, &
     __builtin_ieee_next_up
   intrinsic :: scale ! for ieee_scalb
   intrinsic :: __builtin_ieee_selected_real_kind
   intrinsic :: __builtin_ieee_support_datatype, &
     __builtin_ieee_support_denormal, __builtin_ieee_support_divide, &

flang/module/ieee_arithmetic.f90

 
 module ieee_arithmetic
   ! 17.1: "The module IEEE_ARITHMETIC behaves as if it contained a
   ! USE statement for IEEE_EXCEPTIONS; everything that is public in
   ! IEEE_EXCEPTIONS is public in IEEE_ARITHMETIC."
   use __Fortran_ieee_exceptions
 
   use __Fortran_builtins, only: &
+    ieee_fma => __builtin_fma, &
     ieee_is_nan => __builtin_ieee_is_nan, &
     ieee_is_negative => __builtin_ieee_is_negative, &
     ieee_is_normal => __builtin_ieee_is_normal, &
     ieee_next_after => __builtin_ieee_next_after, &
     ieee_next_down => __builtin_ieee_next_down, &
     ieee_next_up => __builtin_ieee_next_up, &
     ieee_scalb => scale, &
     ieee_selected_real_kind => __builtin_ieee_selected_real_kind, &
elemental real(XKIND) function ieee_copy_sign_a##XKIND##_a##YKIND(x, y); \
     real(YKIND), intent(in) :: y; \
   end function ieee_copy_sign_a##XKIND##_a##YKIND;
   interface ieee_copy_sign
     SPECIFICS_RR(IEEE_COPY_SIGN_RR)
   end interface ieee_copy_sign
   PRIVATE_RR(IEEE_COPY_SIGN)
 #undef IEEE_COPY_SIGN_RR
 
-#define IEEE_FMA_R(AKIND) \
-  elemental real(AKIND) function ieee_fma_a##AKIND(a, b, c); \
-    real(AKIND), intent(in) :: a, b, c; \
-  end function ieee_fma_a##AKIND;
-  interface ieee_fma
-    SPECIFICS_R(IEEE_FMA_R)
-  end interface ieee_fma
-  PRIVATE_R(IEEE_FMA)
-#undef IEEE_FMA_R
-
 #define IEEE_GET_ROUNDING_MODE_I(RKIND) \
   subroutine ieee_get_rounding_mode_i##RKIND(round_value, radix); \
     import ieee_round_type; \
     type(ieee_round_type), intent(out) :: round_value; \
     integer(RKIND), intent(in) :: radix; \
   end subroutine ieee_get_rounding_mode_i##RKIND;
   interface ieee_get_rounding_mode
     subroutine ieee_get_rounding_mode_0(round_value)

flang/test/Lower/Intrinsics/fma.f90

+! RUN: bbc -emit-fir %s -o - | FileCheck --check-prefix=CHECK-FIR %s
+! RUN: %flang_fc1 -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-LLVMIR %s
+
+function test_real4(a, x, y)
+  use ieee_arithmetic, only: ieee_fma
+  real :: a, x, y
+  test_real4 = ieee_fma(a, x, y)
+end function
+
+! CHECK-LABEL: @_QPtest_real4
+! CHECK-FIR: {{%[A-Za-z0-9._]+}} = math.fma {{%[0-9]+}}, {{%[0-9]+}}, {{%[0-9]+}} {{.*}} : f32
+! CHECK-LLVMIR: {{%[A-Za-z0-9._]+}} = call {{.*}} float @llvm.fma.f32(float {{%[0-9]+}}, float {{%[0-9]+}}, float {{%[0-9]+}})
+
+function test_real8(a, x, y)
+  use ieee_arithmetic, only: ieee_fma
+  real(8) :: a, x, y
+  test_real8 = ieee_fma(a, x, y)
+end function
+
+! CHECK-LABEL: @_QPtest_real8
+! CHECK-FIR: {{%[A-Za-z0-9._]+}} = math.fma {{%[0-9]+}}, {{%[0-9]+}}, {{%[0-9]+}} {{.*}} : f64
+! CHECK-LLVMIR: {{%[A-Za-z0-9._]+}} = call {{.*}} double @llvm.fma.f64(double {{%[0-9]+}}, double {{%[0-9]+}}, double {{%[0-9]+}})