[libc] Add ADD_FMA_FLAG macro to add -mfma flag to functions that requires it.

[lntue]

Summary

Add ADD_FMA_FLAG macro to add -mfma flag to functions that requires it.

[sivachandra]

libc/src/__support/architectures.h

41

Nit: s/ADD_FMA_FLAG/INLINE_FMA

[lntue]

[libc] Add INLINE_FMA macro to add -mfma flag to functions that requires it.

libc/src/__support/FPUtil/PolyEval.h

 #include "FMA.h"
 
 namespace __llvm_libc {
 namespace fputil {
 
 template <typename T> static inline T polyeval(T x, T a0) { return a0; }
 
 template <typename T, typename... Ts>
-static inline T polyeval(T x, T a0, Ts... a) {
+INLINE_FMA static inline T polyeval(T x, T a0, Ts... a) {
   return fma(x, polyeval(x, a...), a0);
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #ifdef LLVM_LIBC_ARCH_X86_64
 

libc/src/__support/FPUtil/x86_64/FMA.h

 //===-- x86_64 implementations of the fma function --------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H
 #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H
 
 #include "src/__support/architectures.h"
 
-#if !defined(LLVM_LIBC_ARCH_X86)
+#if !defined(LLVM_LIBC_ARCH_X86_64)
 #error "Invalid include"
 #endif
 
 #include "src/__support/CPP/TypeTraits.h"
 #include <immintrin.h>
 
 namespace __llvm_libc {
 namespace fputil {
 
 template <typename T>
-__attribute__((target(
-    "fma"))) static inline cpp::EnableIfType<cpp::IsSame<T, float>::Value, T>
+INLINE_FMA static inline cpp::EnableIfType<cpp::IsSame<T, float>::Value, T>
 fma(T x, T y, T z) {
   float result;
   __m128 xmm = _mm_load_ss(&x);           // NOLINT
   __m128 ymm = _mm_load_ss(&y);           // NOLINT
   __m128 zmm = _mm_load_ss(&z);           // NOLINT
   __m128 r = _mm_fmadd_ss(xmm, ymm, zmm); // NOLINT
   _mm_store_ss(&result, r);               // NOLINT
   return result;
 }
 
 template <typename T>
-__attribute__((target(
-    "fma"))) static inline cpp::EnableIfType<cpp::IsSame<T, double>::Value, T>
+INLINE_FMA static inline cpp::EnableIfType<cpp::IsSame<T, double>::Value, T>
 fma(T x, T y, T z) {
   double result;
   __m128d xmm = _mm_load_sd(&x);           // NOLINT
   __m128d ymm = _mm_load_sd(&y);           // NOLINT
   __m128d zmm = _mm_load_sd(&z);           // NOLINT
   __m128d r = _mm_fmadd_sd(xmm, ymm, zmm); // NOLINT
   _mm_store_sd(&result, r);                // NOLINT
   return result;
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H

libc/src/__support/FPUtil/x86_64/PolyEval.h

 #include <immintrin.h>
 
 namespace __llvm_libc {
 namespace fputil {
 
 // Cubic polynomials:
 //   polyeval(x, a0, a1, a2, a3) = a3*x^3 + a2*x^2 + a1*x + a0
 template <>
-__attribute__((target("fma"))) inline float
-polyeval(float x, float a0, float a1, float a2, float a3) {
-  __m128 xmm = _mm_set1_ps(x);                 // NOLINT
-  __m128 a13 = _mm_set_ps(0.0f, x, a3, a1);    // NOLINT
-  __m128 a02 = _mm_set_ps(0.0f, 0.0f, a2, a0); // NOLINT
+INLINE_FMA inline float polyeval(float x, float a0, float a1, float a2,
+                                 float a3) {
+  __m128 xmm = _mm_set1_ps(x);
+  __m128 a13 = _mm_set_ps(0.0f, x, a3, a1);
+  __m128 a02 = _mm_set_ps(0.0f, 0.0f, a2, a0);
   // r = (0, x^2, a3*x + a2, a1*x + a0)
   __m128 r = _mm_fmadd_ps(a13, xmm, a02); // NOLINT
   // result = (a3*x + a2) * x^2 + (a1*x + a0)
   return fma(r[2], r[1], r[0]);
 }
 
 template <>
-__attribute__((target("fma"))) inline double
-polyeval(double x, double a0, double a1, double a2, double a3) {
-  __m256d xmm = _mm256_set1_pd(x);               // NOLINT
-  __m256d a13 = _mm256_set_pd(0.0, x, a3, a1);   // NOLINT
-  __m256d a02 = _mm256_set_pd(0.0, 0.0, a2, a0); // NOLINT
+INLINE_FMA inline double polyeval(double x, double a0, double a1, double a2,
+                                  double a3) {
+  __m256d xmm = _mm256_set1_pd(x);
+  __m256d a13 = _mm256_set_pd(0.0, x, a3, a1);
+  __m256d a02 = _mm256_set_pd(0.0, 0.0, a2, a0);
   // r = (0, x^2, a3*x + a2, a1*x + a0)
   __m256d r = _mm256_fmadd_pd(a13, xmm, a02); // NOLINT
   // result = (a3*x + a2) * x^2 + (a1*x + a0)
   return fma(r[2], r[1], r[0]);
 }
 
 // Quintic polynomials:
 //   polyeval(x, a0, a1, a2, a3, a4, a5) = a5*x^5 + a4*x^4 + a3*x^3 + a2*x^2 +
 //                                         + a1*x + a0
 template <>
-__attribute__((target("fma"))) inline float
-polyeval(float x, float a0, float a1, float a2, float a3, float a4, float a5) {
-  __m128 xmm = _mm_set1_ps(x);                 // NOLINT
-  __m128 a25 = _mm_set_ps(0.0f, x, a5, a2);    // NOLINT
-  __m128 a14 = _mm_set_ps(0.0f, 0.0f, a4, a1); // NOLINT
-  __m128 a03 = _mm_set_ps(0.0f, 0.0f, a3, a0); // NOLINT
+INLINE_FMA inline float polyeval(float x, float a0, float a1, float a2,
+                                 float a3, float a4, float a5) {
+  __m128 xmm = _mm_set1_ps(x);
+  __m128 a25 = _mm_set_ps(0.0f, x, a5, a2);
+  __m128 a14 = _mm_set_ps(0.0f, 0.0f, a4, a1);
+  __m128 a03 = _mm_set_ps(0.0f, 0.0f, a3, a0);
   // r1 = (0, x^2, a5*x + a4, a2*x + a1)
   __m128 r1 = _mm_fmadd_ps(a25, xmm, a14); // NOLINT
   // r2 = (0, x^3, (a5*x + a4)*x + a3, (a2*x + a1)*x + a0
   __m128 r2 = _mm_fmadd_ps(r1, xmm, a03); // NOLINT
   // result = ((a5*x + a4)*x + a3) * x^3 + ((a2*x + a1)*x + a0)
   return fma(r2[2], r2[1], r2[0]);
 }
 
 template <>
-__attribute__((target("fma"))) inline double
-polyeval(double x, double a0, double a1, double a2, double a3, double a4,
-         double a5) {
-  __m256d xmm = _mm256_set1_pd(x);               // NOLINT
-  __m256d a25 = _mm256_set_pd(0.0, x, a5, a2);   // NOLINT
-  __m256d a14 = _mm256_set_pd(0.0, 0.0, a4, a1); // NOLINT
-  __m256d a03 = _mm256_set_pd(0.0, 0.0, a3, a0); // NOLINT
+INLINE_FMA inline double polyeval(double x, double a0, double a1, double a2,
+                                  double a3, double a4, double a5) {
+  __m256d xmm = _mm256_set1_pd(x);
+  __m256d a25 = _mm256_set_pd(0.0, x, a5, a2);
+  __m256d a14 = _mm256_set_pd(0.0, 0.0, a4, a1);
+  __m256d a03 = _mm256_set_pd(0.0, 0.0, a3, a0);
   // r1 = (0, x^2, a5*x + a4, a2*x + a1)
   __m256d r1 = _mm256_fmadd_pd(a25, xmm, a14); // NOLINT
   // r2 = (0, x^3, (a5*x + a4)*x + a3, (a2*x + a1)*x + a0
   __m256d r2 = _mm256_fmadd_pd(r1, xmm, a03); // NOLINT
   // result = ((a5*x + a4)*x + a3) * x^3 + ((a2*x + a1)*x + a0)
   return fma(r2[2], r2[1], r2[0]);
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_POLYEVAL_H

libc/src/__support/architectures.h

 #if defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64)
 #define LLVM_LIBC_ARCH_AARCH64
 #endif
 
 #if (defined(LLVM_LIBC_ARCH_AARCH64) || defined(LLVM_LIBC_ARCH_ARM))
 #define LLVM_LIBC_ARCH_ANY_ARM
 #endif
 
+#if defined(LLVM_LIBC_ARCH_X86_64)
+#define INLINE_FMA __attribute__((target("fma")))
+#else
+#define INLINE_FMA
+#endif // LLVM_LIBC_ARCH_X86_64
+
 #endif // LLVM_LIBC_SUPPORT_ARCHITECTURES_H

libc/src/math/fma.cpp

 //===-- Implementation of fma function ------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/math/fma.h"
 #include "src/__support/common.h"
 
 #include "src/__support/FPUtil/FMA.h"
 
 namespace __llvm_libc {
 
+INLINE_FMA
 LLVM_LIBC_FUNCTION(double, fma, (double x, double y, double z)) {
   return fputil::fma(x, y, z);
 }
 
 } // namespace __llvm_libc

libc/src/math/fmaf.cpp

 //===-- Implementation of fmaf function -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/math/fmaf.h"
 #include "src/__support/common.h"
 
 #include "src/__support/FPUtil/FMA.h"
 
 namespace __llvm_libc {
 
+INLINE_FMA
 LLVM_LIBC_FUNCTION(float, fmaf, (float x, float y, float z)) {
   return fputil::fma(x, y, z);
 }
 
 } // namespace __llvm_libc

libc/src/math/generic/expm1f.cpp

 // When |x| > Ln2, catastrophic cancellation does not occur with the
 // subtraction expf(x) - 1.0f, so we use it to compute expm1f(x).
 //
 // We divide [-Ln2; Ln2] into 3 subintervals [-Ln2; -1/8], [-1/8; 1/8],
 // [1/8; Ln2]. And we use a degree-6 polynomial to approximate exp(x) - 1 in
 // each interval. The coefficients were generated by Sollya's fpminmax.
 //
 // See libc/utils/mathtools/expm1f.sollya for more detail.
+INLINE_FMA
 LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
   const float ln2 =
       0.69314718055994530941723212145817656807550013436025f; // For C++17:
                                                              // 0x1.62e'42ffp-1
   float abs_x = __llvm_libc::fputil::abs(x);
 
   if (abs_x <= ln2) {
     if (abs_x <= 0.125f) {