[libc] Automatically add -mfma flag for architectures supporting FMA.

[lntue]

Summary

Detect if the architecture supports FMA instructions and if
the targets depend on fma.

[michaelrj]

libc/cmake/modules/LLVMLibCObjectRules.cmake

25

would it be possible to make this a more generic option than just FMA? Perhaps there's a way to use generator expressions?

[lntue]

Use the FLAGS option from https://reviews.llvm.org/D125174

[lntue]

Remove debugging messages.

[lntue]

libc/cmake/modules/LLVMLibCObjectRules.cmake

25

A more general option is now added with https://reviews.llvm.org/D125174.

[lntue]

Only apply FMA_OPT flags for x86-64 targets with FMA instructions.

[lntue]

Sync.

[gchatelet]

libc/cmake/modules/LLVMLibCObjectRules.cmake

13

It would make sense to factor the two conditions inside the fma variable.

libc/test/src/math/CMakeLists.txt

1204–1205 ↗

(On Diff #433280)

I'm not sure I understand the semantic here. Does that mean that the test requires FMA?
If so does it? It would make sense to test it with and without FMA.

Maybe I missed something.

[lntue]

libc/test/src/math/CMakeLists.txt

1204–1205 ↗

(On Diff #433280)

Yes, for this patch, this test does require FMA. That is because without FMA instructions, the current expm1f implementation is not correctly rounded for all float inputs (1 extra exceptional value). This will be fixed in the followup patch: https://reviews.llvm.org/D123440

[lntue]

Refactor the condition for adding fma compile flags.

[gchatelet]

libc/cmake/modules/LLVMLibCFlagRules.cmake

135 ↗

(On Diff #433374)

"Skip FMA_OPT flag for target that don't support fma"

libc/test/src/math/CMakeLists.txt

1204–1205 ↗

(On Diff #433280)

Do you mind adding a comment ?

[lntue]

Address comments.

libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake

 # ------------------------------------------------------------------------------
 # Cpu features definition and flags
 # ------------------------------------------------------------------------------
 
 # Initialize ALL_CPU_FEATURES as empty list.
 set(ALL_CPU_FEATURES "")
 
 if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
-  set(ALL_CPU_FEATURES SSE2 SSE4_2 AVX2 AVX512F)
+  set(ALL_CPU_FEATURES SSE2 SSE4_2 AVX2 AVX512F FMA)
   set(LIBC_COMPILE_OPTIONS_NATIVE -march=native)
 elseif(${LIBC_TARGET_ARCHITECTURE_IS_AARCH64})
   set(LIBC_COMPILE_OPTIONS_NATIVE -mcpu=native)
 endif()
 
 # Making sure ALL_CPU_FEATURES is sorted.
 list(SORT ALL_CPU_FEATURES)
 

 
 set(LIBC_CPU_FEATURES "" CACHE PATH "Host supported CPU features")
 
 if(CMAKE_CROSSCOMPILING)
   _intersection(cpu_features "${ALL_CPU_FEATURES}" "${LIBC_CPU_FEATURES}")
   if(NOT "${cpu_features}" STREQUAL "${LIBC_CPU_FEATURES}")
     message(FATAL_ERROR "Unsupported CPU features: ${cpu_features}")
   endif()
+  message(STATUS "Check CPU features: ${run_output}")
   set(LIBC_CPU_FEATURES "${cpu_features}")
 else()
   # Populates the LIBC_CPU_FEATURES list from host.
   try_run(
     run_result compile_result "${CMAKE_CURRENT_BINARY_DIR}/check_${feature}"
     "${CMAKE_CURRENT_BINARY_DIR}/cpu_features/check_cpu_features.cpp"
     COMPILE_DEFINITIONS ${LIBC_COMPILE_OPTIONS_NATIVE}
     COMPILE_OUTPUT_VARIABLE compile_output
     RUN_OUTPUT_VARIABLE run_output)
   if("${run_result}" EQUAL 0)
+    message(STATUS "Check CPU features: ${run_output}")
     set(LIBC_CPU_FEATURES "${run_output}")
   elseif(NOT ${compile_result})
     message(FATAL_ERROR "Failed to compile: ${compile_output}")
   else()
     message(FATAL_ERROR "Failed to run: ${run_output}")
   endif()
 endif()

libc/cmake/modules/LLVMLibCLibraryRules.cmake

 #    add_header_library(
 #      <target name>
 #      HDRS  <list of .h files part of the library>
 #      DEPENDS <list of dependencies>
 #    )
 function(add_header_library target_name)
   cmake_parse_arguments(
     "ADD_HEADER"
-    "" # No optional arguments
+    "FMA_OPT" # Use FMA if target supports
     "" # No Single value arguments
     "HDRS;DEPENDS" # Multi-value arguments
     ${ARGN}
   )
 
   if(NOT ADD_HEADER_HDRS)
     message(FATAL_ERROR "'add_header_library' target requires a HDRS list of .h files.")
   endif()
function(add_header_library target_name)
 
   set(interface_target_name "${fq_target_name}.__header_library__")
 
   add_library(${interface_target_name} INTERFACE)
   target_sources(${interface_target_name} INTERFACE ${FULL_HDR_PATHS})
   get_fq_deps_list(fq_deps_list ${ADD_HEADER_DEPENDS})
   if(ADD_HEADER_DEPENDS)
     add_dependencies(${interface_target_name} ${fq_deps_list})
+    foreach(dep IN LISTS fq_deps_list)
+      if(TARGET ${dep})
+        get_target_property(fma_opt ${dep} "FMA_OPT")
+        if(fma_opt)
+          set(ADD_HEADER_FMA_OPT TRUE)
+        endif()
+      endif()
+    endforeach()
   endif()
 
   add_custom_target(${fq_target_name})
   add_dependencies(${fq_target_name} ${interface_target_name})
   set_target_properties(
     ${fq_target_name}
     PROPERTIES
       "TARGET_TYPE" "${HDR_LIBRARY_TARGET_TYPE}"
       "DEPS" "${fq_deps_list}"
   )
+  if(ADD_HEADER_FMA_OPT)
+    message(STATUS "Header library ${fq_target_name} will enable FMA for dependecy")
+    set_target_properties(
+      ${interface_target_name}
+      PROPERTIES INTERFACE_FMA_OPT TRUE
+    )
+    set_target_properties(
+      ${fq_target_name}
+      PROPERTIES FMA_OPT TRUE
+    )
+  endif()
 endfunction(add_header_library)

libc/cmake/modules/LLVMLibCObjectRules.cmake

 set(OBJECT_LIBRARY_TARGET_TYPE "OBJECT_LIBRARY")
 
 function(_get_common_compile_options output_var)
   set(compile_options ${LLVM_CXX_STD_default} ${LIBC_COMPILE_OPTIONS_DEFAULT} ${ARGN})
   if(NOT ${LIBC_TARGET_OS} STREQUAL "windows")
     set(compile_options ${compile_options} -fpie -ffreestanding)
   endif()
   if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
     list(APPEND compile_options "-fno-exceptions")
     list(APPEND compile_options "-fno-unwind-tables")
     list(APPEND compile_options "-fno-asynchronous-unwind-tables")
     list(APPEND compile_options "-fno-rtti")
   elseif(MSVC)
     list(APPEND compile_options "/EHs-c-")
     list(APPEND compile_options "/GR-")
   endif()
   set(${output_var} ${compile_options} PARENT_SCOPE)
 endfunction()
 
 # Rule which is essentially a wrapper over add_library to compile a set of
 # sources to object files.
 # Usage:
 #     add_object_library(
 #       <target_name>
+#       FMA_OPT [optional] enable FMA if target supports.
 #       HDRS <list of header files>
 #       SRCS <list of source files>
 #       DEPENDS <list of dependencies>
 #       COMPILE_OPTIONS <optional list of special compile options for this target>
 function(add_object_library target_name)
   cmake_parse_arguments(
     "ADD_OBJECT"
-    "" # No optional arguments
+    "FMA_OPT" # Optional arguments
     "CXX_STANDARD" # Single value arguments
     "SRCS;HDRS;COMPILE_OPTIONS;DEPENDS" # Multivalue arguments
     ${ARGN}
   )
 
   if(NOT ADD_OBJECT_SRCS)
     message(FATAL_ERROR "'add_object_library' rule requires SRCS to be specified.")
   endif()
 
   get_fq_target_name(${target_name} fq_target_name)
   add_library(
     ${fq_target_name}
     EXCLUDE_FROM_ALL
     OBJECT
     ${ADD_OBJECT_SRCS}
     ${ADD_OBJECT_HDRS}
   )
   target_include_directories(
     ${fq_target_name}
     PRIVATE
       ${LIBC_BUILD_DIR}/include
       ${LIBC_SOURCE_DIR}
       ${LIBC_BUILD_DIR}
   )
-  _get_common_compile_options(compile_options ${ADD_OBJECT_COMPILE_OPTIONS})
-  target_compile_options(${fq_target_name} PRIVATE ${compile_options})
-
   get_fq_deps_list(fq_deps_list ${ADD_OBJECT_DEPENDS})
   if(fq_deps_list)
     add_dependencies(${fq_target_name} ${fq_deps_list})
+
+    foreach(dep IN LISTS ${fq_deps_list})
+      if(TARGET ${dep})
+        get_target_property(fma_opt ${dep} "FMA_OPT")
+      endif()
+      if(fma_opt)
+        set(ADD_OBJECT_FMA_OPT TRUE)
+      endif()
+    endforeach()
+
+  endif()
+
+  if(ADD_OBJECT_FMA_OPT)
+    set_target_properties(
+      ${fq_target_name}
+      PROPERTIES FMA_OPT TRUE
+    )
+    if(LIBC_CPU_FEATURES MATCHES "FMA")
+      message(STATUS "FMA is enabled for ${fq_target_name}")
+      list(APPEND ADD_OBJECT_COMPILE_OPTIONS "-mfma")
+    endif()
   endif()
 
+  _get_common_compile_options(compile_options ${ADD_OBJECT_COMPILE_OPTIONS})
+  target_compile_options(${fq_target_name} PRIVATE ${compile_options})
+
   if(ADD_OBJECT_CXX_STANDARD)
     set_target_properties(
       ${fq_target_name}
       PROPERTIES
         CXX_STANDARD ${ADD_OBJECT_CXX_STANDARD}
     )
   endif()
 
   set_target_properties(
     ${fq_target_name}
     PROPERTIES
       "TARGET_TYPE" ${OBJECT_LIBRARY_TARGET_TYPE}
       "OBJECT_FILES" "$<TARGET_OBJECTS:${fq_target_name}>"
       "DEPS" "${fq_deps_list}"
   )
 endfunction(add_object_library)
 
 set(ENTRYPOINT_OBJ_TARGET_TYPE "ENTRYPOINT_OBJ")
 
 # A rule for entrypoint object targets.
 # Usage:
 #     add_entrypoint_object(
 #       <target_name>
-#       [ALIAS|REDIRECTED] # Specified if the entrypoint is redirected or an alias.
+#       [ALIAS|REDIRECTED|FMA_OPT] # Specified if the entrypoint is redirected or an alias,
+#                                  # or it can take advantage of FMA instructions.
 #       [NAME] <the C name of the entrypoint if different from target_name>
 #       SRCS <list of .cpp files>
 #       HDRS <list of .h files>
 #       DEPENDS <list of dependencies>
 #       COMPILE_OPTIONS <optional list of special compile options for this target>
 #       SPECIAL_OBJECTS <optional list of special object targets added by the rule `add_object`>
 #     )
 function(add_entrypoint_object target_name)
   cmake_parse_arguments(
     "ADD_ENTRYPOINT_OBJ"
-    "ALIAS;REDIRECTED" # Optional argument
+    "ALIAS;REDIRECTED;FMA_OPT" # Optional argument
     "NAME;CXX_STANDARD" # Single value arguments
     "SRCS;HDRS;DEPENDS;COMPILE_OPTIONS"  # Multi value arguments
     ${ARGN}
   )
 
   get_fq_target_name(${target_name} fq_target_name)
   set(entrypoint_name ${target_name})
   if(ADD_ENTRYPOINT_OBJ_NAME)
function(add_entrypoint_object target_name)
 
   if(NOT ADD_ENTRYPOINT_OBJ_SRCS)
     message(FATAL_ERROR "`add_entrypoint_object` rule requires SRCS to be specified.")
   endif()
   if(NOT ADD_ENTRYPOINT_OBJ_HDRS)
     message(FATAL_ERROR "`add_entrypoint_object` rule requires HDRS to be specified.")
   endif()
 
-  _get_common_compile_options(common_compile_options ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS})
   set(internal_target_name ${fq_target_name}.__internal__)
   set(include_dirs ${LIBC_BUILD_DIR}/include ${LIBC_SOURCE_DIR} ${LIBC_BUILD_DIR})
   get_fq_deps_list(fq_deps_list ${ADD_ENTRYPOINT_OBJ_DEPENDS})
   set(full_deps_list ${fq_deps_list} libc.src.__support.common)
 
+  foreach(dep IN LISTS fq_deps_list)
+    if(TARGET ${dep})
+      get_target_property(fma_opt ${dep} "FMA_OPT")
+    endif()
+    if(fma_opt)
+      set(ADD_ENTRYPOINT_OBJ_FMA_OPT TRUE)
+    endif()
+  endforeach()
+
+  if(ADD_ENTRYPOINT_OBJ_FMA_OPT AND (LIBC_CPU_FEATURES MATCHES "FMA"))
+    message(STATUS "FMA is enabled for ${entrypoint_name}")
+    list(APPEND ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS "-mfma")
+  endif()
+
+  _get_common_compile_options(common_compile_options ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS})
+
   add_library(
     ${internal_target_name}
     # TODO: We don't need an object library for internal consumption.
     # A future change should switch this to a normal static library.
     EXCLUDE_FROM_ALL
     OBJECT
     ${ADD_ENTRYPOINT_OBJ_SRCS}
     ${ADD_ENTRYPOINT_OBJ_HDRS}
function(add_entrypoint_object target_name)
   if(ADD_ENTRYPOINT_OBJ_CXX_STANDARD)
     set_target_properties(
       ${fq_target_name} ${internal_target_name}
       PROPERTIES
         CXX_STANDARD ${ADD_ENTRYPOINT_OBJ_CXX_STANDARD}
     )
   endif()
 
+  if(ADD_ENTRYPOINT_OBJ_FMA_OPT)
+    set_target_properties(
+      ${fq_target_name}
+      PROPERTIES FMA_OPT TRUE
+    )
+  endif()
+
   set_target_properties(
     ${fq_target_name}
     PROPERTIES
       "ENTRYPOINT_NAME" ${entrypoint_name}
       "TARGET_TYPE" ${ENTRYPOINT_OBJ_TARGET_TYPE}
       "OBJECT_FILE" $<TARGET_OBJECTS:${fq_target_name}>
       # TODO: We don't need to list internal object files if the internal
       # target is a normal static library.

libc/src/__support/FPUtil/CMakeLists.txt

HDRS
     sqrt.h
   DEPENDS
     .fputil
     libc.src.__support.FPUtil.generic.sqrt
 )
 
 add_header_library(
   fma
+  FMA_OPT
   HDRS
     FMA.h
   DEPENDS
     .fputil
     libc.src.__support.FPUtil.generic.fma
 )
 
 add_header_library(

libc/src/__support/FPUtil/PolyEval.h

 //   polyeval( x, 4.0, 3.0, 2.0, 1.0 )
 
 namespace __llvm_libc {
 namespace fputil {
 
 template <typename T> static inline T polyeval(T x, T a0) { return a0; }
 
 template <typename T, typename... Ts>
-INLINE_FMA static inline T polyeval(T x, T a0, Ts... a) {
+static inline T polyeval(T x, T a0, Ts... a) {
   return multiply_add(x, polyeval(x, a...), a0);
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_POLYEVAL_H

libc/src/__support/FPUtil/x86_64/FMA.h

 
 #include "src/__support/CPP/TypeTraits.h"
 #include <immintrin.h>
 
 namespace __llvm_libc {
 namespace fputil {
 
 template <typename T>
-INLINE_FMA static inline cpp::EnableIfType<cpp::IsSame<T, float>::Value, T>
-fma(T x, T y, T z) {
+static inline cpp::EnableIfType<cpp::IsSame<T, float>::Value, T> fma(T x, T y,
+                                                                     T z) {
   float result;
   __m128 xmm = _mm_load_ss(&x);           // NOLINT
   __m128 ymm = _mm_load_ss(&y);           // NOLINT
   __m128 zmm = _mm_load_ss(&z);           // NOLINT
   __m128 r = _mm_fmadd_ss(xmm, ymm, zmm); // NOLINT
   _mm_store_ss(&result, r);               // NOLINT
   return result;
 }
 
 template <typename T>
-INLINE_FMA static inline cpp::EnableIfType<cpp::IsSame<T, double>::Value, T>
-fma(T x, T y, T z) {
+static inline cpp::EnableIfType<cpp::IsSame<T, double>::Value, T> fma(T x, T y,
+                                                                      T z) {
   double result;
   __m128d xmm = _mm_load_sd(&x);           // NOLINT
   __m128d ymm = _mm_load_sd(&y);           // NOLINT
   __m128d zmm = _mm_load_sd(&z);           // NOLINT
   __m128d r = _mm_fmadd_sd(xmm, ymm, zmm); // NOLINT
   _mm_store_sd(&result, r);                // NOLINT
   return result;
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H

libc/src/__support/FPUtil/x86_64/PolyEval.h

 #include <immintrin.h>
 
 namespace __llvm_libc {
 namespace fputil {
 
 // Cubic polynomials:
 //   polyeval(x, a0, a1, a2, a3) = a3*x^3 + a2*x^2 + a1*x + a0
 template <>
-INLINE_FMA inline float polyeval(float x, float a0, float a1, float a2,
-                                 float a3) {
+inline float polyeval(float x, float a0, float a1, float a2, float a3) {
   __m128 xmm = _mm_set1_ps(x);                 // NOLINT
   __m128 a13 = _mm_set_ps(0.0f, x, a3, a1);    // NOLINT
   __m128 a02 = _mm_set_ps(0.0f, 0.0f, a2, a0); // NOLINT
   // r = (0, x^2, a3*x + a2, a1*x + a0)
   __m128 r = _mm_fmadd_ps(a13, xmm, a02); // NOLINT
   // result = (a3*x + a2) * x^2 + (a1*x + a0)
   return fma(r[2], r[1], r[0]);
 }
 
 template <>
-INLINE_FMA inline double polyeval(double x, double a0, double a1, double a2,
-                                  double a3) {
+inline double polyeval(double x, double a0, double a1, double a2, double a3) {
   __m256d xmm = _mm256_set1_pd(x);               // NOLINT
   __m256d a13 = _mm256_set_pd(0.0, x, a3, a1);   // NOLINT
   __m256d a02 = _mm256_set_pd(0.0, 0.0, a2, a0); // NOLINT
   // r = (0, x^2, a3*x + a2, a1*x + a0)
   __m256d r = _mm256_fmadd_pd(a13, xmm, a02); // NOLINT
   // result = (a3*x + a2) * x^2 + (a1*x + a0)
   return fma(r[2], r[1], r[0]);
 }
 
 // Quintic polynomials:
 //   polyeval(x, a0, a1, a2, a3, a4, a5) = a5*x^5 + a4*x^4 + a3*x^3 + a2*x^2 +
 //                                         + a1*x + a0
 template <>
-INLINE_FMA inline float polyeval(float x, float a0, float a1, float a2,
-                                 float a3, float a4, float a5) {
+inline float polyeval(float x, float a0, float a1, float a2, float a3, float a4,
+                      float a5) {
   __m128 xmm = _mm_set1_ps(x);                 // NOLINT
   __m128 a25 = _mm_set_ps(0.0f, x, a5, a2);    // NOLINT
   __m128 a14 = _mm_set_ps(0.0f, 0.0f, a4, a1); // NOLINT
   __m128 a03 = _mm_set_ps(0.0f, 0.0f, a3, a0); // NOLINT
   // r1 = (0, x^2, a5*x + a4, a2*x + a1)
   __m128 r1 = _mm_fmadd_ps(a25, xmm, a14); // NOLINT
   // r2 = (0, x^3, (a5*x + a4)*x + a3, (a2*x + a1)*x + a0
   __m128 r2 = _mm_fmadd_ps(r1, xmm, a03); // NOLINT
   // result = ((a5*x + a4)*x + a3) * x^3 + ((a2*x + a1)*x + a0)
   return fma(r2[2], r2[1], r2[0]);
 }
 
 template <>
-INLINE_FMA inline double polyeval(double x, double a0, double a1, double a2,
-                                  double a3, double a4, double a5) {
+inline double polyeval(double x, double a0, double a1, double a2, double a3,
+                       double a4, double a5) {
   __m256d xmm = _mm256_set1_pd(x);               // NOLINT
   __m256d a25 = _mm256_set_pd(0.0, x, a5, a2);   // NOLINT
   __m256d a14 = _mm256_set_pd(0.0, 0.0, a4, a1); // NOLINT
   __m256d a03 = _mm256_set_pd(0.0, 0.0, a3, a0); // NOLINT
   // r1 = (0, x^2, a5*x + a4, a2*x + a1)
   __m256d r1 = _mm256_fmadd_pd(a25, xmm, a14); // NOLINT
   // r2 = (0, x^3, (a5*x + a4)*x + a3, (a2*x + a1)*x + a0
   __m256d r2 = _mm256_fmadd_pd(r1, xmm, a03); // NOLINT
   // result = ((a5*x + a4)*x + a3) * x^3 + ((a2*x + a1)*x + a0)
   return fma(r2[2], r2[1], r2[0]);
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_POLYEVAL_H

libc/src/__support/architectures.h

 #if defined(LLVM_LIBC_ARCH_AARCH64)
 #define LIBC_TARGET_HAS_FMA
 #elif defined(LLVM_LIBC_ARCH_X86_64)
 #if (defined(__AVX2__) || defined(__FMA__))
 #define LIBC_TARGET_HAS_FMA
 #endif
 #endif
 
-#if (defined(LLVM_LIBC_ARCH_X86_64) && defined(LIBC_TARGET_HAS_FMA))
-#define INLINE_FMA __attribute__((target("fma")))
-#else
-#define INLINE_FMA
-#endif // LLVM_LIBC_ARCH_X86_64
-
 #endif // LLVM_LIBC_SUPPORT_ARCHITECTURES_H

libc/src/math/CMakeLists.txt

SRCS
     fmaf.cpp
   HDRS
     fmaf.h
   DEPENDS
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.fma
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   fma
   SRCS
     fma.cpp
   HDRS
     fma.h
   DEPENDS
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.fma
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_math_entrypoint_object(ceil)
 add_math_entrypoint_object(ceilf)
 add_math_entrypoint_object(ceill)
 
 add_math_entrypoint_object(copysign)
 add_math_entrypoint_object(copysignf)

libc/src/math/fma.cpp

 //===-- Implementation of fma function ------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/math/fma.h"
 #include "src/__support/common.h"
 
 #include "src/__support/FPUtil/FMA.h"
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(double, fma, (double x, double y, double z)) {
   return fputil::fma(x, y, z);
 }
 
 } // namespace __llvm_libc

libc/src/math/fmaf.cpp

 //===-- Implementation of fmaf function -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/math/fmaf.h"
 #include "src/__support/common.h"
 
 #include "src/__support/FPUtil/FMA.h"
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, fmaf, (float x, float y, float z)) {
   return fputil::fma(x, y, z);
 }
 
 } // namespace __llvm_libc

libc/src/math/generic/CMakeLists.txt

HDRS
     ../expf.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.polyeval
     libc.include.math
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   exp2f
   SRCS
     exp2f.cpp
   HDRS
     ../exp2f.h
   DEPENDS
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.polyeval
     libc.include.math
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   expm1f
   SRCS
     expm1f.cpp
   HDRS
     ../expm1f.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.include.math
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   copysign
   SRCS
     copysign.cpp
   HDRS
     ../copysign.h
HDRS
     ../log10f.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   log1pf
   SRCS
     log1pf.cpp
   HDRS
     ../log1pf.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   log2f
   SRCS
     log2f.cpp
   HDRS
     ../log2f.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.polyeval
     COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   logf
   SRCS
     logf.cpp
   HDRS
     ../logf.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   logb
   SRCS
     logb.cpp
   HDRS
     ../logb.h

libc/src/math/generic/exp2f.cpp

static constexpr double EXP_M[64] = {
     0x1.ae89f995ad3adp0, 0x1.b33a2b84f15fbp0, 0x1.b7f76f2fb5e47p0,
     0x1.bcc1e904bc1d2p0, 0x1.c199bdd85529cp0, 0x1.c67f12e57d14bp0,
     0x1.cb720dcef9069p0, 0x1.d072d4a07897cp0, 0x1.d5818dcfba487p0,
     0x1.da9e603db3285p0, 0x1.dfc97337b9b5fp0, 0x1.e502ee78b3ff6p0,
     0x1.ea4afa2a490dap0, 0x1.efa1bee615a27p0, 0x1.f50765b6e4540p0,
     0x1.fa7c1819e90d8p0,
 };
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, exp2f, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
   // Exceptional values.

libc/src/math/generic/expf.cpp

 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/common.h"
 
 #include <errno.h>
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, expf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
   // Exceptional values

libc/src/math/generic/expm1f.cpp

 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/common.h"
 
 #include <errno.h>
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
   // Exceptional value

libc/src/math/generic/log10f.cpp

static constexpr double LOG10_F[128] = {
     0x1.1014319e661bdp-2, 0x1.11f594839a5bdp-2, 0x1.13d4f0862b2e1p-2,
     0x1.15b24a0004a92p-2, 0x1.178da53d1ee01p-2, 0x1.1967067bb94b8p-2,
     0x1.1b3e71ec94f7bp-2, 0x1.1d13ebb32d7f9p-2, 0x1.1ee777e5f0dc3p-2,
     0x1.20b91a8e76105p-2, 0x1.2288d7a9b2b64p-2, 0x1.2456b3282f786p-2,
     0x1.2622b0ee3b79dp-2, 0x1.27ecd4d41eb67p-2, 0x1.29b522a64b609p-2,
     0x1.2b7b9e258e422p-2, 0x1.2d404b073e27ep-2, 0x1.2f032cf56a5bep-2,
     0x1.30c4478f0835fp-2, 0x1.32839e681fc62p-2};
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, log10f, (float x)) {
   constexpr double LOG10_2 = 0x1.34413509f79ffp-2;
 
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   double m = 0.0;
 
   // Exact powers of 10 and other hard-to-round cases.

libc/src/math/generic/log1pf.cpp

 // generated with Sollya using the following command:
 //   fpminimax(log(1 + x)/x, 5, [|D...|], [-2^-8; 2^-8]);
 
 namespace __llvm_libc {
 
 namespace internal {
 
 // We don't need to treat denormal
-INLINE_FMA static inline float log(double x) {
+static inline float log(double x) {
   constexpr double LOG_2 = 0x1.62e42fefa39efp-1;
 
   using FPBits = typename fputil::FPBits<double>;
   FPBits xbits(x);
 
   if (xbits.is_zero()) {
     return static_cast<float>(fputil::FPBits<float>::neg_inf());
   }
double r = fputil::polyeval(d, extra_factor, 0x1.fffffffffffacp-1,
                               -0x1.fffffffef9cb2p-2, 0x1.5555513bc679ap-2,
                               -0x1.fff4805ea441p-3, 0x1.930180dbde91ap-3);
 
   return static_cast<float>(r);
 }
 
 } // namespace internal
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, log1pf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   double xd = static_cast<double>(x);
 
   if (xbits.get_exponent() >= -8) {
     // Hard-to-round cases.
     switch (xbits.uintval()) {

libc/src/math/generic/log2f.cpp

static constexpr double LOG2_F[128] = {
     0x1.c3e9ca2e1a055p-1, 0x1.c7095ae91e1c7p-1, 0x1.ca258dca93316p-1,
     0x1.cd3e6a0ca8907p-1, 0x1.d053f6d260896p-1, 0x1.d3663b27f31d5p-1,
     0x1.d6753e032ea0fp-1, 0x1.d9810643d6615p-1, 0x1.dc899ab3ff56cp-1,
     0x1.df8f02086af2cp-1, 0x1.e29142e0e0140p-1, 0x1.e59063c8822cep-1,
     0x1.e88c6b3626a73p-1, 0x1.eb855f8ca88fbp-1, 0x1.ee7b471b3a950p-1,
     0x1.f16e281db7630p-1, 0x1.f45e08bcf0655p-1, 0x1.f74aef0efafaep-1,
     0x1.fa34e1177c233p-1, 0x1.fd1be4c7f2af9p-1};
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, log2f, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   int m = 0;
 
   // Hard to round value(s).
   switch (FPBits(x).uintval()) {
   case 0x3f81d0b5U: {

libc/src/math/generic/logf.cpp

 // Correctly Rounded Results of an Elementary Function for Multiple
 // Representations and Rounding Modes", Proceedings of the 49th ACM SIGPLAN
 // Symposium on Principles of Programming Languages (POPL-2022), Philadelphia,
 // USA, January 16-22, 2022.
 // https://people.cs.rutgers.edu/~sn349/papers/rlibmall-popl-2022.pdf
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, logf, (float x)) {
   constexpr double LOG_2 = 0x1.62e42fefa39efp-1;
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   switch (FPBits(x).uintval()) {
   case 0x41178febU: // x = 0x1.2f1fd6p+3f
     if (fputil::get_round() == FE_TONEAREST)