[libc] Automatically add -mfma flag for architectures supporting FMA.

[lntue]

Summary

Detect if the architecture supports FMA instructions and if
the targets depend on fma.

[michaelrj]

libc/cmake/modules/LLVMLibCObjectRules.cmake

39

would it be possible to make this a more generic option than just FMA? Perhaps there's a way to use generator expressions?

[lntue]

Use the FLAGS option from https://reviews.llvm.org/D125174

[lntue]

Remove debugging messages.

[lntue]

libc/cmake/modules/LLVMLibCObjectRules.cmake

39

A more general option is now added with https://reviews.llvm.org/D125174.

[lntue]

Only apply FMA_OPT flags for x86-64 targets with FMA instructions.

[lntue]

Sync.

[gchatelet]

libc/cmake/modules/LLVMLibCObjectRules.cmake

21

It would make sense to factor the two conditions inside the fma variable.

libc/test/src/math/CMakeLists.txt

1207–1208

I'm not sure I understand the semantic here. Does that mean that the test requires FMA?
If so does it? It would make sense to test it with and without FMA.

Maybe I missed something.

[lntue]

libc/test/src/math/CMakeLists.txt

1207–1208

Yes, for this patch, this test does require FMA. That is because without FMA instructions, the current expm1f implementation is not correctly rounded for all float inputs (1 extra exceptional value). This will be fixed in the followup patch: https://reviews.llvm.org/D123440

[lntue]

Refactor the condition for adding fma compile flags.

[gchatelet]

libc/cmake/modules/LLVMLibCFlagRules.cmake

135

"Skip FMA_OPT flag for target that don't support fma"

libc/test/src/math/CMakeLists.txt

1207–1208

Do you mind adding a comment ?

[lntue]

Address comments.

libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake

 # ------------------------------------------------------------------------------
 # Cpu features definition and flags
 # ------------------------------------------------------------------------------
 
 # Initialize ALL_CPU_FEATURES as empty list.
 set(ALL_CPU_FEATURES "")
 
 if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
-  set(ALL_CPU_FEATURES SSE2 SSE4_2 AVX2 AVX512F)
+  set(ALL_CPU_FEATURES SSE2 SSE4_2 AVX2 AVX512F FMA)
   set(LIBC_COMPILE_OPTIONS_NATIVE -march=native)
 elseif(${LIBC_TARGET_ARCHITECTURE_IS_AARCH64})
   set(LIBC_COMPILE_OPTIONS_NATIVE -mcpu=native)
 endif()
 
 # Making sure ALL_CPU_FEATURES is sorted.
 list(SORT ALL_CPU_FEATURES)
 

 
 set(LIBC_CPU_FEATURES "" CACHE PATH "Host supported CPU features")
 
 if(CMAKE_CROSSCOMPILING)
   _intersection(cpu_features "${ALL_CPU_FEATURES}" "${LIBC_CPU_FEATURES}")
   if(NOT "${cpu_features}" STREQUAL "${LIBC_CPU_FEATURES}")
     message(FATAL_ERROR "Unsupported CPU features: ${cpu_features}")
   endif()
+  message(STATUS "Set CPU features: ${cpu_features}")
   set(LIBC_CPU_FEATURES "${cpu_features}")
 else()
   # Populates the LIBC_CPU_FEATURES list from host.
   try_run(
     run_result compile_result "${CMAKE_CURRENT_BINARY_DIR}/check_${feature}"
     "${CMAKE_CURRENT_BINARY_DIR}/cpu_features/check_cpu_features.cpp"
     COMPILE_DEFINITIONS ${LIBC_COMPILE_OPTIONS_NATIVE}
     COMPILE_OUTPUT_VARIABLE compile_output
     RUN_OUTPUT_VARIABLE run_output)
   if("${run_result}" EQUAL 0)
+    message(STATUS "Set CPU features: ${run_output}")
     set(LIBC_CPU_FEATURES "${run_output}")
   elseif(NOT ${compile_result})
     message(FATAL_ERROR "Failed to compile: ${compile_output}")
   else()
     message(FATAL_ERROR "Failed to run: ${run_output}")
   endif()
 endif()

libc/cmake/modules/LLVMLibCFlagRules.cmake

else()
       list(APPEND fq_dep_no_flag_list ${fq_dep_name})
     endif()
   endforeach(dep)
   set(${output_list} ${fq_dep_no_flag_list} PARENT_SCOPE)
 endfunction(get_fq_dep_list_without_flag)
 
 # Special flags
 set(FMA_OPT_FLAG "FMA_OPT")
+
+# Skip FMA_OPT flag for targets that don't support fma.
+if(NOT(LIBC_TARGET_ARCHITECTURE_IS_X86 AND (LIBC_CPU_FEATURES MATCHES "FMA")))
+  set(SKIP_FLAG_EXPANSION_FMA_OPT TRUE)
+endif()

libc/cmake/modules/LLVMLibCObjectRules.cmake

 set(OBJECT_LIBRARY_TARGET_TYPE "OBJECT_LIBRARY")
 
-function(_get_common_compile_options output_var)
+function(_get_common_compile_options output_var flags)
+  list(FIND flags ${FMA_OPT_FLAG} fma)
+  if(${fma} LESS 0)
+    list(FIND flags "${FMA_OPT_FLAG}__ONLY" fma)
+  endif()
+  if((${fma} GREATER -1) AND (LIBC_CPU_FEATURES MATCHES "FMA"))
+    set(ADD_FMA_FLAG TRUE)
+  endif()
+
   set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT} ${ARGN})
   if(NOT ${LIBC_TARGET_OS} STREQUAL "windows")
     set(compile_options ${compile_options} -fpie -ffreestanding -fno-builtin)
   endif()
   if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
     list(APPEND compile_options "-fno-exceptions")
     list(APPEND compile_options "-fno-unwind-tables")
     list(APPEND compile_options "-fno-asynchronous-unwind-tables")
     list(APPEND compile_options "-fno-rtti")
+    if(ADD_FMA_FLAG)
+      list(APPEND compile_options "-mfma")
+    endif()
   elseif(MSVC)
     list(APPEND compile_options "/EHs-c-")
     list(APPEND compile_options "/GR-")
+    if(ADD_FMA_FLAG)
+      list(APPEND compile_options "/arch:AVX2")
+    endif()
   endif()
   set(${output_var} ${compile_options} PARENT_SCOPE)
 endfunction()
 
 # Rule which is essentially a wrapper over add_library to compile a set of
 # sources to object files.
 # Usage:
 #     add_object_library(
 #       <target_name>
 #       HDRS <list of header files>
 #       SRCS <list of source files>
 #       DEPENDS <list of dependencies>
 #       COMPILE_OPTIONS <optional list of special compile options for this target>
 #       FLAGS <optional list of flags>
 function(create_object_library fq_target_name)
   cmake_parse_arguments(
     "ADD_OBJECT"
     "" # No optional arguments
function(create_object_library fq_target_name)
   )
   target_include_directories(
     ${fq_target_name}
     PRIVATE
       ${LIBC_BUILD_DIR}/include
       ${LIBC_SOURCE_DIR}
       ${LIBC_BUILD_DIR}
   )
-  _get_common_compile_options(compile_options ${ADD_OBJECT_COMPILE_OPTIONS})
+  _get_common_compile_options(
+    compile_options
+    "${ADD_OBJECT_FLAGS}"
+    ${ADD_OBJECT_COMPILE_OPTIONS}
+  )
   target_compile_options(${fq_target_name} PRIVATE ${compile_options})
 
   get_fq_deps_list(fq_deps_list ${ADD_OBJECT_DEPENDS})
 
   if(SHOW_INTERMEDIATE_OBJECTS)
     message(STATUS "Adding object library ${fq_target_name}")
     if(${SHOW_INTERMEDIATE_OBJECTS} STREQUAL "DEPS")
       foreach(dep IN LISTS ADD_OBJECT_DEPENDS)
function(create_entrypoint_object fq_target_name)
   endif()
   if(NOT ADD_ENTRYPOINT_OBJ_HDRS)
     message(FATAL_ERROR "`add_entrypoint_object` rule requires HDRS to be specified.")
   endif()
   if(NOT ADD_ENTRYPOINT_OBJ_CXX_STANDARD)
     set(ADD_ENTRYPOINT_OBJ_CXX_STANDARD ${CMAKE_CXX_STANDARD})
   endif()
 
-  _get_common_compile_options(common_compile_options ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS})
+  _get_common_compile_options(
+    common_compile_options
+    "${ADD_ENTRYPOINT_OBJ_FLAGS}"
+    ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS}
+  )
   set(internal_target_name ${fq_target_name}.__internal__)
   set(include_dirs ${LIBC_BUILD_DIR}/include ${LIBC_SOURCE_DIR} ${LIBC_BUILD_DIR})
   get_fq_deps_list(fq_deps_list ${ADD_ENTRYPOINT_OBJ_DEPENDS})
   set(full_deps_list ${fq_deps_list} libc.src.__support.common)
 
   if(SHOW_INTERMEDIATE_OBJECTS)
     message(STATUS "Adding entrypoint object ${fq_target_name}")
     if(${SHOW_INTERMEDIATE_OBJECTS} STREQUAL "DEPS")

libc/src/__support/FPUtil/CMakeLists.txt

 
 add_header_library(
   fma
   HDRS
     FMA.h
   DEPENDS
     .fputil
     libc.src.__support.FPUtil.generic.fma
+  FLAGS
+    FMA_OPT
 )
 
 add_header_library(
   multiply_add
   HDRS
     multiply_add.h
   DEPENDS
     .fma

libc/src/__support/FPUtil/PolyEval.h

 //   polyeval( x, 4.0, 3.0, 2.0, 1.0 )
 
 namespace __llvm_libc {
 namespace fputil {
 
 template <typename T> static inline T polyeval(T x, T a0) { return a0; }
 
 template <typename T, typename... Ts>
-INLINE_FMA static inline T polyeval(T x, T a0, Ts... a) {
+static inline T polyeval(T x, T a0, Ts... a) {
   return multiply_add(x, polyeval(x, a...), a0);
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_POLYEVAL_H

libc/src/__support/FPUtil/x86_64/FMA.h

 
 #include "src/__support/CPP/TypeTraits.h"
 #include <immintrin.h>
 
 namespace __llvm_libc {
 namespace fputil {
 
 template <typename T>
-INLINE_FMA static inline cpp::EnableIfType<cpp::IsSame<T, float>::Value, T>
-fma(T x, T y, T z) {
+static inline cpp::EnableIfType<cpp::IsSame<T, float>::Value, T> fma(T x, T y,
+                                                                     T z) {
   float result;
   __m128 xmm = _mm_load_ss(&x);           // NOLINT
   __m128 ymm = _mm_load_ss(&y);           // NOLINT
   __m128 zmm = _mm_load_ss(&z);           // NOLINT
   __m128 r = _mm_fmadd_ss(xmm, ymm, zmm); // NOLINT
   _mm_store_ss(&result, r);               // NOLINT
   return result;
 }
 
 template <typename T>
-INLINE_FMA static inline cpp::EnableIfType<cpp::IsSame<T, double>::Value, T>
-fma(T x, T y, T z) {
+static inline cpp::EnableIfType<cpp::IsSame<T, double>::Value, T> fma(T x, T y,
+                                                                      T z) {
   double result;
   __m128d xmm = _mm_load_sd(&x);           // NOLINT
   __m128d ymm = _mm_load_sd(&y);           // NOLINT
   __m128d zmm = _mm_load_sd(&z);           // NOLINT
   __m128d r = _mm_fmadd_sd(xmm, ymm, zmm); // NOLINT
   _mm_store_sd(&result, r);                // NOLINT
   return result;
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H

libc/src/__support/FPUtil/x86_64/PolyEval.h

 #include <immintrin.h>
 
 namespace __llvm_libc {
 namespace fputil {
 
 // Cubic polynomials:
 //   polyeval(x, a0, a1, a2, a3) = a3*x^3 + a2*x^2 + a1*x + a0
 template <>
-INLINE_FMA inline float polyeval(float x, float a0, float a1, float a2,
-                                 float a3) {
+inline float polyeval(float x, float a0, float a1, float a2, float a3) {
   __m128 xmm = _mm_set1_ps(x);                 // NOLINT
   __m128 a13 = _mm_set_ps(0.0f, x, a3, a1);    // NOLINT
   __m128 a02 = _mm_set_ps(0.0f, 0.0f, a2, a0); // NOLINT
   // r = (0, x^2, a3*x + a2, a1*x + a0)
   __m128 r = _mm_fmadd_ps(a13, xmm, a02); // NOLINT
   // result = (a3*x + a2) * x^2 + (a1*x + a0)
   return fma(r[2], r[1], r[0]);
 }
 
 template <>
-INLINE_FMA inline double polyeval(double x, double a0, double a1, double a2,
-                                  double a3) {
+inline double polyeval(double x, double a0, double a1, double a2, double a3) {
   __m256d xmm = _mm256_set1_pd(x);               // NOLINT
   __m256d a13 = _mm256_set_pd(0.0, x, a3, a1);   // NOLINT
   __m256d a02 = _mm256_set_pd(0.0, 0.0, a2, a0); // NOLINT
   // r = (0, x^2, a3*x + a2, a1*x + a0)
   __m256d r = _mm256_fmadd_pd(a13, xmm, a02); // NOLINT
   // result = (a3*x + a2) * x^2 + (a1*x + a0)
   return fma(r[2], r[1], r[0]);
 }
 
 // Quintic polynomials:
 //   polyeval(x, a0, a1, a2, a3, a4, a5) = a5*x^5 + a4*x^4 + a3*x^3 + a2*x^2 +
 //                                         + a1*x + a0
 template <>
-INLINE_FMA inline float polyeval(float x, float a0, float a1, float a2,
-                                 float a3, float a4, float a5) {
+inline float polyeval(float x, float a0, float a1, float a2, float a3, float a4,
+                      float a5) {
   __m128 xmm = _mm_set1_ps(x);                 // NOLINT
   __m128 a25 = _mm_set_ps(0.0f, x, a5, a2);    // NOLINT
   __m128 a14 = _mm_set_ps(0.0f, 0.0f, a4, a1); // NOLINT
   __m128 a03 = _mm_set_ps(0.0f, 0.0f, a3, a0); // NOLINT
   // r1 = (0, x^2, a5*x + a4, a2*x + a1)
   __m128 r1 = _mm_fmadd_ps(a25, xmm, a14); // NOLINT
   // r2 = (0, x^3, (a5*x + a4)*x + a3, (a2*x + a1)*x + a0
   __m128 r2 = _mm_fmadd_ps(r1, xmm, a03); // NOLINT
   // result = ((a5*x + a4)*x + a3) * x^3 + ((a2*x + a1)*x + a0)
   return fma(r2[2], r2[1], r2[0]);
 }
 
 template <>
-INLINE_FMA inline double polyeval(double x, double a0, double a1, double a2,
-                                  double a3, double a4, double a5) {
+inline double polyeval(double x, double a0, double a1, double a2, double a3,
+                       double a4, double a5) {
   __m256d xmm = _mm256_set1_pd(x);               // NOLINT
   __m256d a25 = _mm256_set_pd(0.0, x, a5, a2);   // NOLINT
   __m256d a14 = _mm256_set_pd(0.0, 0.0, a4, a1); // NOLINT
   __m256d a03 = _mm256_set_pd(0.0, 0.0, a3, a0); // NOLINT
   // r1 = (0, x^2, a5*x + a4, a2*x + a1)
   __m256d r1 = _mm256_fmadd_pd(a25, xmm, a14); // NOLINT
   // r2 = (0, x^3, (a5*x + a4)*x + a3, (a2*x + a1)*x + a0
   __m256d r2 = _mm256_fmadd_pd(r1, xmm, a03); // NOLINT
   // result = ((a5*x + a4)*x + a3) * x^3 + ((a2*x + a1)*x + a0)
   return fma(r2[2], r2[1], r2[0]);
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_POLYEVAL_H

libc/src/__support/architectures.h

 #if defined(LLVM_LIBC_ARCH_AARCH64)
 #define LIBC_TARGET_HAS_FMA
 #elif defined(LLVM_LIBC_ARCH_X86_64)
 #if (defined(__AVX2__) || defined(__FMA__))
 #define LIBC_TARGET_HAS_FMA
 #endif
 #endif
 
-#if (defined(LLVM_LIBC_ARCH_X86_64) && defined(LIBC_TARGET_HAS_FMA))
-#define INLINE_FMA __attribute__((target("fma")))
-#else
-#define INLINE_FMA
-#endif // LLVM_LIBC_ARCH_X86_64
-
 #endif // LLVM_LIBC_SUPPORT_ARCHITECTURES_H

libc/src/math/CMakeLists.txt

SRCS
     fmaf.cpp
   HDRS
     fmaf.h
   DEPENDS
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.fma
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   fma
   SRCS
     fma.cpp
   HDRS
     fma.h
   DEPENDS
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.fma
   COMPILE_OPTIONS
     -O3
-    -mfma
+  FLAGS
+    FMA_OPT__ONLY
 )
 
 add_math_entrypoint_object(ceil)
 add_math_entrypoint_object(ceilf)
 add_math_entrypoint_object(ceill)
 
 add_math_entrypoint_object(copysign)
 add_math_entrypoint_object(copysignf)

libc/src/math/fma.cpp

 //===-- Implementation of fma function ------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/math/fma.h"
 #include "src/__support/common.h"
 
 #include "src/__support/FPUtil/FMA.h"
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(double, fma, (double x, double y, double z)) {
   return fputil::fma(x, y, z);
 }
 
 } // namespace __llvm_libc

libc/src/math/fmaf.cpp

 //===-- Implementation of fmaf function -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/math/fmaf.h"
 #include "src/__support/common.h"
 
 #include "src/__support/FPUtil/FMA.h"
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, fmaf, (float x, float y, float z)) {
   return fputil::fma(x, y, z);
 }
 
 } // namespace __llvm_libc

libc/src/math/generic/CMakeLists.txt

HDRS
     ../expf.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.polyeval
     libc.include.math
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   exp2f
   SRCS
     exp2f.cpp
   HDRS
     ../exp2f.h
   DEPENDS
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.polyeval
     libc.include.math
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   expm1f
   SRCS
     expm1f.cpp
   HDRS
     ../expm1f.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.include.math
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   copysign
   SRCS
     copysign.cpp
   HDRS
     ../copysign.h
HDRS
     ../log10f.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   log1pf
   SRCS
     log1pf.cpp
   HDRS
     ../log1pf.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   log2f
   SRCS
     log2f.cpp
   HDRS
     ../log2f.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.polyeval
     COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   logf
   SRCS
     logf.cpp
   HDRS
     ../logf.h
   DEPENDS
     .common_constants
     libc.src.__support.FPUtil.fputil
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
   COMPILE_OPTIONS
     -O3
-    -mfma
 )
 
 add_entrypoint_object(
   logb
   SRCS
     logb.cpp
   HDRS
     ../logb.h

libc/src/math/generic/exp2f.cpp

static constexpr double EXP_M[64] = {
     0x1.ae89f995ad3adp0, 0x1.b33a2b84f15fbp0, 0x1.b7f76f2fb5e47p0,
     0x1.bcc1e904bc1d2p0, 0x1.c199bdd85529cp0, 0x1.c67f12e57d14bp0,
     0x1.cb720dcef9069p0, 0x1.d072d4a07897cp0, 0x1.d5818dcfba487p0,
     0x1.da9e603db3285p0, 0x1.dfc97337b9b5fp0, 0x1.e502ee78b3ff6p0,
     0x1.ea4afa2a490dap0, 0x1.efa1bee615a27p0, 0x1.f50765b6e4540p0,
     0x1.fa7c1819e90d8p0,
 };
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, exp2f, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
   // Exceptional values.

libc/src/math/generic/expf.cpp

 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/common.h"
 
 #include <errno.h>
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, expf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
   // Exceptional values

libc/src/math/generic/expm1f.cpp

 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/common.h"
 
 #include <errno.h>
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
   // Exceptional value

libc/src/math/generic/log10f.cpp

static constexpr double LOG10_F[128] = {
     0x1.1014319e661bdp-2, 0x1.11f594839a5bdp-2, 0x1.13d4f0862b2e1p-2,
     0x1.15b24a0004a92p-2, 0x1.178da53d1ee01p-2, 0x1.1967067bb94b8p-2,
     0x1.1b3e71ec94f7bp-2, 0x1.1d13ebb32d7f9p-2, 0x1.1ee777e5f0dc3p-2,
     0x1.20b91a8e76105p-2, 0x1.2288d7a9b2b64p-2, 0x1.2456b3282f786p-2,
     0x1.2622b0ee3b79dp-2, 0x1.27ecd4d41eb67p-2, 0x1.29b522a64b609p-2,
     0x1.2b7b9e258e422p-2, 0x1.2d404b073e27ep-2, 0x1.2f032cf56a5bep-2,
     0x1.30c4478f0835fp-2, 0x1.32839e681fc62p-2};
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, log10f, (float x)) {
   constexpr double LOG10_2 = 0x1.34413509f79ffp-2;
 
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   double m = 0.0;
 
   // Exact powers of 10 and other hard-to-round cases.

libc/src/math/generic/log1pf.cpp

 // generated with Sollya using the following command:
 //   fpminimax(log(1 + x)/x, 5, [|D...|], [-2^-8; 2^-8]);
 
 namespace __llvm_libc {
 
 namespace internal {
 
 // We don't need to treat denormal
-INLINE_FMA static inline float log(double x) {
+static inline float log(double x) {
   constexpr double LOG_2 = 0x1.62e42fefa39efp-1;
 
   using FPBits = typename fputil::FPBits<double>;
   FPBits xbits(x);
 
   if (xbits.is_zero()) {
     return static_cast<float>(fputil::FPBits<float>::neg_inf());
   }
double r = fputil::polyeval(d, extra_factor, 0x1.fffffffffffacp-1,
                               -0x1.fffffffef9cb2p-2, 0x1.5555513bc679ap-2,
                               -0x1.fff4805ea441p-3, 0x1.930180dbde91ap-3);
 
   return static_cast<float>(r);
 }
 
 } // namespace internal
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, log1pf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   double xd = static_cast<double>(x);
 
   if (xbits.get_exponent() >= -8) {
     // Hard-to-round cases.
     switch (xbits.uintval()) {

libc/src/math/generic/log2f.cpp

static constexpr double LOG2_F[128] = {
     0x1.c3e9ca2e1a055p-1, 0x1.c7095ae91e1c7p-1, 0x1.ca258dca93316p-1,
     0x1.cd3e6a0ca8907p-1, 0x1.d053f6d260896p-1, 0x1.d3663b27f31d5p-1,
     0x1.d6753e032ea0fp-1, 0x1.d9810643d6615p-1, 0x1.dc899ab3ff56cp-1,
     0x1.df8f02086af2cp-1, 0x1.e29142e0e0140p-1, 0x1.e59063c8822cep-1,
     0x1.e88c6b3626a73p-1, 0x1.eb855f8ca88fbp-1, 0x1.ee7b471b3a950p-1,
     0x1.f16e281db7630p-1, 0x1.f45e08bcf0655p-1, 0x1.f74aef0efafaep-1,
     0x1.fa34e1177c233p-1, 0x1.fd1be4c7f2af9p-1};
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, log2f, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   int m = 0;
 
   // Hard to round value(s).
   switch (FPBits(x).uintval()) {
   case 0x3f81d0b5U: {

libc/src/math/generic/logf.cpp

 // Correctly Rounded Results of an Elementary Function for Multiple
 // Representations and Rounding Modes", Proceedings of the 49th ACM SIGPLAN
 // Symposium on Principles of Programming Languages (POPL-2022), Philadelphia,
 // USA, January 16-22, 2022.
 // https://people.cs.rutgers.edu/~sn349/papers/rlibmall-popl-2022.pdf
 
 namespace __llvm_libc {
 
-INLINE_FMA
 LLVM_LIBC_FUNCTION(float, logf, (float x)) {
   constexpr double LOG_2 = 0x1.62e42fefa39efp-1;
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
   switch (FPBits(x).uintval()) {
   case 0x41178febU: // x = 0x1.2f1fd6p+3f
     if (fputil::get_round() == FE_TONEAREST)

libc/test/src/math/CMakeLists.txt

SUITE
     libc_math_unittests
   SRCS
     tan_test.cpp
   DEPENDS
     libc.src.math.tan
     libc.src.__support.FPUtil.fputil
 )
 
+# Without FMA instructions, the current expm1f implementation is not correctly
+# rounded for all float inputs (1 extra exceptional value). This will be fixed
+# in the followup patch: https://reviews.llvm.org/D123440
 add_fp_unittest(
   expm1f_test
   NEED_MPFR
   SUITE
     libc_math_unittests
   SRCS
     expm1f_test.cpp
   DEPENDS
     libc.include.errno
     libc.include.math
     libc.src.math.expm1f
     libc.src.__support.FPUtil.fputil
+  FLAGS
+    FMA_OPT__ONLY
 )
 
 add_fp_unittest(
   mod_k_pi_test
   NEED_MPFR
   SUITE
     libc-long-running-tests
   SRCS