[libc] Match x86 long double NaN classification with that of the compiler.

[MaskRay]

libc/test/utils/FPUtil/x86_long_double_test.cpp

27

You can test whether some combinations of bits are significant, instead of iterating over all integers from 0 to 1000000 (it is slow).

[asteinhauser]

libc/test/utils/FPUtil/x86_long_double_test.cpp

29

There is some ASSERT_NE(a, b) macro which might be useful. It asserts directly the inequality.

53

I would add one or two tests also for the false case - e.g.:

bits.exponent = 1;
bits.implicit = 1;
for (unsigned int i = 0; i < 1000000; ++i) {

// If exponent is non-zero and also not max, and the implicit bit is 0,
// then the number is a NaN for all values of mantissa.
bits.mantissa = i;
long double nan = bits;
ASSERT_EQ(isnan(nan), 0);
ASSERT_FALSE(bits.isNaN());

}

[asteinhauser]

Sorry, I kept there the old comment.

libc/test/utils/FPUtil/x86_long_double_test.cpp

53

Sorry, I left there an old comment from copy-pasting. There should be something like:
If the implicit bit is true and the exponent is not the max value,
the number is not a NaN for any mantissa.

[sivachandra]

Add tests for non-NaN numbers.

[sivachandra]

Added tests for non-NaN numbers.

libc/test/CMakeLists.txt

 add_custom_target(check-libc)
 
 add_subdirectory(config)
 add_subdirectory(loader)
 add_subdirectory(src)
+add_subdirectory(utils)

libc/test/utils/CMakeLists.txt

+add_subdirectory(FPUtil)

libc/test/utils/FPUtil/CMakeLists.txt

+if((${LIBC_TARGET_OS} STREQUAL "linux") AND (${LIBC_TARGET_MACHINE} MATCHES "i386|x86_64"))
+  add_libc_unittest(
+    x86_long_double_test
+    SRCS
+      x86_long_double_test.cpp
+    DEPENDS
+      libc.include.math
+      libc.utils.FPUtil.fputil
+  )
+endif()

libc/test/utils/FPUtil/x86_long_double_test.cpp

+//===-- Unittests for x86 long double -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+TEST(X86LongDoubleTest, NaN) {
+  // In the nan checks below, we use the macro isnan from math.h to ensure that
+  // a number is actually a NaN. The isnan macro resolves to the compiler
+  // builtin function. Hence, matching LLVM-libc's notion of NaN with the
+  // isnan result ensures that LLVM-libc's behavior matches the compiler's
+  // behavior.
+
+  FPBits bits(0.0l);
+  bits.exponent = FPBits::maxExponent;
+  for (unsigned int i = 0; i < 1000000; ++i) {
+    // If exponent has the max value and the implicit bit is 0,
+    // then the number is a NaN for all values of mantissa.
+    bits.mantissa = i;
+    long double nan = bits;
+    ASSERT_TRUE(isnan(nan) != 0);
+    ASSERT_TRUE(bits.isNaN());
+  }
+
+  bits.implicitBit = 1;
+  for (unsigned int i = 1; i < 1000000; ++i) {
+    // If exponent has the max value and the implicit bit is 1,
+    // then the number is a NaN for all non-zero values of mantissa.
+    // Note the initial value of |i| of 1 to avoid a zero mantissa.
+    bits.mantissa = i;
+    long double nan = bits;
+    ASSERT_TRUE(isnan(nan) != 0);
+    ASSERT_TRUE(bits.isNaN());
+  }
+
+  bits.exponent = 1;
+  bits.implicitBit = 0;
+  for (unsigned int i = 0; i < 1000000; ++i) {
+    // If exponent is non-zero and also not max, and the implicit bit is 0,
+    // then the number is a NaN for all values of mantissa.
+    bits.mantissa = i;
+    long double nan = bits;
+    ASSERT_TRUE(isnan(nan) != 0);
+    ASSERT_TRUE(bits.isNaN());
+  }
+}

libc/utils/FPUtil/LongDoubleBitsX86.h

template <> struct __attribute__((packed)) FPBits<long double> {
   bool isZero() const {
     return exponent == 0 && mantissa == 0 && implicitBit == 0;
   }
 
   bool isInf() const {
     return exponent == maxExponent && mantissa == 0 && implicitBit == 1;
   }
 
-  bool isNaN() const { return exponent == maxExponent && mantissa != 0; }
+  bool isNaN() const {
+    if (exponent == maxExponent) {
+      return (implicitBit == 0) || mantissa != 0;
+    } else if (exponent != 0) {
+      return implicitBit == 0;
+    }
+    return false;
+  }
 
   bool isInfOrNaN() const { return exponent == maxExponent; }
 
   // Methods below this are used by tests.
 
   template <typename XType,
             cpp::EnableIfType<cpp::IsSame<UIntType, XType>::Value, int> = 0>
   explicit FPBits<long double>(XType x) {