[flang] Fix interpretations of x87 80-bit Inf/NaN

[klausler]

Summary

Current implementations of x87 80-bit extended precision floating
point interpret 7FFF8000000000000000 as +Inf, not a Nan. The explicit
MSB in the significand must be set for an infinity.

[tschuett]

There is again no context. Do you want support such an old compute facility or just say no we don't support x87? 80 bits is a precise indicator for x87?

[vdonaldson]

Looks good to me

flang/include/flang/Decimal/binary-floating-point.h

 
   constexpr bool IsZero() const {
     return (raw_ & ((RawType{1} << (bits - 1)) - 1)) == 0;
   }
   constexpr bool IsNaN() const {
-    return BiasedExponent() == maxExponent && Significand() != 0;
+    auto expo{BiasedExponent()};
+    auto sig{Significand()};
+    if constexpr (bits == 80) { // x87
+      if (expo == maxExponent) {
+        return sig != (significandMask >> 1) + 1;
+      } else {
+        return expo != 0 && !(sig & (RawType{1} << (significandBits - 1)));
+        ;
+      }
+    } else {
+      return expo == maxExponent && sig != 0;
+    }
   }
   constexpr bool IsInfinite() const {
-    return BiasedExponent() == maxExponent && Significand() == 0;
+    if constexpr (bits == 80) { // x87
+      return BiasedExponent() == maxExponent &&
+          Significand() == ((significandMask >> 1) + 1);
+    } else {
+      return BiasedExponent() == maxExponent && Significand() == 0;
+    }
   }
   constexpr bool IsMaximalFiniteMagnitude() const {
     return BiasedExponent() == maxExponent - 1 &&
         Significand() == significandMask;
   }

flang/include/flang/Evaluate/real.h

   constexpr bool IsSignBitSet() const { return word_.BTEST(bits - 1); }
   constexpr bool IsNegative() const {
     return !IsNotANumber() && IsSignBitSet();
   }
   constexpr bool IsNotANumber() const {
-    return Exponent() == maxExponent && !GetSignificand().IsZero();
+    auto expo{Exponent()};
+    auto sig{GetSignificand()};
+    if constexpr (bits == 80) { // x87
+      // 7FFF8000000000000000 is Infinity, not NaN, on 80387 & later.
+      if (expo == maxExponent) {
+        return sig != Significand{}.IBSET(63);
+      } else {
+        return expo != 0 && !sig.BTEST(63);
+      }
+    } else {
+      return expo == maxExponent && !sig.IsZero();
+    }
   }
   constexpr bool IsQuietNaN() const {
-    return Exponent() == maxExponent &&
-        GetSignificand().BTEST(significandBits - 1);
+    auto expo{Exponent()};
+    auto sig{GetSignificand()};
+    if constexpr (bits == 80) { // x87
+      if (expo == maxExponent) {
+        return sig.IBITS(62, 2) == 3;
+      } else {
+        return expo != 0 && !sig.BTEST(63);
+      }
+    } else {
+      return expo == maxExponent && sig.BTEST(significandBits - 1);
+    }
   }
   constexpr bool IsSignalingNaN() const {
-    return IsNotANumber() && !GetSignificand().BTEST(significandBits - 1);
+    auto expo{Exponent()};
+    auto sig{GetSignificand()};
+    if constexpr (bits == 80) { // x87
+      return expo == maxExponent && sig != Significand{}.IBSET(63) &&
+          sig.IBITS(62, 2) != 3;
+    } else {
+      return expo == maxExponent && !sig.IsZero() &&
+          !sig.BTEST(significandBits - 1);
+    }
   }
   constexpr bool IsInfinite() const {
-    return Exponent() == maxExponent && GetSignificand().IsZero();
+    if constexpr (bits == 80) { // x87
+      // 7FFF8000000000000000 is Infinity, not NaN, on 80387 & later.
+      return Exponent() == maxExponent &&
+          GetSignificand() == Significand{}.IBSET(63);
+    } else {
+      return Exponent() == maxExponent && GetSignificand().IsZero();
+    }
+  }
+  constexpr bool IsFinite() const {
+    auto expo{Exponent()};
+    if constexpr (bits == 80) { // x87
+      return expo != maxExponent && (expo == 0 || GetSignificand().BTEST(63));
+    } else {
+      return expo != maxExponent;
+    }
   }
-  constexpr bool IsFinite() const { return Exponent() != maxExponent; }
   constexpr bool IsZero() const {
     return Exponent() == 0 && GetSignificand().IsZero();
   }
   constexpr bool IsSubnormal() const {
     return Exponent() == 0 && !GetSignificand().IsZero();

     Word infinity{maxExponent};
     infinity = infinity.SHIFTL(significandBits);
     if (negative) {
       infinity = infinity.IBSET(infinity.bits - 1);
     }
+    if constexpr (bits == 80) { // x87
+      // 7FFF8000000000000000 is Infinity, not NaN, on 80387 & later.
+      infinity.IBSET(63);
+    }
     return {infinity};
   }
 
   template <typename INT>
   static ValueWithRealFlags<Real> FromInteger(const INT &n,

flang/lib/Decimal/big-radix-floating-point.h

   bool ParseNumber(const char *&, bool &inexact, const char *end);
 
   using Raw = typename Real::RawType;
   constexpr Raw SignBit() const { return Raw{isNegative_} << (Real::bits - 1); }
   constexpr Raw Infinity() const {
-    return (Raw{Real::maxExponent} << Real::significandBits) | SignBit();
+    Raw result{static_cast<Raw>(Real::maxExponent)};
+    result <<= Real::significandBits;
+    result |= SignBit();
+    if constexpr (Real::bits == 80) { // x87
+      result |= Raw{1} << 63;
+    }
+    return result;
   }
   constexpr Raw NaN(bool isQuiet = true) {
-    return (Raw{Real::maxExponent} << Real::significandBits) |
-        (Raw{1} << (Real::significandBits - (isQuiet ? 1 : 2))) | SignBit();
+    Raw result{Real::maxExponent};
+    result <<= Real::significandBits;
+    result |= SignBit();
+    if constexpr (Real::bits == 80) { // x87
+      result |= Raw{isQuiet ? 3u : 2u} << 62;
+    } else {
+      Raw quiet{isQuiet ? Raw{2} : Raw{1}};
+      quiet <<= Real::significandBits - 2;
+      result |= quiet;
+    }
+    return result;
   }
 
   Digit digit_[maxDigits]; // in little-endian order: digit_[0] is LSD
   int digits_{0}; // # of elements in digit_[] array; zero when zero
   int digitLimit_{maxDigits}; // precision clamp

flang/lib/Decimal/decimal-to-binary.cpp

     expo = 0; // subnormal
   }
   if (expo >= Binary::maxExponent) {
     expo = Binary::maxExponent; // Inf
     flags |= Overflow;
-    fraction = 0;
+    if constexpr (Binary::bits == 80) { // x87
+      fraction = IntType{1} << 63;
+    } else {
+      fraction = 0;
+    }
   }
   using Raw = typename Binary::RawType;
   Raw raw = static_cast<Raw>(isNegative) << (Binary::bits - 1);
   raw |= static_cast<Raw>(expo) << Binary::significandBits;
   if constexpr (Binary::isImplicitMSB) {

flang/test/Evaluate/folding03.f90

   TEST_ISNAN(r4_nan_add1)
   !WARN: warning: invalid argument on addition
   real(4), parameter :: r4_nan_add2 = r4_pinf + r4_ninf
   TEST_ISNAN(r4_nan_add2)
 
-  ! No warnings expected here (quite NaN propagation)
+  ! No warnings expected here (quiet NaN propagation)
   real(4), parameter :: r4_nan_sub3 = 0._4 - r4_nan
   TEST_ISNAN(r4_nan_sub3)
   real(4), parameter :: r4_nan_sub4 = r4_nan - r4_pmax
   TEST_ISNAN(r4_nan_sub4)
   real(4), parameter :: r4_nan_sub5 = r4_nan - r4_nmax

   TEST_ISNAN(r4_nan_mult1)
   !WARN: warning: invalid argument on multiplication
   real(4), parameter :: r4_nan_mult2 = 0._4*r4_ninf
   TEST_ISNAN(r4_nan_mult2)
 
-  ! No warnings expected here (quite NaN propagation)
+  ! No warnings expected here (quiet NaN propagation)
   real(4), parameter :: r4_nan_div6 = 0._4/r4_nan
   TEST_ISNAN(r4_nan_div6)
   real(4), parameter :: r4_nan_div7 = r4_nan/r4_nan
   TEST_ISNAN(r4_nan_div7)
   real(4), parameter :: r4_nan_div8 = r4_nan/0._4

flang/unittests/Evaluate/real.cpp

   TEST(nan.Compare(nan) == Relation::Unordered)(desc);
   int significandBits{R::binaryPrecision - R::isImplicitMSB};
   int exponentBits{R::bits - significandBits - 1};
   std::uint64_t maxExponent{(std::uint64_t{1} << exponentBits) - 1};
   MATCH(nan.Exponent(), maxExponent)(desc);
-  R inf{Word{maxExponent}.SHIFTL(significandBits)};
+  Word infWord{Word{maxExponent}.SHIFTL(significandBits)};
+  Word negInfWord{
+      Word{maxExponent}.SHIFTL(significandBits).IOR(Word::MASKL(1))};
+  if constexpr (kind == 10) { // x87
+    infWord = infWord.IBSET(63);
+    negInfWord = negInfWord.IBSET(63);
+  }
+  R inf{infWord};
+  R negInf{negInfWord};
   TEST(!inf.IsNegative())(desc);
   TEST(!inf.IsNotANumber())(desc);
   TEST(inf.IsInfinite())(desc);
   TEST(!inf.IsZero())(desc);
   TEST(inf.RawBits().CompareUnsigned(inf.ABS().RawBits()) == Ordering::Equal)
   (desc);
   TEST(zero.Compare(inf) == Relation::Less)(desc);
   TEST(minusZero.Compare(inf) == Relation::Less)(desc);
   TEST(nan.Compare(inf) == Relation::Unordered)(desc);
   TEST(inf.Compare(inf) == Relation::Equal)(desc);
-  R negInf{Word{maxExponent}.SHIFTL(significandBits).IOR(Word::MASKL(1))};
   TEST(negInf.IsNegative())(desc);
   TEST(!negInf.IsNotANumber())(desc);
   TEST(negInf.IsInfinite())(desc);
   TEST(!negInf.IsZero())(desc);
   TEST(inf.RawBits().CompareUnsigned(negInf.ABS().RawBits()) == Ordering::Equal)