[libc] Return correct values for hypot when overflowed.

[lntue]

Summary

Hypot incorrectly returns +Inf when overflowed with FE_DOWNWARD and
FE_TOWARDZERO rounding modes.

[zimmermann6]

ok for me. I have added more checks near the underflow and overflow boundaries in CORE-MATH check.sh, and it passes all tests.

libc/src/__support/FPUtil/Hypot.h

if (sum >= (DUIntType(1) << (2 * y_mant_width + 2))) {
     // a^2 + b^2 >= 4* leading_one^2, so we will need an extra bit to the left.
     if (leading_one == ONE) {
       // For normal result, we discard the last 2 bits of the sum and increase
       // the exponent.
       sticky_bits = sticky_bits || ((sum & 0x3U) != 0);
       sum >>= 2;
       ++out_exp;
       if (out_exp >= FPBits_t::MAX_EXPONENT) {
+        if (int round_mode = get_round();
+            round_mode == FE_TONEAREST || round_mode == FE_UPWARD)
           return T(FPBits_t::inf());
+        return T(FPBits_t(FPBits_t::MAX_NORMAL));
       }
     } else {
       // For denormal result, we simply move the leading bit of the result to
       // the left by 1.
       leading_one <<= 1;
       ++y_mant_width;
     }
   }
if (y_new >= ONE) {
     if (out_exp == 0) {
       out_exp = 1;
     }
   }
 
   y_new >>= 1;
 
   // Round to the nearest, tie to even.
-  switch (get_round()) {
+  int round_mode = get_round();
+  switch (round_mode) {
   case FE_TONEAREST:
     // Round to nearest, ties to even
     if (round_bit && (lsb || sticky_bits || (r != 0)))
       ++y_new;
     break;
   case FE_UPWARD:
     if (round_bit || sticky_bits || (r != 0))
       ++y_new;
     break;
   }
 
   if (y_new >= (ONE >> 1)) {
     y_new -= ONE >> 1;
     ++out_exp;
     if (out_exp >= FPBits_t::MAX_EXPONENT) {
+      if (round_mode == FE_TONEAREST || round_mode == FE_UPWARD)
         return T(FPBits_t::inf());
+      return T(FPBits_t(FPBits_t::MAX_NORMAL));
     }
   }
 
   y_new |= static_cast<UIntType>(out_exp) << MantissaWidth<T>::VALUE;
   return cpp::bit_cast<T>(y_new);
 }
 
 } // namespace fputil
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_HYPOT_H

libc/test/src/math/HypotTest.h

 namespace mpfr = __llvm_libc::testing::mpfr;
 
 template <typename T>
 class HypotTestTemplate : public __llvm_libc::testing::Test {
 private:
   using Func = T (*)(T, T);
   using FPBits = __llvm_libc::fputil::FPBits<T>;
   using UIntType = typename FPBits::UIntType;
-  const T nan = T(__llvm_libc::fputil::FPBits<T>::build_nan(1));
-  const T inf = T(__llvm_libc::fputil::FPBits<T>::inf());
-  const T neg_inf = T(__llvm_libc::fputil::FPBits<T>::neg_inf());
-  const T zero = T(__llvm_libc::fputil::FPBits<T>::zero());
-  const T neg_zero = T(__llvm_libc::fputil::FPBits<T>::neg_zero());
+  const T nan = T(FPBits::build_nan(1));
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
+  const T max_normal = T(FPBits(FPBits::MAX_NORMAL));
+  const T min_normal = T(FPBits(FPBits::MIN_NORMAL));
+  const T max_subnormal = T(FPBits(FPBits::MAX_SUBNORMAL));
+  const T min_subnormal = T(FPBits(FPBits::MIN_SUBNORMAL));
 
 public:
   void test_special_numbers(Func func) {
+    constexpr int N = 13;
+    const T SpecialInputs[N] = {inf,           neg_inf,        zero,
+                                neg_zero,      max_normal,     min_normal,
+                                max_subnormal, min_subnormal,  -max_normal,
+                                -min_normal,   -max_subnormal, -min_subnormal};
+
     EXPECT_FP_EQ(func(inf, nan), inf);
     EXPECT_FP_EQ(func(nan, neg_inf), inf);
-    EXPECT_FP_EQ(func(zero, inf), inf);
-    EXPECT_FP_EQ(func(neg_inf, neg_zero), inf);
-
     EXPECT_FP_EQ(func(nan, nan), nan);
     EXPECT_FP_EQ(func(nan, zero), nan);
     EXPECT_FP_EQ(func(neg_zero, nan), nan);
 
-    EXPECT_FP_EQ(func(neg_zero, zero), zero);
+    for (int i = 0; i < N; ++i) {
+      for (int j = 0; j < N; ++j) {
+        mpfr::BinaryInput<T> input{SpecialInputs[i], SpecialInputs[j]};
+        EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Hypot, input,
+                                       func(SpecialInputs[i], SpecialInputs[j]),
+                                       0.5);
+      }
+    }
   }
 
   void test_subnormal_range(Func func) {
     constexpr UIntType COUNT = 1000001;
     for (unsigned scale = 0; scale < 4; ++scale) {
       UIntType max_value = FPBits::MAX_SUBNORMAL << scale;
       UIntType step = (max_value - FPBits::MIN_SUBNORMAL) / COUNT;
       for (int signs = 0; signs < 4; ++signs) {