[libc] fix strtointeger behavior on max values

[michaelrj]

Summary

Previously the check to just return MAX or MIN used the caclulated
number being the maximum absolute value. This was right in every case
except for an unsigned conversion being passed its maximum value with a
negative sign on the front. This should return -MAX, but was returning
just MAX.

libc/src/__support/str_to_integer.h

if (result > abs_max - cur_digit) {
       error_val = ERANGE;
     } else {
       result = result + cur_digit;
     }
   }
 
   ptrdiff_t str_len = is_number ? (src - original_src) : 0;
 
-  if (result == abs_max) {
+  if (error_val == ERANGE) {
     if (is_positive || IS_UNSIGNED)
       return {cpp::numeric_limits<T>::max(), str_len, error_val};
     else // T is signed and there is a negative overflow
       return {cpp::numeric_limits<T>::min(), str_len, error_val};
   }
 
   return {is_positive ? static_cast<T>(result) : -static_cast<T>(result),
           str_len, error_val};
 }
 
 } // namespace internal
 } // namespace __llvm_libc
 
 #endif // LIBC_SRC_SUPPORT_STR_TO_INTEGER_H

libc/test/src/stdlib/StrtolTest.h

void CleanBaseSixteenDecode(FunctionT func) {
     ASSERT_EQ(libc_errno, 0);
     EXPECT_EQ(str_end - yes_prefix, ptrdiff_t(8));
 
     const char *letter_after_prefix = "0xabc123";
     libc_errno = 0;
     ASSERT_EQ(func(letter_after_prefix, &str_end, 16), ReturnT(0xabc123));
     ASSERT_EQ(libc_errno, 0);
     EXPECT_EQ(str_end - letter_after_prefix, ptrdiff_t(8));
+
+    // These tests check what happens when the number passed is exactly the max
+    // value for the conversion.
+
+    // Max size for unsigned 32 bit numbers
+
+    const char *max_32_bit_value = "0xFFFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(max_32_bit_value, &str_end, 0),
+              ((is_signed_v<ReturnT> && sizeof(ReturnT) == 4)
+                   ? T_MAX
+                   : ReturnT(0xFFFFFFFF)));
+    ASSERT_EQ(libc_errno, 0);
+    EXPECT_EQ(str_end - max_32_bit_value, ptrdiff_t(10));
+
+    const char *negative_max_32_bit_value = "-0xFFFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(negative_max_32_bit_value, &str_end, 0),
+              ((is_signed_v<ReturnT> && sizeof(ReturnT) == 4)
+                   ? T_MIN
+                   : -ReturnT(0xFFFFFFFF)));
+    ASSERT_EQ(libc_errno, 0);
+    EXPECT_EQ(str_end - negative_max_32_bit_value, ptrdiff_t(11));
+
+    // Max size for signed 32 bit numbers
+
+    const char *max_31_bit_value = "0x7FFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(max_31_bit_value, &str_end, 0), ReturnT(0x7FFFFFFF));
+    ASSERT_EQ(libc_errno, 0);
+    EXPECT_EQ(str_end - max_31_bit_value, ptrdiff_t(10));
+
+    const char *negative_max_31_bit_value = "-0x7FFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(negative_max_31_bit_value, &str_end, 0),
+              -ReturnT(0x7FFFFFFF));
+    ASSERT_EQ(libc_errno, 0);
+    EXPECT_EQ(str_end - negative_max_31_bit_value, ptrdiff_t(11));
+
+    // Max size for unsigned 64 bit numbers
+
+    const char *max_64_bit_value = "0xFFFFFFFFFFFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(max_64_bit_value, &str_end, 0),
+              (is_signed_v<ReturnT> ? T_MAX : ReturnT(0xFFFFFFFFFFFFFFFF)));
+    ASSERT_EQ(libc_errno, (is_signed_v<ReturnT> ? ERANGE : 0));
+    EXPECT_EQ(str_end - max_64_bit_value, ptrdiff_t(18));
+
+    const char *negative_max_64_bit_value = "-0xFFFFFFFFFFFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(negative_max_64_bit_value, &str_end, 0),
+              (is_signed_v<ReturnT> ? T_MIN : -ReturnT(0xFFFFFFFFFFFFFFFF)));
+    ASSERT_EQ(libc_errno, (is_signed_v<ReturnT> ? ERANGE : 0));
+    EXPECT_EQ(str_end - negative_max_64_bit_value, ptrdiff_t(19));
+
+    // Max size for signed 64 bit numbers
+
+    const char *max_63_bit_value = "0x7FFFFFFFFFFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(max_63_bit_value, &str_end, 0), ReturnT(0x7FFFFFFFFFFFFFFF));
+    ASSERT_EQ(libc_errno, 0);
+    EXPECT_EQ(str_end - max_63_bit_value, ptrdiff_t(18));
+
+    const char *negative_max_63_bit_value = "-0x7FFFFFFFFFFFFFFF";
+    libc_errno = 0;
+    ASSERT_EQ(func(negative_max_63_bit_value, &str_end, 0),
+              -ReturnT(0x7FFFFFFFFFFFFFFF));
+    ASSERT_EQ(libc_errno, 0);
+    EXPECT_EQ(str_end - negative_max_63_bit_value, ptrdiff_t(19));
   }
 
   void MessyBaseSixteenDecode(FunctionT func) {
     char *str_end = nullptr;
 
     const char *just_prefix = "0x";
     libc_errno = 0;
     ASSERT_EQ(func(just_prefix, &str_end, 16), ReturnT(0));