[libcxx] Check return value for asprintf()

[brad]

Summary

Submitting upstream from OpenBSD tree:

local __libcpp_asprintf_l() -> libc asprintf() was inspecting the pointer (with indeterminate value) for failure, rather than the return value of -1.

[ldionne]

Please fix the CI issues and ping again if you'd like to get this reviewed.

[brad]

Fix the comparison between signed and unsigned values.

[ldionne]

LGTM, thanks!

libcxx/include/locale

         __nc = __libcpp_snprintf_l(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, __v);
     unique_ptr<char, void(*)(void*)> __nbh(nullptr, free);
     if (__nc > static_cast<int>(__nbuf-1))
     {
         if (__specify_precision)
             __nc = __libcpp_asprintf_l(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
         else
             __nc = __libcpp_asprintf_l(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, __v);
-        if (__nb == nullptr)
+        if (__nc == -1)
             __throw_bad_alloc();
         __nbh.reset(__nb);
     }
     char* __ne = __nb + __nc;
     char* __np = this->__identify_padding(__nb, __ne, __iob);
     // Stage 2 - Widen __nar while adding thousands separators
     char_type __o[2*(__nbuf-1) - 1];
     char_type* __ob = __o;

         __nc = __libcpp_snprintf_l(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, __v);
     unique_ptr<char, void(*)(void*)> __nbh(nullptr, free);
     if (__nc > static_cast<int>(__nbuf-1))
     {
         if (__specify_precision)
             __nc = __libcpp_asprintf_l(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
         else
             __nc = __libcpp_asprintf_l(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, __v);
-        if (__nb == nullptr)
+        if (__nc == -1)
             __throw_bad_alloc();
         __nbh.reset(__nb);
     }
     char* __ne = __nb + __nc;
     char* __np = this->__identify_padding(__nb, __ne, __iob);
     // Stage 2 - Widen __nar while adding thousands separators
     char_type __o[2*(__nbuf-1) - 1];
     char_type* __ob = __o;

                                            long double __units) const
 {
     // convert to char
     const size_t __bs = 100;
     char __buf[__bs];
     char* __bb = __buf;
     char_type __digits[__bs];
     char_type* __db = __digits;
-    size_t __n = static_cast<size_t>(snprintf(__bb, __bs, "%.0Lf", __units));
+    int __n = snprintf(__bb, __bs, "%.0Lf", __units);
     unique_ptr<char, void(*)(void*)> __hn(nullptr, free);
     unique_ptr<char_type, void(*)(void*)> __hd(0, free);
     // secure memory for digit storage
-    if (__n > __bs-1)
+    if (static_cast<size_t>(__n) > __bs-1)
     {
-        __n = static_cast<size_t>(__libcpp_asprintf_l(&__bb, _LIBCPP_GET_C_LOCALE, "%.0Lf", __units));
-        if (__bb == nullptr)
+        __n = __libcpp_asprintf_l(&__bb, _LIBCPP_GET_C_LOCALE, "%.0Lf", __units);
+        if (__n == -1)
             __throw_bad_alloc();
         __hn.reset(__bb);
-        __hd.reset((char_type*)malloc(__n * sizeof(char_type)));
+        __hd.reset((char_type*)malloc(static_cast<size_t>(__n) * sizeof(char_type)));
         if (__hd == nullptr)
             __throw_bad_alloc();
         __db = __hd.get();
     }
     // gather info
     locale __loc = __iob.getloc();
     const ctype<char_type>& __ct = use_facet<ctype<char_type> >(__loc);
     __ct.widen(__bb, __bb + __n, __db);
     bool __neg = __n > 0 && __bb[0] == '-';
     money_base::pattern __pat;
     char_type __dp;
     char_type __ts;
     string __grp;
     string_type __sym;
     string_type __sn;
     int __fd;
     this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
     // secure memory for formatting
     char_type __mbuf[__bs];
     char_type* __mb = __mbuf;
     unique_ptr<char_type, void(*)(void*)> __hw(0, free);
-    size_t __exn = static_cast<int>(__n) > __fd ?
-                   (__n - static_cast<size_t>(__fd)) * 2 + __sn.size() +
-                    __sym.size() + static_cast<size_t>(__fd) + 1
+    size_t __exn = __n > __fd ?
+                   (static_cast<size_t>(__n) - static_cast<size_t>(__fd)) * 2 +
+                    __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 1
                  : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
     if (__exn > __bs)
     {
         __hw.reset((char_type*)malloc(__exn * sizeof(char_type)));
         __mb = __hw.get();
         if (__mb == 0)
             __throw_bad_alloc();
     }