[libc++] Fix support for multibyte thousands_sep and decimal_point in moneypunct_byname and numpunct_byname.

[EricWF]

Summary

The underlying C locales provide the thousands_sep and decimal_point as strings, possible with more than one character. We currently don't handle this case even for wchar_t.

This patch properly converts the mbs -> wide character for moneypunct_byname<wchar_t>. For the moneypunct_byname<char> case we attempt to narrow the WC and if that fails we also attempt to translate it to some reasonable value. For example we translate U00A0 (non-breaking space) into U0020 (regular space). If none of these conversions succeed then we simply allow the base class to provide a fallback value.

[EricWF]

Remove secondary locale fix.

[george.burgess.iv]

Just a few drive-by nits. Thanks for the patch!

src/locale.cpp

5849–5851

Are the newly-added braces necessary?

test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/decimal_point.pass.cpp

113

s/belowe/below

test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/thousands_sep.pass.cpp

112–116

s/belowe/below/

[EricWF]

Address all inline comments.

[vangyzen]

src/locale.cpp

4339

While you're here, you might change char to wchar_t.

4348

Maybe fix the spelling of "localization".

[EricWF]

Address inline comments.

src/locale.cpp

         break;
     }
     pat.field[0] = symbol;
     pat.field[1] = sign;
     pat.field[2] = none;
     pat.field[3] = value;
 }
 
+static char checked_string_to_char_convert(const char* ptr) {
+  /*
+  // FIXME: The input character is multiple bytes. Do something...
+  if (ptr[0] && ptr[1])
+    __throw_runtime_error("locale not supported");
+  */
+  return *ptr;
+}
+
+static wchar_t checked_string_to_wchar_convert(const char* ptr,
+                                               __locale_struct* loc) {
+  mbstate_t mb = {0};
+  wchar_t out;
+  size_t ret = __libcpp_mbrtowc_l(&out, ptr, strlen(ptr), &mb, loc);
+  if (ret == static_cast<size_t>(-1) || ret == static_cast<size_t>(-2)) {
+      __throw_runtime_error("locale not supported");
+  }
+  return out;
+}
+
 template<>
 void
 moneypunct_byname<char, false>::init(const char* nm)
 {
     typedef moneypunct<char, false> base;
     __locale_unique_ptr loc(newlocale(LC_ALL_MASK, nm, 0), freelocale);
     if (loc == nullptr)
         __throw_runtime_error("moneypunct_byname"
                             " failed to construct for " + string(nm));
 
     lconv* lc = __libcpp_localeconv_l(loc.get());
     if (*lc->mon_decimal_point)
-        __decimal_point_ = *lc->mon_decimal_point;
+        __decimal_point_ = checked_string_to_char_convert(lc->mon_decimal_point);
     else
         __decimal_point_ = base::do_decimal_point();
     if (*lc->mon_thousands_sep)
-        __thousands_sep_ = *lc->mon_thousands_sep;
+        __thousands_sep_ = checked_string_to_char_convert(lc->mon_thousands_sep);
     else
         __thousands_sep_ = base::do_thousands_sep();
     __grouping_ = lc->mon_grouping;
     __curr_symbol_ = lc->currency_symbol;
     if (lc->frac_digits != CHAR_MAX)
         __frac_digits_ = lc->frac_digits;
     else
         __frac_digits_ = base::do_frac_digits();

     typedef moneypunct<char, true> base;
     __locale_unique_ptr loc(newlocale(LC_ALL_MASK, nm, 0), freelocale);
     if (loc == nullptr)
         __throw_runtime_error("moneypunct_byname"
                             " failed to construct for " + string(nm));
 
     lconv* lc = __libcpp_localeconv_l(loc.get());
     if (*lc->mon_decimal_point)
-        __decimal_point_ = *lc->mon_decimal_point;
+        __decimal_point_ = checked_string_to_char_convert(lc->mon_decimal_point);
     else
         __decimal_point_ = base::do_decimal_point();
-    if (*lc->mon_thousands_sep)
-        __thousands_sep_ = *lc->mon_thousands_sep;
-    else
+    if (*lc->mon_thousands_sep) {
+        __thousands_sep_ = checked_string_to_char_convert(lc->mon_thousands_sep);
+    } else
         __thousands_sep_ = base::do_thousands_sep();
     __grouping_ = lc->mon_grouping;
     __curr_symbol_ = lc->int_curr_symbol;
     if (lc->int_frac_digits != CHAR_MAX)
         __frac_digits_ = lc->int_frac_digits;
     else
         __frac_digits_ = base::do_frac_digits();
 #if defined(_LIBCPP_MSVCRT) || defined(__MINGW32__)

 {
     typedef moneypunct<wchar_t, false> base;
     __locale_unique_ptr loc(newlocale(LC_ALL_MASK, nm, 0), freelocale);
     if (loc == nullptr)
         __throw_runtime_error("moneypunct_byname"
                             " failed to construct for " + string(nm));
     lconv* lc = __libcpp_localeconv_l(loc.get());
     if (*lc->mon_decimal_point)
-        __decimal_point_ = static_cast<wchar_t>(*lc->mon_decimal_point);
+        __decimal_point_ = checked_string_to_wchar_convert(lc->mon_decimal_point,
+                                                           loc.get());
     else
         __decimal_point_ = base::do_decimal_point();
     if (*lc->mon_thousands_sep)
-        __thousands_sep_ = static_cast<wchar_t>(*lc->mon_thousands_sep);
+        __thousands_sep_ = checked_string_to_wchar_convert(lc->mon_thousands_sep,
+                                                            loc.get());
     else
         __thousands_sep_ = base::do_thousands_sep();
     __grouping_ = lc->mon_grouping;
     wchar_t wbuf[100];
     mbstate_t mb = {0};
     const char* bb = lc->currency_symbol;
     size_t j = __libcpp_mbsrtowcs_l(wbuf, &bb, countof(wbuf), &mb, loc.get());
     if (j == size_t(-1))

     typedef moneypunct<wchar_t, true> base;
     __locale_unique_ptr loc(newlocale(LC_ALL_MASK, nm, 0), freelocale);
     if (loc == nullptr)
         __throw_runtime_error("moneypunct_byname"
                             " failed to construct for " + string(nm));
 
     lconv* lc = __libcpp_localeconv_l(loc.get());
     if (*lc->mon_decimal_point)
-        __decimal_point_ = static_cast<wchar_t>(*lc->mon_decimal_point);
+        __decimal_point_ = checked_string_to_wchar_convert(lc->mon_decimal_point,
+                                                           loc.get());
     else
         __decimal_point_ = base::do_decimal_point();
     if (*lc->mon_thousands_sep)
-        __thousands_sep_ = static_cast<wchar_t>(*lc->mon_thousands_sep);
+      __thousands_sep_ = checked_string_to_wchar_convert(lc->mon_thousands_sep,
+                                                         loc.get());
     else
         __thousands_sep_ = base::do_thousands_sep();
     __grouping_ = lc->mon_grouping;
     wchar_t wbuf[100];
     mbstate_t mb = {0};
     const char* bb = lc->int_curr_symbol;
     size_t j = __libcpp_mbsrtowcs_l(wbuf, &bb, countof(wbuf), &mb, loc.get());
     if (j == size_t(-1))

test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/curr_symbol.pass.cpp

 // class moneypunct_byname<charT, International>
 
 // string_type curr_symbol() const;
 
 #include <locale>
 #include <limits>
 #include <cassert>
 
+#include "test_macros.h"
 #include "platform_support.h" // locale name macros
 
 class Fnf
     : public std::moneypunct_byname<char, false>
 {
 public:
     explicit Fnf(const std::string& nm, std::size_t refs = 0)
         : std::moneypunct_byname<char, false>(nm, refs) {}
int main()
     }
     {
         Fwt f(LOCALE_fr_FR_UTF_8, 1);
         assert(f.curr_symbol() == L" EUR");
     }
 
     {
         Fnf f(LOCALE_ru_RU_UTF_8, 1);
+        // GLIBC <= 2.23 uses currency_symbol="<U0440><U0443><U0431>"
+        // GLIBC >= 2.24 uses currency_symbol="<U20BD>"
+        // See also: http://www.fileformat.info/info/unicode/char/20bd/index.htm
+#if defined(TEST_GLIBC_PREREQ) && TEST_GLIBC_PREREQ(2, 24)
+        assert(f.curr_symbol() == " \xE2\x82\xBD");
+#else
         assert(f.curr_symbol() == " \xD1\x80\xD1\x83\xD0\xB1");
+#endif
     }
     {
         Fnt f(LOCALE_ru_RU_UTF_8, 1);
         assert(f.curr_symbol() == " RUB");
     }
     {
         Fwf f(LOCALE_ru_RU_UTF_8, 1);
         assert(f.curr_symbol() == L" \x440\x443\x431");

test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/decimal_point.pass.cpp

 //===----------------------------------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is dual licensed under the MIT and the University of Illinois Open
 // Source Licenses. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
 // REQUIRES: locale.zh_CN.UTF-8
 
-// Russia uses ',' for the decimal separator. GLIBC returns '.'
-// XFAIL: linux
-
 // <locale>
 
 // class moneypunct_byname<charT, International>
 
 // charT decimal_point() const;
 
 #include <locale>
 #include <limits>
 #include <cassert>
 
+#include "test_macros.h"
 #include "platform_support.h" // locale name macros
 
 class Fnf
     : public std::moneypunct_byname<char, false>
 {
 public:
     explicit Fnf(const std::string& nm, std::size_t refs = 0)
         : std::moneypunct_byname<char, false>(nm, refs) {}
int main()
         Fwf f(LOCALE_fr_FR_UTF_8, 1);
         assert(f.decimal_point() == L',');
     }
     {
         Fwt f(LOCALE_fr_FR_UTF_8, 1);
         assert(f.decimal_point() == L',');
     }
 
+// The belowe tests fail due to GLIBC's use of U00A0 as mon_decimal_point
+// and U002E as mon_decimal_point.
+// TODO: Fix decimal_point for 'char'.
+// related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=16006
+#ifndef TEST_HAS_GLIBC
     {
         Fnf f(LOCALE_ru_RU_UTF_8, 1);
-        assert(f.decimal_point() == ',');
+        assert(f.decimal_point() == '.');
     }
     {
         Fnt f(LOCALE_ru_RU_UTF_8, 1);
-        assert(f.decimal_point() == ',');
+        assert(f.decimal_point() == '.');
     }
+    const wchar_t sep = L'.';
+#else
+    const wchar_t sep = L'\u002E';
+#endif
     {
         Fwf f(LOCALE_ru_RU_UTF_8, 1);
-        assert(f.decimal_point() == L',');
+        assert(f.decimal_point() == sep);
     }
     {
         Fwt f(LOCALE_ru_RU_UTF_8, 1);
-        assert(f.decimal_point() == L',');
+        assert(f.decimal_point() == sep);
     }
 
     {
         Fnf f(LOCALE_zh_CN_UTF_8, 1);
         assert(f.decimal_point() == '.');
     }
     {
         Fnt f(LOCALE_zh_CN_UTF_8, 1);

test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/thousands_sep.pass.cpp

 // REQUIRES: locale.zh_CN.UTF-8
 
 // <locale>
 
 // class moneypunct_byname<charT, International>
 
 // charT thousands_sep() const;
 
-// Failure related to GLIBC's use of U00A0 as mon_thousands_sep
-// and U002E as mon_decimal_point.
-// TODO: U00A0 should be investigated.
-// Possibly related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=16006
-// XFAIL: linux-gnu
-
 #include <locale>
 #include <limits>
 #include <cassert>
 
+#include "test_macros.h"
 #include "platform_support.h" // locale name macros
 
 class Fnf
     : public std::moneypunct_byname<char, false>
 {
 public:
     explicit Fnf(const std::string& nm, std::size_t refs = 0)
         : std::moneypunct_byname<char, false>(nm, refs) {}
int main()
     {
         Fwf f(LOCALE_fr_FR_UTF_8, 1);
         assert(f.thousands_sep() == L' ');
     }
     {
         Fwt f(LOCALE_fr_FR_UTF_8, 1);
         assert(f.thousands_sep() == L' ');
     }
-
+// The belowe tests fail due to GLIBC's use of U00A0 as mon_thousands_sep
+// and U002E as mon_decimal_point.
+// TODO: Fix thousands_sep for 'char'.
+// related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=16006
+#ifndef TEST_HAS_GLIBC
     {
         Fnf f(LOCALE_ru_RU_UTF_8, 1);
         assert(f.thousands_sep() == ' ');
     }
     {
         Fnt f(LOCALE_ru_RU_UTF_8, 1);
         assert(f.thousands_sep() == ' ');
     }
+    const wchar_t sep = L' ';
+#else
+    const wchar_t sep = L'\u00A0';
+#endif
     {
         Fwf f(LOCALE_ru_RU_UTF_8, 1);
-        assert(f.thousands_sep() == L' ');
+        assert(f.thousands_sep() == sep);
     }
     {
         Fwt f(LOCALE_ru_RU_UTF_8, 1);
-        assert(f.thousands_sep() == L' ');
+        assert(f.thousands_sep() == sep);
     }
 
     {
         Fnf f(LOCALE_zh_CN_UTF_8, 1);
         assert(f.thousands_sep() == ',');
     }
     {
         Fnt f(LOCALE_zh_CN_UTF_8, 1);

test/support/test_macros.h

 # define TEST_STD_VER 11
 #elif __cplusplus <= 201402L
 # define TEST_STD_VER 14
 #else
 # define TEST_STD_VER 16    // current year; greater than current standard
 #endif
 #endif
 
+// Attempt to deduce GCC version
+#if defined(_LIBCPP_VERSION) && __has_include(<features.h>)
+#include <features.h>
+#define TEST_HAS_GLIBC
+#define TEST_GLIBC_PREREQ(major, minor) __GLIBC_PREREQ(major, minor)
+#endif
+
 /* Features that were introduced in C++14 */
 #if TEST_STD_VER >= 14
 #define TEST_HAS_EXTENDED_CONSTEXPR
 #define TEST_HAS_VARIABLE_TEMPLATES
 #endif
 
 /* Features that were introduced after C++14 */
 #if TEST_STD_VER > 14