[libc++] Correctly propagate user-defined lookup_classname().

[timshen]

Summary

Always lookup the class name, even when the traits type is regex_traits<>.
The lookup happens at regex compile time, so it shouldn't affect the performance.

I also added ja_JP.UTF-8 as a common locale.

This patch fixes PR18501.

[timshen]

I'm not sure if we need to change the CI server to suport ja_JP.UTF-8.

[timshen]

Update description.

[timshen]

Stylize template decl to "template <class ...".

[• Quuxplusone]

libcxx/test/std/re/re.traits/lookup_classname_user_defined.pass.cpp

47

Could you add a test here for

std::wstring re3 = L"([[:ALNUM:]]+)";
std::regex_search(in, m, std::wregex(re3, std::regex_constants::icase));

std::wstring re4 = L"(\\W+)";
std::regex_search(in, m, std::wregex(re4, std::regex_constants::icase));

documenting the expected outputs? It's unclear to me from cppreference
http://en.cppreference.com/w/cpp/regex/regex_traits/lookup_classname
whether lookup_classname("W") is supposed to produce a result or not (but you seem to assume it does).

My understanding is that the "icase" parameter to lookup_classname is talking about the icaseness of the regex matcher; classnames should always be matched with exact case, i.e. [[:alnum:]] is always a valid classname and [[:ALNUM:]] is always invalid, regardless of regex_constants::icase. But I'm not sure.

[timshen]

Propagate __icase_ correctly into lookup_classname.

[timshen]

libcxx/test/std/re/re.traits/lookup_classname_user_defined.pass.cpp

47

[re.req] says that, for lookup_classname(), "The value returned shall be independent of the case of the characters in the sequence."

I take it as regardless of lookup_classname()'s icase argument, [[:ALNUM:]] is always valid.

There are existing tests that confirms it in std/re/re.traits/lookup_classname.pass.cpp. Search for "AlNum".

I fixed my patch, since I was misunderstanding it as well (I thought icase is for the input char sequence). Now they are just forwarded into lookup_classname().

[• Quuxplusone]

libcxx/test/std/re/re.traits/lookup_classname_user_defined.pass.cpp

47

[re.req] says that, for lookup_classname(), "The value returned shall be independent of the case of the characters in the sequence."

Huh. That seems like a defect in the Standard, since although lookup_classname() is parameterized on a locale, there is no standard way to get case-insensitive string comparison out of an arbitrary locale AFAIK. However, that's a problem for the implementer of lookup_classname(), not for you. Forwarding straight to lookup_classname() and letting it deal with questions of "case" sounds like the right approach to me.

If you *wanted* to go down this rabbit hole, a good test case would be "[[:DIGIT:]]" in Turkish locale (where lowercasing "[[:DIGIT:]]" produces "[[:dıgıt:]]" not "[[:digit:]]").

(Note— The only place I find "case-insensitive" in N4659 is in the informative note on time_get::get: "It is unspecified by what means the function performs case-insensitive comparison or whether multi-character sequences are considered while doing so." —end note)

libcxx/include/regex

class __bracket_expression
     typedef typename _Traits::string_type string_type;
 
     _Traits __traits_;
     vector<_CharT> __chars_;
     vector<_CharT> __neg_chars_;
     vector<pair<string_type, string_type> > __ranges_;
     vector<pair<_CharT, _CharT> > __digraphs_;
     vector<string_type> __equivalences_;
-    typename regex_traits<_CharT>::char_class_type __mask_;
-    typename regex_traits<_CharT>::char_class_type __neg_mask_;
+    typename _Traits::char_class_type __mask_;
+    typename _Traits::char_class_type __neg_mask_;
     bool __negate_;
     bool __icase_;
     bool __collate_;
     bool __might_have_digraph_;
 
     __bracket_expression(const __bracket_expression&);
     __bracket_expression& operator=(const __bracket_expression&);
 public:
void __add_digraph(_CharT __c1, _CharT __c2)
                 __digraphs_.push_back(make_pair(__traits_.translate(__c1),
                                                 __traits_.translate(__c2)));
             else
                 __digraphs_.push_back(make_pair(__c1, __c2));
         }
     _LIBCPP_INLINE_VISIBILITY
     void __add_equivalence(const string_type& __s)
         {__equivalences_.push_back(__s);}
+
+    template <class _Iter>
     _LIBCPP_INLINE_VISIBILITY
-    void __add_class(typename regex_traits<_CharT>::char_class_type __mask)
-        {__mask_ |= __mask;}
+    void __add_class(_Iter __begin, _Iter __end)
+    {
+      auto __class_type = __traits_.lookup_classname(__begin, __end, __icase_);
+      if (__class_type == 0)
+          __throw_regex_error<regex_constants::error_brack>();
+      __mask_ |= __class_type;
+    }
+
+    template <class _Iter>
     _LIBCPP_INLINE_VISIBILITY
-    void __add_neg_class(typename regex_traits<_CharT>::char_class_type __mask)
-        {__neg_mask_ |= __mask;}
+    void __add_neg_class(_Iter __begin, _Iter __end)
+    {
+      auto __class_type = __traits_.lookup_classname(__begin, __end, __icase_);
+      if (__class_type == 0)
+          __throw_regex_error<regex_constants::error_brack>();
+      __neg_mask_ |= __class_type;
+    }
 };
 
 template <class _CharT, class _Traits>
 void
 __bracket_expression<_CharT, _Traits>::__exec(__state& __s) const
 {
     bool __found = false;
     unsigned __consumed = 0;
basic_regex<_CharT, _Traits>::__parse_class_escape(_ForwardIterator __first,
     {
     case 0:
         __str = *__first;
         return ++__first;
     case 'b':
         __str = _CharT(8);
         return ++__first;
     case 'd':
-        __ml->__add_class(ctype_base::digit);
+        __ml->__add_class(__first, std::next(__first));
         return ++__first;
     case 'D':
-        __ml->__add_neg_class(ctype_base::digit);
+        __ml->__add_neg_class(__first, std::next(__first));
         return ++__first;
     case 's':
-        __ml->__add_class(ctype_base::space);
+        __ml->__add_class(__first, std::next(__first));
         return ++__first;
     case 'S':
-        __ml->__add_neg_class(ctype_base::space);
+        __ml->__add_neg_class(__first, std::next(__first));
         return ++__first;
     case 'w':
-        __ml->__add_class(ctype_base::alnum);
+        __ml->__add_class(__first, std::next(__first));
         __ml->__add_char('_');
         return ++__first;
     case 'W':
-        __ml->__add_neg_class(ctype_base::alnum);
+        __ml->__add_neg_class(__first, std::next(__first));
         __ml->__add_neg_char('_');
         return ++__first;
     }
     __first = __parse_character_escape(__first, __last, &__str);
     return __first;
 }
 
 template <class _CharT, class _Traits>
basic_regex<_CharT, _Traits>::__parse_character_class(_ForwardIterator __first,
     // Found [:
     //   This means :] must exist
     value_type _Colon_close[2] = {':', ']'};
     _ForwardIterator __temp = _VSTD::search(__first, __last, _Colon_close,
                                                             _Colon_close+2);
     if (__temp == __last)
         __throw_regex_error<regex_constants::error_brack>();
     // [__first, __temp) contains all text in [: ... :]
-    typedef typename _Traits::char_class_type char_class_type;
-    char_class_type __class_type =
-        __traits_.lookup_classname(__first, __temp, __flags_ & icase);
-    if (__class_type == 0)
-        __throw_regex_error<regex_constants::error_brack>();
-    __ml->__add_class(__class_type);
+    __ml->__add_class(__first, __temp);
     __first = _VSTD::next(__temp, 2);
     return __first;
 }
 
 template <class _CharT, class _Traits>
 template <class _ForwardIterator>
 _ForwardIterator
 basic_regex<_CharT, _Traits>::__parse_collating_symbol(_ForwardIterator __first,

 {
     if (__first != __last)
     {
         __bracket_expression<_CharT, _Traits>* __ml;
         switch (*__first)
         {
         case 'd':
             __ml = __start_matching_list(false);
-            __ml->__add_class(ctype_base::digit);
+            __ml->__add_class(__first, std::next(__first));
             ++__first;
             break;
         case 'D':
             __ml = __start_matching_list(true);
-            __ml->__add_class(ctype_base::digit);
+            __ml->__add_class(__first, std::next(__first));
             ++__first;
             break;
         case 's':
             __ml = __start_matching_list(false);
-            __ml->__add_class(ctype_base::space);
+            __ml->__add_class(__first, std::next(__first));
             ++__first;
             break;
         case 'S':
             __ml = __start_matching_list(true);
-            __ml->__add_class(ctype_base::space);
+            __ml->__add_class(__first, std::next(__first));
             ++__first;
             break;
         case 'w':
             __ml = __start_matching_list(false);
-            __ml->__add_class(ctype_base::alnum);
+            __ml->__add_class(__first, std::next(__first));
             __ml->__add_char('_');
             ++__first;
             break;
         case 'W':
             __ml = __start_matching_list(true);
-            __ml->__add_class(ctype_base::alnum);
+            __ml->__add_class(__first, std::next(__first));
             __ml->__add_char('_');
             ++__first;
             break;
         }
     }
     return __first;
 }
 

libcxx/test/std/re/re.traits/lookup_classname_user_defined.pass.cpp

+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <regex>
+
+// template <class charT> struct regex_traits;
+
+// template <class ForwardIterator>
+//   char_class_type
+//   lookup_classname(ForwardIterator first, ForwardIterator last,
+//                    bool icase = false) const;
+
+#include <regex>
+#include <cassert>
+#include "test_macros.h"
+#include "platform_support.h" // locale name macros
+
+struct wctype_traits : std::regex_traits<wchar_t>
+{
+    using char_class_type = std::wctype_t;
+    template<class ForwardIt>
+    char_class_type lookup_classname(ForwardIt first, ForwardIt last, bool icase = false ) const {
+        (void)icase;
+        return std::wctype(std::string(first, last).c_str());
+    }
+    bool isctype(wchar_t c, char_class_type f) const {
+        return std::iswctype(c, f);
+    }
+};
+
+int main()
+{
+    std::locale::global(std::locale("ja_JP.utf8"));
+    std::wsmatch m;
+    std::wstring in = L"風の谷のナウシカ";
+
+    // matches all characters (they are classified as alnum)
+    std::wstring re1 = L"([[:alnum:]]+)";
+    std::regex_search(in, m, std::wregex(re1));
+    assert(m[1] == L"風の谷のナウシカ");
+
+    // matches only the kanji
+    std::wstring re2 = L"([[:jkata:]]+)";
+    std::regex_search(in, m, std::basic_regex<wchar_t, wctype_traits>(re2));
+    assert(m[1] == L"ナウシカ");
+}

libcxx/utils/libcxx/test/target_info.py

def add_common_locales(features, lit_config, is_windows=False):
     # A list of locales needed by the test-suite.
     # The list uses the canonical name for the locale used in the test-suite
     # TODO: On Linux ISO8859 *may* needs to hyphenated.
     locales = [
         ('en_US.UTF-8', 'English_United States.1252'),
         ('fr_FR.UTF-8', 'French_France.1252'),
         ('ru_RU.UTF-8', 'Russian_Russia.1251'),
         ('zh_CN.UTF-8', 'Chinese_China.936'),
+        ('ja_JP.UTF-8', 'Japanese_Japan.932'),
         ('fr_CA.ISO8859-1', 'French_Canada.1252'),
         ('cs_CZ.ISO8859-2', 'Czech_Czech Republic.1250')
     ]
     for loc_id, windows_loc_name in locales:
         loc_name = windows_loc_name if is_windows else loc_id
         if test_locale(loc_name):
             features.add('locale.{0}'.format(loc_id))
         else: