[libc++][format] Fixes UTF-8 continuation.

[Mordante]

Summary

The mask used to check whether a code unit is a valid continuation was
incorrect and accepts non-continuation code points. This fixes the
issue.

[Mordante]

Rebased.

[Mordante]

@tahonermann when you have time can you have a look?

[tahonermann]

The change looks right to me and the added tests look good. I left a couple of suggested edits with comments for you to do with as you please.

libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp

508–513

I'm not sure what purpose is served by the reference to PR-121 here. The test appears to demonstrate that code unit values are preserved as escaped values which, I think, is the right behavior according to the C++ standard, but is not a match for any of the PR-121 policies.

libcxx/test/std/utilities/format/format.functions/unicode.pass.cpp

277–279

I eventually figured out how the mask in the comment correlated with the format string and why each was relevant, but I wouldn't expect many people to understand the intended relevance without a more descriptive comment. I suggest just removing these comments.

[Mordante]

Thanks for the review!

libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp

508–513

That is a left over debug message for me, removed it.

[Mordante]

Rebased and address review comments.

[tahonermann]

Looks good!

[Mordante]

Rebased so D150044 can be rebased.

[Mordante]

Rebased to fix CI.

libcxx/include/__format/unicode.h

 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_scalar_value(char32_t __value) {
   return __unicode::__is_code_point(__value) && !__unicode::__is_surrogate(__value);
 }
 
 template <contiguous_iterator _Iterator>
   requires same_as<iter_value_t<_Iterator>, char>
 _LIBCPP_HIDE_FROM_ABI constexpr bool __is_continuation(_Iterator __char, int __count) {
   do {
-    if ((*__char & 0b1000'0000) != 0b1000'0000)
+    if ((*__char & 0b1100'0000) != 0b1000'0000)
       return false;
     --__count;
     ++__char;
   } while (__count);
   return true;
 }
 
 /// Helper class to extract a code unit from a Unicode character range.

libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp

test_format(R"("\x{f7}\x{bf}a")"sv,
               "{:?}",
               "\xf7\xbf"
               "a");
   test_format(R"("\x{f7}\x{bf}\x{bf}a")"sv,
               "{:?}",
               "\xf7\xbf\xbf"
               "a");
 
+  // http://unicode.org/review/pr-121.html
+  test_format(R"("a\x{f1}\x{80}\x{80}\x{e1}\x{80}\x{c2}b")"sv,
+              "{:?}",
+              "a"
+              "\xf1\x80\x80\xe1\x80\xc2"
+              "b");
+
   // Code unit out of range
   test_format(R"("\u{10ffff}")"sv, "{:?}", "\xf4\x8f\xbf\xbf");               // last valid code point
   test_format(R"("\x{f4}\x{90}\x{80}\x{80}")"sv, "{:?}", "\xf4\x90\x80\x80"); // first invalid code point
   test_format(R"("\x{f5}\x{b1}\x{b2}\x{b3}")"sv, "{:?}", "\xf5\xb1\xb2\xb3");
   test_format(R"("\x{f7}\x{bf}\x{bf}\x{bf}")"sv, "{:?}", "\xf7\xbf\xbf\xbf"); // largest encoded code point
 }
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS

libcxx/test/std/utilities/format/format.functions/unicode.pass.cpp

if constexpr (sizeof(CharT) == 1) {
     check(SV("*ZZZZ\xefZZZ*"), SV("{:*^10}"), SV("ZZZZ\xefZZZ"));
     check(SV("*ZZZZ\xffZZZ*"), SV("{:*^10}"), SV("ZZZZ\xffZZZ"));
 
     check(SV("*ZZZZ\x8fZZZZ*"), SV("{:*^11}"), SV("ZZZZ\x8fZZZZ"));
     check(SV("*ZZZZ\xcfZZZZ*"), SV("{:*^11}"), SV("ZZZZ\xcfZZZZ"));
     check(SV("*ZZZZ\xefZZZZ*"), SV("{:*^11}"), SV("ZZZZ\xefZZZZ"));
     check(SV("*ZZZZ\xffZZZZ*"), SV("{:*^11}"), SV("ZZZZ\xffZZZZ"));
 
+    // Invalid continuations
+    check(SV("\xc2\x00"), SV("{}"), SV("\xc2\x00")); // 0b0000'0000
+    check(SV("\xc2\x40"), SV("{}"), SV("\xc2\x40")); // 0b0100'0000
+    check(SV("\xc2\xc0"), SV("{}"), SV("\xc2\xc0")); // 0b1100'0000
+
+    check(SV("\xe0\x00\x80"), SV("{}"), SV("\xe0\x00\x80"));
+    check(SV("\xe0\x40\x80"), SV("{}"), SV("\xe0\x40\x80"));
+    check(SV("\xe0\xc0\x80"), SV("{}"), SV("\xe0\xc0\x80"));
+
+    check(SV("\xe0\x80\x00"), SV("{}"), SV("\xe0\x80\x00"));
+    check(SV("\xe0\x80\x40"), SV("{}"), SV("\xe0\x80\x40"));
+    check(SV("\xe0\x80\xc0"), SV("{}"), SV("\xe0\x80\xc0"));
+
+    check(SV("\xf0\x80\x80\x00"), SV("{}"), SV("\xf0\x80\x80\x00"));
+    check(SV("\xf0\x80\x80\x40"), SV("{}"), SV("\xf0\x80\x80\x40"));
+    check(SV("\xf0\x80\x80\xc0"), SV("{}"), SV("\xf0\x80\x80\xc0"));
+
+    check(SV("\xf0\x80\x00\x80"), SV("{}"), SV("\xf0\x80\x00\x80"));
+    check(SV("\xf0\x80\x40\x80"), SV("{}"), SV("\xf0\x80\x40\x80"));
+    check(SV("\xf0\x80\xc0\x80"), SV("{}"), SV("\xf0\x80\xc0\x80"));
+
+    check(SV("\xf0\x00\x80\x80"), SV("{}"), SV("\xf0\x00\x80\x80"));
+    check(SV("\xf0\x40\x80\x80"), SV("{}"), SV("\xf0\x40\x80\x80"));
+    check(SV("\xf0\xc0\x80\x80"), SV("{}"), SV("\xf0\xc0\x80\x80"));
+
     // Premature end.
     check(SV("*ZZZZ\xef\xf5*"), SV("{:*^8}"), SV("ZZZZ\xef\xf5"));
     check(SV("*ZZZZ\xef\xf5ZZZZ*"), SV("{:*^12}"), SV("ZZZZ\xef\xf5ZZZZ"));
     check(SV("*ZZZZ\xff\xf5\xf5*"), SV("{:*^9}"), SV("ZZZZ\xff\xf5\xf5"));
     check(SV("*ZZZZ\xff\xf5\xf5ZZZZ*"), SV("{:*^13}"), SV("ZZZZ\xff\xf5\xf5ZZZZ"));
 
   } else if constexpr (sizeof(CharT) == 2) {
     // TODO FMT Add these tests.