[Clang] Adjust extension warnings for delimited sequences

[cor3ntin]

Summary

WG21 approved delimited escape sequences and named escape
sequences.
Adjust the extension warnings accordingly, and update
the release notes.

[cor3ntin]

Forgot to format!

[cor3ntin]

@aaron.ballman I think we might as well do that now, and revert if WG21 does some funny business in plenary. WDYT?

[aaron.ballman]

LGTM! If for some very surprising reason either of these papers fail in plenary, we can easily revert this as necessary.

[jakubjelinek]

While accepting all these inside of string and character literals in C and C++20 and older is fine, accepting them inside of identifiers can change meaning of valid programs.
https://gcc.gnu.org/pipermail/gcc-patches/2022-August/600620.html
#define z(x) 0
#define a z(
int x = a\N{LATIN SMALL LETTER U WITH DIAERESIS});
int y = a\u{1234});
int z = a\U{12345678});

[tahonermann]

While accepting all these inside of string and character literals in C and C++20 and older is fine, accepting them inside of identifiers can change meaning of valid programs.

Thank you for reporting this impact, @jakubjelinek! I'll copy you on an email to WG21 that discussed the impact, specifically with regard to the glibc header for which Joseph reported failures.

clang/docs/ReleaseNotes.rst

 
 C++2b Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
 - Implemented `P2128R6: Multidimensional subscript operator <https://wg21.link/P2128R6>`_.
 - Implemented `P0849R8: auto(x): decay-copy in the language <https://wg21.link/P0849R8>`_.
 - Implemented `P2242R3: Non-literal variables (and labels and gotos) in constexpr functions <https://wg21.link/P2242R3>`_.
 - Implemented `LWG3659: Consider ATOMIC_FLAG_INIT undeprecation <https://wg21.link/LWG3659>`_.
+- Implemented `P2290 Delimited escape sequences <https://wg21.link/P2290R3>`_.
+  This feature is available as an extension in all C and C++ language modes.
+- Implemented `P2071 Named universal character escapes <https://wg21.link/P2290R2>`_.
+  This feature is available as an extension in all C and C++ language modes.
 
 CUDA/HIP Language Changes in Clang
 ----------------------------------
 
 - Added `__noinline__` as a keyword to avoid diagnostics due to usage of
   `__attribute__((__noinline__))` in CUDA/HIP programs.
 
 Objective-C Language Changes in Clang

clang/include/clang/Basic/DiagnosticLexKinds.td

 def warn_utf8_symbol_homoglyph : Warning<
   "treating Unicode character <U+%0> as identifier character rather than "
   "as '%1' symbol">, InGroup<DiagGroup<"unicode-homoglyph">>;
 def warn_utf8_symbol_zero_width : Warning<
   "identifier contains Unicode character <U+%0> that is invisible in "
   "some environments">, InGroup<DiagGroup<"unicode-zero-width">>;
 
 def ext_delimited_escape_sequence : Extension<
-  "%select{delimited|named}0 escape sequences are a Clang extension">,
+  "%select{delimited|named}0 escape sequences are a "
+  "%select{Clang|C++2b}1 extension">,
   InGroup<DiagGroup<"delimited-escape-sequence-extension">>;
+
+def warn_cxx2b_delimited_escape_sequence : Warning<
+  "%select{delimited|named}0 escape sequences are "
+  "incompatible with C++ standards before C++2b">,
+  InGroup<CXXPre2bCompat>, DefaultIgnore;
+
 def err_delimited_escape_empty : Error<
   "delimited escape sequence cannot be empty">;
 def err_delimited_escape_missing_brace: Error<
   "expected '{' after '\\%0' escape sequence">;
 def err_delimited_escape_invalid : Error<
   "invalid digit '%0' in escape sequence">;
 def err_hex_escape_no_digits : Error<
   "\\%0 used with no following hex digits">;

clang/lib/Lex/Lexer.cpp

if (Diagnose) {
         Diag(KindLoc, diag::note_ucn_four_not_eight)
             << FixItHint::CreateReplacement(URange, "u");
       }
     }
     return llvm::None;
   }
 
   if (Delimited && PP) {
-    Diag(BufferPtr, diag::ext_delimited_escape_sequence) << /*delimited*/ 0;
+    Diag(BufferPtr, PP->getLangOpts().CPlusPlus2b
+                        ? diag::warn_cxx2b_delimited_escape_sequence
+                        : diag::ext_delimited_escape_sequence)
+        << /*delimited*/ 0 << (PP->getLangOpts().CPlusPlus ? 1 : 0);
   }
 
   if (Result) {
     Result->setFlag(Token::HasUCN);
     if (CurPtr - StartPtr == (ptrdiff_t)(Count + 2 + (Delimited ? 2 : 0)))
       StartPtr = CurPtr;
     else
       while (StartPtr != CurPtr)
if (!Res) {
     if (!LooseMatch)
       return llvm::None;
     // We do not offer missspelled character names suggestions here
     // as the set of what would be a valid suggestion depends on context,
     // and we should not make invalid suggestions.
   }
 
   if (Diagnose && PP && !LooseMatch)
-    Diag(BufferPtr, diag::ext_delimited_escape_sequence) << /*named*/ 1;
+    Diag(BufferPtr, PP->getLangOpts().CPlusPlus2b
+                        ? diag::warn_cxx2b_delimited_escape_sequence
+                        : diag::ext_delimited_escape_sequence)
+        << /*named*/ 1 << (PP->getLangOpts().CPlusPlus ? 1 : 0);
 
   if (LooseMatch)
     Res = LooseMatch->CodePoint;
 
   if (Result) {
     Result->setFlag(Token::HasUCN);
     if (CurPtr - StartPtr == (ptrdiff_t)(Buffer.size() + 4))
       StartPtr = CurPtr;

clang/lib/Lex/LiteralSupport.cpp

static unsigned ProcessCharEscape(const char *ThisTokBegin,
 
   if (Delimited && Diags) {
     if (!EndDelimiterFound)
       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
            diag::err_expected)
           << tok::r_brace;
     else if (!HadError) {
       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
-           diag::ext_delimited_escape_sequence)
-          << /*delimited*/ 0;
+           Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence
+                                : diag::ext_delimited_escape_sequence)
+          << /*delimited*/ 0 << (Features.CPlusPlus ? 1 : 0);
     }
   }
 
   return ResultChar;
 }
 
 static void appendCodePoint(unsigned Codepoint,
                             llvm::SmallVectorImpl<char> &Str) {
static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
   }
 
   if (!Features.CPlusPlus && !Features.C99 && Diags)
     Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
          diag::warn_ucn_not_valid_in_c89_literal);
 
   if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags)
     Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
-         diag::ext_delimited_escape_sequence)
-        << (IsNamedEscapeSequence ? 1 : 0);
+         Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence
+                              : diag::ext_delimited_escape_sequence)
+        << (IsNamedEscapeSequence ? 1 : 0) << (Features.CPlusPlus ? 1 : 0);
 
   return true;
 }
 
 /// MeasureUCNEscape - Determine the number of bytes within the resulting string
 /// which this UCN will occupy.
 static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
                             const char *ThisTokEnd, unsigned CharByteWidth,

clang/test/Lexer/char-escapes-delimited.c

-// RUN: %clang_cc1 -x c++ -std=gnu++11 -fsyntax-only -pedantic -verify %s
-// RUN: %clang_cc1 -x c -std=gnu11 -fsyntax-only -pedantic -verify %s
-// RUN: %clang_cc1 -x c++ -std=gnu++11 -fwchar-type=short -fno-signed-wchar -fsyntax-only -pedantic -verify %s
-// RUN: %clang_cc1 -x c -std=gnu11 -fwchar-type=short -fno-signed-wchar -fsyntax-only -pedantic -verify %s
-// RUN: %clang_cc1 -x c++ -std=c++17 -ftrigraphs -fsyntax-only -pedantic -verify -DTRIGRAPHS=1 %s
+// RUN: %clang_cc1 -x c++ -std=gnu++11 -fsyntax-only -pedantic -verify=ext,expected %s
+// RUN: %clang_cc1 -x c -std=gnu11 -fsyntax-only -pedantic -verify=ext,expected %s
+// RUN: %clang_cc1 -x c++ -std=c++2b -fsyntax-only -pedantic -verify=cxx2b,expected -Wpre-c++2b-compat %s
+// RUN: %clang_cc1 -x c++ -std=gnu++11 -fwchar-type=short -fno-signed-wchar -fsyntax-only -pedantic -verify=ext,expected %s
+// RUN: %clang_cc1 -x c -std=gnu11 -fwchar-type=short -fno-signed-wchar -fsyntax-only -pedantic -verify=ext,expected %s
+// RUN: %clang_cc1 -x c++ -std=c++17 -ftrigraphs -fsyntax-only -pedantic -verify=ext,expected -DTRIGRAPHS=1 %s
 
 const char *errors =
     "\u{}"  // expected-error {{delimited escape sequence cannot be empty}}
     "\u{"   // expected-error {{expected '}'}}
     "\u{h}" // expected-error {{invalid digit 'h' in escape sequence}}
     "\x{}"  // expected-error {{delimited escape sequence cannot be empty}}
     "\x{"   // expected-error {{expected '}'}}
     "\x{h}" // expected-error {{invalid digit 'h' in escape sequence}}
     "\o{}"  // expected-error {{delimited escape sequence cannot be empty}}
     "\o{"   // expected-error {{expected '}'}}
     "\o"    // expected-error {{expected '{' after '\o' escape sequence}}
     "\o{8}" // expected-error {{invalid digit '8' in escape sequence}}
     "\U{8}" // expected-error {{\U used with no following hex digits}}
     ;
 
 void ucn(void) {
-  char a = '\u{1234}'; // expected-error {{character too large for enclosing character literal type}}
-                       // expected-warning@-1 {{delimited escape sequences are a Clang extension}}
+  char a = '\u{1234}'; // expected-error {{character too large for enclosing character literal type}} \
+                       // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 
-  unsigned b = U'\u{1234}'; // expected-warning {{extension}}
+  unsigned b = U'\u{1234}';   // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 
 #ifdef __cplusplus
-  unsigned b2 = U'\u{1}'; // expected-warning {{extension}}
+  unsigned b2 = U'\u{1}';     // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 #else
   unsigned b2 = U'\u{1}';     //expected-error {{universal character name refers to a control character}}
 #endif
 
-  unsigned c = U'\u{000000000001234}'; // expected-warning {{extension}}
+  unsigned c = U'\u{000000000001234}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   unsigned d = U'\u{111111111}';       //expected-error {{hex escape sequence out of range}}
 }
 
 void hex(void) {
-  char a = '\x{1}';             // expected-warning {{extension}}
+  char a = '\x{1}';             // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   char b = '\x{abcdegggggabc}'; // expected-error 5{{invalid digit 'g' in escape sequence}}
   char c = '\x{ff1}';           // expected-error {{hex escape sequence out of range}}
 
 #if __WCHAR_MAX__ > 0xFFFF
-  unsigned d = L'\x{FFFFFFFF}';  // expected-warning {{extension}}
+  unsigned d = L'\x{FFFFFFFF}';  // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   unsigned e = L'\x{100000000}'; // expected-error {{hex escape sequence out of range}}
 #else
-  unsigned f = L'\x{FFFF}';   // expected-warning {{extension}}
+  unsigned f = L'\x{FFFF}';   // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   unsigned g = L'\x{10000}';  // expected-error {{hex escape sequence out of range}}
 #endif
-  unsigned h = U'\x{FFFFFFFF}';  // expected-warning {{extension}}
+  unsigned h = U'\x{FFFFFFFF}';  // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   unsigned i = U'\x{100000000}'; // expected-error {{hex escape sequence out of range}}
 }
 
 void octal(void) {
-  char a = '\o{1}';              // expected-warning {{extension}}
+  char a = '\o{1}';              // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   char b = '\o{12345678881238}'; // expected-error 4{{invalid digit '8' in escape sequence}}
   char c = '\o{777}';            // //expected-error {{octal escape sequence out of range}}
 #if __WCHAR_MAX__ > 0xFFFF
-  unsigned d = L'\o{37777777777}'; // expected-warning {{extension}}
+  unsigned d = L'\o{37777777777}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   unsigned e = L'\o{40000000000}'; // expected-error {{octal escape sequence out of range}}
 #else
-  unsigned d = L'\o{177777}'; // expected-warning {{extension}}
+  unsigned d = L'\o{177777}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   unsigned e = L'\o{200000}'; // expected-error {{octal escape sequence out of range}}
 #endif
 }
 
 void concat(void) {
   (void)"\x{" "12}"; // expected-error {{expected '}'}}
   (void)"\u{" "12}"; // expected-error {{expected '}'}}
   (void)"\o{" "12}"; // expected-error {{expected '}'}}
 
   (void)"\x{12" "}"; // expected-error {{expected '}'}}
   (void)"\u{12" "}"; // expected-error {{expected '}'}}
   (void)"\o{12" "}"; // expected-error {{expected '}'}}
 }
 
 void named(void) {
   char a = '\N{LOTUS}'; // expected-error{{character too large for enclosing character literal type}} \
-                        // expected-warning {{extension}}
+                        // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 
-  char b  = '\N{DOLLAR SIGN}'; // expected-warning {{extension}}
+  char b  = '\N{DOLLAR SIGN}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   char b_ = '\N{ DOL-LAR _SIGN }'; // expected-error {{' DOL-LAR _SIGN ' is not a valid Unicode character name}} \
                                // expected-note {{characters names in Unicode escape sequences are sensitive to case and whitespaces}}
 
   char c = '\N{NOTATHING}'; // expected-error {{'NOTATHING' is not a valid Unicode character name}} \
                             // expected-note 5{{did you mean}}
   char d = '\N{}';          // expected-error {{delimited escape sequence cannot be empty}}
   char e = '\N{';           // expected-error {{incomplete universal character name}}
 
-  unsigned f = L'\N{GREEK CAPITAL LETTER DELTA}'; // expected-warning {{extension}}
+  unsigned f = L'\N{GREEK CAPITAL LETTER DELTA}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 
   unsigned g = u'\N{LOTUS}'; // expected-error {{character too large for enclosing character literal type}} \
-                             // expected-warning {{extension}}
+                             // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 
-  unsigned h = U'\N{LOTUS}';                      // expected-warning {{extension}}
-  unsigned i = u'\N{GREEK CAPITAL LETTER DELTA}'; // expected-warning {{extension}}
+  unsigned h = U'\N{LOTUS}';                      // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned i = u'\N{GREEK CAPITAL LETTER DELTA}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
   char j = '\NN';                                 // expected-error {{expected '{' after '\N' escape sequence}}
   unsigned k = u'\N{LOTUS';                       // expected-error {{incomplete universal character name}}
 }
 
 void separators(void) {
   (void)"\x{12'3}"; // expected-error {{invalid digit ''' in escape sequence}}
   (void)"\u{12'3}"; // expected-error {{invalid digit ''' in escape sequence}}
   (void)"\o{12'3}"; // expected-error {{invalid digit ''' in escape sequence}}
 
   '\x{12'3'}';   // expected-error {{expected '}'}}
                  // expected-error@-1 2{{expected ';'}}
                  // expected-warning@-2 3{{expression result unused}}
 }
 
-#if L'\N{GREEK CAPITAL LETTER GAMMA}' != L'Γ' // expected-warning {{extension}}
+#if L'\N{GREEK CAPITAL LETTER GAMMA}' != L'Γ' // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 #error "oh no!"
 #endif
 
 #ifdef TRIGRAPHS
 static_assert('\N??<DOLLAR SIGN??>' == '$'); // expected-warning 2{{trigraph converted}} \
-                                             // expected-warning {{named escape sequences are a Clang extension}}
+                                             // ext-warning {{extension}} cxx2b-warning {{C++2b}}
 #endif

clang/test/Preprocessor/ucn-pp-identifier.c

-// RUN: %clang_cc1 %s -fsyntax-only -std=c99 -pedantic -verify -Wundef
-// RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify -Wundef
-// RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify -Wundef -ftrigraphs -DTRIGRAPHS=1
+// RUN: %clang_cc1 %s -fsyntax-only -std=c99 -pedantic -verify=expected,ext -Wundef
+// RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify=expected,ext -Wundef
+// RUN: %clang_cc1 %s -fsyntax-only -x c++ -std=c++2b -pedantic -ftrigraphs -verify=expected,cxx2b -Wundef -Wpre-c++2b-compat
+// RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify=expected,ext -Wundef -ftrigraphs -DTRIGRAPHS=1
 // RUN: not %clang_cc1 %s -fsyntax-only -std=c99 -pedantic -Wundef 2>&1 | FileCheck -strict-whitespace %s
 
 #define \u00FC
 #define a\u00FD() 0
 #ifndef \u00FC
 #error "This should never happen"
 #endif
 
 #if a\u00FD()
 #error "This should never happen"
 #endif
 
 #if a\U000000FD()
 #error "This should never happen"
 #endif
 
-#if a\u{FD}() //expected-warning {{Clang extension}}
+#if a\u{FD}() // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
 #error "This should never happen"
 #endif
 
 #if \uarecool // expected-warning{{incomplete universal character name; treating as '\' followed by identifier}} expected-error {{invalid token at start of a preprocessor expression}}
 #endif
 #if \uwerecool // expected-warning{{\u used with no following hex digits; treating as '\' followed by identifier}} expected-error {{invalid token at start of a preprocessor expression}}
 #endif
 #if \U0001000  // expected-warning{{incomplete universal character name; treating as '\' followed by identifier}} expected-error {{invalid token at start of a preprocessor expression}}
 #endif
 
 // Make sure we reject disallowed UCNs
 #define \ufffe // expected-error {{macro name must be an identifier}}
-#define \U10000000       // expected-error {{macro name must be an identifier}}
-#define \u0061           // expected-error {{character 'a' cannot be specified by a universal character name}} expected-error {{macro name must be an identifier}}
-#define \u{fffe}        // expected-error {{macro name must be an identifier}} expected-warning {{Clang extension}}
+#define \U10000000      // expected-error {{macro name must be an identifier}}
+#define \u0061          // expected-error {{character 'a' cannot be specified by a universal character name}} expected-error {{macro name must be an identifier}}
+#define \u{fffe}        // expected-error {{macro name must be an identifier}} \
+                        // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
 #define \N{ALERT}       // expected-error {{universal character name refers to a control character}} \
                    // expected-error {{macro name must be an identifier}} \
-                   // expected-warning {{Clang extension}}
+                   // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
 #define \N{WASTEBASKET} // expected-error {{macro name must be an identifier}} \
-                        // expected-warning {{Clang extension}}
+                        // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
 
 #define a\u0024
 
 #if \u0110 // expected-warning {{is not defined, evaluates to 0}}
 #endif
 
 
 #define \u0110 1 / 0

 #define \N{GREEK_SMALL-LETTERALPHA}  // expected-error {{'GREEK_SMALL-LETTERALPHA' is not a valid Unicode character name}} \
                                      // expected-note {{characters names in Unicode escape sequences are sensitive to case and whitespaces}}
 
 #define CONCAT(A, B) A##B
 int CONCAT(\N{GREEK, CAPITALLETTERALPHA}); // expected-error{{expected}} \
                                            // expected-warning {{incomplete delimited universal character name}}
 
 #ifdef TRIGRAPHS
-int \N??<GREEK CAPITAL LETTER ALPHA??> = 0; // expected-warning{{amed escape sequences are a Clang extension}} \
+int \N??<GREEK CAPITAL LETTER ALPHA??> = 0; // expected-warning{{extension}} cxx2b-warning {{before C++2b}} \
                                             // expected-warning 2{{trigraph converted}}
 
 #endif

clang/test/Sema/ucn-identifiers.c

 // expected-note@-4 + {{declared here}}
 #endif
 
 void goodCalls(void) {
   \u00FCber(0);
   \u00fcber(1);
   über(2);
   \U000000FCber(3);
-  \u{FC}ber(4); // expected-warning {{Clang extension}}
-  \N{LATIN SMALL LETTER U WITH DIAERESIS}ber(4); // expected-warning {{Clang extension}}
+  \u{FC}ber(4); // expected-warning {{extension}}
+  \N{LATIN SMALL LETTER U WITH DIAERESIS}ber(4); // expected-warning {{extension}}
 }
 
 void badCalls(void) {
   \u00FCber(0.5); // expected-warning{{implicit conversion from 'double' to 'int'}}
   \u00fcber = 0; // expected-error{{non-object type 'void (int)' is not assignable}}
 
   über(1, 2);
   \U000000FCber();
 #ifdef __cplusplus
   // expected-error@-3 {{no matching function}}
   // expected-error@-3 {{no matching function}}
 #else
   // expected-error@-6 {{too many arguments to function call, expected 1, have 2}}
   // expected-error@-6 {{too few arguments to function call, expected 1, have 0}}
 #endif
 }