Add support for digit separators in C2x

[aaron.ballman]

Summary

WG14 adopted N2626 at the meetings this week. This proposal adds support for using ' as a digit separator in a numeric literal which is compatible with the C++ feature.

[rjmccall]

LGTM

[aaron.ballman]

Thanks for the review, I've commit in e448310059053d69fbdd6c66dd95ad5d3cfc6243

clang/include/clang/Basic/DiagnosticLexKinds.td

def ext_unknown_escape : ExtWarn<"unknown escape sequence '\\%0'">,
   InGroup<DiagGroup<"unknown-escape-sequence">>;
 def err_invalid_digit : Error<
   "invalid digit '%0' in %select{decimal|octal|binary}1 constant">;
 def err_invalid_suffix_constant : Error<
   "invalid suffix '%0' on %select{integer|floating|fixed-point}1 constant">;
 def warn_cxx11_compat_digit_separator : Warning<
   "digit separators are incompatible with C++ standards before C++14">,
   InGroup<CXXPre14Compat>, DefaultIgnore;
+def warn_c2x_compat_digit_separator : Warning<
+  "digit separators are incompatible with C standards before C2x">,
+  InGroup<CPre2xCompat>, DefaultIgnore;
 def err_digit_separator_not_between_digits : Error<
   "digit separator cannot appear at %select{start|end}0 of digit sequence">;
 def warn_extraneous_char_constant : Warning<
   "extraneous characters in character constant ignored">;
 def warn_char_constant_too_large : Warning<
   "character constant too long for its type">;
 def err_multichar_utf_character_literal : Error<
   "Unicode character literals may not contain multiple characters">;

clang/lib/Lex/Lexer.cpp

                std::find(BufferPtr, CurPtr, '_') != CurPtr)
         IsHexFloat = false;
     }
     if (IsHexFloat)
       return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
   }
 
   // If we have a digit separator, continue.
-  if (C == '\'' && getLangOpts().CPlusPlus14) {
+  if (C == '\'' && (getLangOpts().CPlusPlus14 || getLangOpts().C2x)) {
     unsigned NextSize;
     char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts());
     if (isIdentifierBody(Next)) {
       if (!isLexingRawMode())
-        Diag(CurPtr, diag::warn_cxx11_compat_digit_separator);
+        Diag(CurPtr, getLangOpts().CPlusPlus
+                         ? diag::warn_cxx11_compat_digit_separator
+                         : diag::warn_c2x_compat_digit_separator);
       CurPtr = ConsumeChar(CurPtr, Size, Result);
       CurPtr = ConsumeChar(CurPtr, NextSize, Result);
       return LexNumericConstant(Result, CurPtr);
     }
   }
 
   // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
   if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))

clang/test/Lexer/c2x_digit_separators.c

+// RUN: %clang_cc1 -std=c2x -verify %s
+
+_Static_assert(1'2'3 == 12'3, "");
+_Static_assert(1'000'000 == 0xf'4240, "");
+_Static_assert(0'004'000'000 == 0x10'0000, "");
+_Static_assert(0b0101'0100 == 0x54, "");
+
+int a0 = 123'; //'; // expected-error {{expected ';'}}
+int b0 = 0'xff; // expected-error {{digit separator cannot appear at end of digit sequence}} expected-error {{suffix 'xff' on integer}}
+int c0 = 0x'ff; // expected-error {{suffix 'x'ff' on integer}}
+int d0 = 0'1234; // ok, octal
+int e0 = 0'b1010; // expected-error {{digit 'b' in octal constant}}
+int f0 = 0b'1010; // expected-error {{invalid digit 'b' in octal}}
+int h0 = 0x1e+1; // expected-error {{invalid suffix '+1' on integer constant}}
+int i0 = 0x1'e+1; // ok, 'e+' is not recognized after a digit separator
+
+float a1 = 1'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+float b1 = 1'0e1;
+float c1 = 1.'0e1; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float d1 = 1.0'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+float e1 = 1e'1; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float g1 = 0.'0; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float h1 = .'0; // '; // expected-error {{expected expression}}, lexed as . followed by character literal
+float i1 = 0x.'0p0; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float j1 = 0x'0.0p0; // expected-error {{invalid suffix 'x'0.0p0'}}
+float k1 = 0x0'.0p0; // '; // expected-error {{expected ';'}}
+float l1 = 0x0.'0p0; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float m1 = 0x0.0'p0; // expected-error {{digit separator cannot appear at end of digit sequence}}
+float n1 = 0x0.0p'0; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float p1 = 0'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+float q1 = 0'0e1;
+float r1 = 0.'0e1; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float s1 = 0.0'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+float t1 = 0.0e'1; // expected-error {{digit separator cannot appear at start of digit sequence}}
+float u1 = 0x.'p1f; // expected-error {{hexadecimal floating constant requires a significand}}
+float v1 = 0e'f; // expected-error {{exponent has no digits}}
+float w1 = 0x0p'f; // expected-error {{exponent has no digits}}
+float x1 = 0'e+1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+float y1 = 0x0'p+1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+
+#line 123'456
+_Static_assert(__LINE__ == 123456, "");
+
+// UCNs can appear before digit separators but not after.
+int a2 = 0\u1234'5; // expected-error {{invalid suffix '\u1234'5' on integer constant}}
+int b2 = 0'\u12345; // '; // expected-error {{expected ';'}}
+
+// extended characters can appear before digit separators but not after.
+int a3 = 0ሴ'5; // expected-error {{invalid suffix 'ሴ'5' on integer constant}}
+int b3 = 0'ሴ5; // '; // expected-error {{expected ';'}}
+

clang/test/Sema/pre-c2x-compat.c

+// RUN: %clang_cc1 %s -std=c2x -Wpre-c2x-compat -pedantic -fsyntax-only -verify
+
+int digit_seps = 123'456; // expected-warning {{digit separators are incompatible with C standards before C2x}}