Don't split NumericConstant tokens if a digit separator proceeds a period.

[craig.topper]

Summary

Currently we split a token if a digit separator proceeds a period. This causes the digit separator to be interpreted instead as the start of a character constant token. gcc seems to keep it as a whole token for this case and issues an error for a digit separator being adjacent to a period.

Parts of NumericLiteralParser already expected that the digit separator could appear before a period and attempted to diagnose accordingly.

lib/Lex/Lexer.cpp

if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) {
     if (IsHexFloat)
       return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
   }
 
   // If we have a digit separator, continue.
   if (C == '\'' && getLangOpts().CPlusPlus14) {
     unsigned NextSize;
     char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts());
-    if (isIdentifierBody(Next)) {
+    if (isPreprocessingNumberBody(Next)) {
       if (!isLexingRawMode())
         Diag(CurPtr, diag::warn_cxx11_compat_digit_separator);
       CurPtr = ConsumeChar(CurPtr, Size, Result);
       CurPtr = ConsumeChar(CurPtr, NextSize, Result);
       return LexNumericConstant(Result, CurPtr);
     }
   }
 

lib/Lex/LiteralSupport.cpp

if ((c1 == 'x' || c1 == 'X') && (isHexDigit(s[1]) || s[1] == '.')) {
     assert(s < ThisTokEnd && "didn't maximally munch?");
     radix = 16;
     DigitsBegin = s;
     s = SkipHexDigits(s);
     bool noSignificand = (s == DigitsBegin);
     if (s == ThisTokEnd) {
       // Done.
     } else if (*s == '.') {
+      checkSeparator(TokLoc, s, CSK_AfterDigits);
       s++;
       saw_period = true;
       const char *floatDigitsBegin = s;
       checkSeparator(TokLoc, s, CSK_BeforeDigits);
       s = SkipHexDigits(s);
       noSignificand &= (floatDigitsBegin == s);
     }
 

test/Lexer/cxx1y_digit_separators.cpp

namespace floating {
   float c = 1.'0e1; // expected-error {{digit separator cannot appear at start of digit sequence}}
   float d = 1.0'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
   float e = 1e'1; // expected-error {{digit separator cannot appear at start of digit sequence}}
   float f = 1e1'ms; // expected-error {{digit separator cannot appear at end of digit sequence}}
   float g = 0.'0; // expected-error {{digit separator cannot appear at start of digit sequence}}
   float h = .'0; // '; // expected-error {{expected expression}}, lexed as . followed by character literal
   float i = 0x.'0p0; // expected-error {{digit separator cannot appear at start of digit sequence}}
   float j = 0x'0.0p0; // expected-error {{invalid suffix 'x'0.0p0'}}
-  float k = 0x0'.0p0; // '; // expected-error {{expected ';'}}
+  float k = 0x0'.0p0; // expected-error {{digit separator cannot appear at end of digit sequence}}
   float l = 0x0.'0p0; // expected-error {{digit separator cannot appear at start of digit sequence}}
   float m = 0x0.0'p0; // expected-error {{digit separator cannot appear at end of digit sequence}}
   float n = 0x0.0p'0; // expected-error {{digit separator cannot appear at start of digit sequence}}
   float o = 0x0.0p0'ms; // expected-error {{digit separator cannot appear at end of digit sequence}}
   float p = 0'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
   float q = 0'0e1;
   float r = 0.'0e1; // expected-error {{digit separator cannot appear at start of digit sequence}}
   float s = 0.0'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
   float t = 0.0e'1; // expected-error {{digit separator cannot appear at start of digit sequence}}
+  float u = 0'.0; // expected-error {{digit separator cannot appear at end of digit sequence}}
+  float v = 1'.0; // expected-error {{digit separator cannot appear at end of digit sequence}}
 }
 
 #line 123'456
 static_assert(__LINE__ == 123456, "");
 
 // x has value 0 in C++11 and 34 in C++1y.
 #define M(x, ...) __VA_ARGS__
 constexpr int x = { M(1'2,3'4) };