[Clang] Handle non-ASCII after line splicing

[cor3ntin]

Summary

int a\
ス;

Failed to be parsed as a valid identifier.

Fixes #65156

[shafik]

clang/lib/Lex/Lexer.cpp

1757

I think that is what you meant to say

1761

Do we need to verify that FirstCodeUnitSize is not zero?

1796

So if we get here it mean we have the splice to skip?

[cor3ntin]

Add and fix comments

clang/lib/Lex/Lexer.cpp

1761

Nope, we always get a char of getCharAndSize so it's either 1 in the normal case, or N if there are escaped newlines or trigraphs

1796

Yes. And we need to emit a diag. It's a bit subtle so i added a comment.

[shafik]

I think this looks good but I would like @tahonermann to review this as well.

[tahonermann]

This looks good to me modulo a couple of nits.

clang/include/clang/Lex/Lexer.h

806–808

Tok should be documented here.

clang/lib/Lex/Lexer.cpp

1798

[cor3ntin]

Address Tom's feedback

[tahonermann]

Even better than I asked for. I held back on suggesting the change of Tok to Result to match tryConsumeIdentifierUCN(), but you made that change anyway! You must have read my mind! :)

[cor3ntin]

In D159345#4640147, @tahonermann wrote:

Even better than I asked for. I held back on suggesting the change of Tok to Result to match tryConsumeIdentifierUCN(), but you made that change anyway! You must have read my mind! :)

Haha. Once i added docs the lack of symmetry with the previous function became jarring!
Thanks!

clang/docs/ReleaseNotes.rst

 - For function multi-versioning using the ``target``, ``target_clones``, or
   ``target_version`` attributes, remove comdat for internal linkage functions.
   (`#65114 <https://github.com/llvm/llvm-project/issues/65114>`_)
 - Clang now reports ``-Wformat`` for bool value and char specifier confusion
   in scanf. Fixes
   (`#64987 <https://github.com/llvm/llvm-project/issues/64987>`_)
 - Support MSVC predefined macro expressions in constant expressions and in
   local structs.
+- Correctly parse non-ascii identifiers that appear immediately after a line splicing
+  (`#65156 <https://github.com/llvm/llvm-project/issues/65156>`_`)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Bug Fixes to Attribute Support
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Bug Fixes to C++ Support

clang/include/clang/Lex/Lexer.h

private:
   /// \param Result The token being produced. Marked as containing a UCN on
   ///               success.
   /// \return \c true if a UCN was lexed and it produced an acceptable
   ///         identifier character, \c false otherwise.
   bool tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
                                Token &Result);
 
   /// Try to consume an identifier character encoded in UTF-8.
   /// \param CurPtr Points to the start of the (potential) UTF-8 code unit
   ///        sequence. On success, updated to point past the end of it.
+  /// \param Result The token being formed.
   /// \return \c true if a UTF-8 sequence mapping to an acceptable identifier
   ///         character was lexed, \c false otherwise.
-  bool tryConsumeIdentifierUTF8Char(const char *&CurPtr);
+  bool tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result);
 };
 
 } // namespace clang
 
 #endif // LLVM_CLANG_LEX_LEXER_H

clang/lib/Lex/Lexer.cpp

if ((UCNPtr - CurPtr ==  6 && CurPtr[1] == 'u') ||
       (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U'))
     CurPtr = UCNPtr;
   else
     while (CurPtr != UCNPtr)
       (void)getAndAdvanceChar(CurPtr, Result);
   return true;
 }
 
-bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) {
-  const char *UnicodePtr = CurPtr;
+bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result) {
   llvm::UTF32 CodePoint;
-  llvm::ConversionResult Result =
-      llvm::convertUTF8Sequence((const llvm::UTF8 **)&UnicodePtr,
-                                (const llvm::UTF8 *)BufferEnd,
-                                &CodePoint,
-                                llvm::strictConversion);
-  if (Result != llvm::conversionOK)
+
+  // If a UTF-8 codepoint appears immediately after an escaped new line,
+  // CurPtr may point to the splicing \ on the preceding line,
+  // so we need to skip it.
+  unsigned FirstCodeUnitSize;
+  getCharAndSize(CurPtr, FirstCodeUnitSize);
+  const char *CharStart = CurPtr + FirstCodeUnitSize - 1;
+  const char *UnicodePtr = CharStart;
+
+  llvm::ConversionResult ConvResult = llvm::convertUTF8Sequence(
+      (const llvm::UTF8 **)&UnicodePtr, (const llvm::UTF8 *)BufferEnd,
+      &CodePoint, llvm::strictConversion);
+  if (ConvResult != llvm::conversionOK)
     return false;
 
   bool IsExtension = false;
   if (!isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts,
                        IsExtension)) {
     if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
       return false;
 
     if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
         !PP->isPreprocessedOutput())
       diagnoseInvalidUnicodeCodepointInIdentifier(
           PP->getDiagnostics(), LangOpts, CodePoint,
-          makeCharRange(*this, CurPtr, UnicodePtr), /*IsFirst=*/false);
+          makeCharRange(*this, CharStart, UnicodePtr), /*IsFirst=*/false);
     // We got a unicode codepoint that is neither a space nor a
     // a valid identifier part. Carry on as if the codepoint was
     // valid for recovery purposes.
   } else if (!isLexingRawMode()) {
     if (IsExtension)
-      diagnoseExtensionInIdentifier(PP->getDiagnostics(), CodePoint,
-                                    makeCharRange(*this, CurPtr, UnicodePtr));
+      diagnoseExtensionInIdentifier(
+          PP->getDiagnostics(), CodePoint,
+          makeCharRange(*this, CharStart, UnicodePtr));
     maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,
-                              makeCharRange(*this, CurPtr, UnicodePtr),
+                              makeCharRange(*this, CharStart, UnicodePtr),
                               /*IsFirst=*/false);
     maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), CodePoint,
-                               makeCharRange(*this, CurPtr, UnicodePtr));
+                               makeCharRange(*this, CharStart, UnicodePtr));
   }
 
+  // Once we sucessfully parsed some UTF-8,
+  // calling ConsumeChar ensures the NeedsCleaning flag is set on the token
+  // being lexed, and that warnings about trailing spaces are emitted.
+  ConsumeChar(CurPtr, FirstCodeUnitSize, Result);
   CurPtr = UnicodePtr;
   return true;
 }
 
 bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
                                       const char *CurPtr) {
   bool IsExtension = false;
   if (isAllowedInitiallyIDChar(C, LangOpts, IsExtension)) {
if (C == '$') {
       // Otherwise, emit a diagnostic and continue.
       if (!isLexingRawMode())
         Diag(CurPtr, diag::ext_dollar_in_identifier);
       CurPtr = ConsumeChar(CurPtr, Size, Result);
       continue;
     }
     if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
       continue;
-    if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr))
+    if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
       continue;
     // Neither an expected Unicode codepoint nor a UCN.
     break;
   }
 
   const char *IdStart = BufferPtr;
   FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
   Result.setRawIdentifierData(IdStart);
if (isAsciiIdentifierContinue(Next)) {
       CurPtr = ConsumeChar(CurPtr, NextSize, Result);
       return LexNumericConstant(Result, CurPtr);
     }
   }
 
   // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
   if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
     return LexNumericConstant(Result, CurPtr);
-  if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr))
+  if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
     return LexNumericConstant(Result, CurPtr);
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = BufferPtr;
   FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
   Result.setLiteralData(TokStart);
   return true;
 }
 
 /// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
 /// in C++11, or warn on a ud-suffix in C++98.
 const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
                                bool IsStringLiteral) {
   assert(LangOpts.CPlusPlus);
 
   // Maximally munch an identifier.
   unsigned Size;
   char C = getCharAndSize(CurPtr, Size);
   bool Consumed = false;
 
   if (!isAsciiIdentifierStart(C)) {
     if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
       Consumed = true;
-    else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr))
+    else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr, Result))
       Consumed = true;
     else
       return CurPtr;
   }
 
   if (!LangOpts.CPlusPlus11) {
     if (!isLexingRawMode())
       Diag(CurPtr,
const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
   }
 
   Result.setFlag(Token::HasUDSuffix);
   while (true) {
     C = getCharAndSize(CurPtr, Size);
     if (isAsciiIdentifierContinue(C)) {
       CurPtr = ConsumeChar(CurPtr, Size, Result);
     } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {
-    } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {
+    } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr, Result)) {
     } else
       break;
   }
 
   return CurPtr;
 }
 
 /// LexStringLiteral - Lex the remainder of a string literal, after having lexed

clang/test/Lexer/escape_newline_unicode.c

+// RUN: %clang_cc1 -verify=expected,c -x c -Wunused %s
+// RUN: %clang_cc1 -verify=expected,cpp -x c++ -Wunused %s
+
+void gh65156(void) {
+
+int a\
+ス = 42;
+// expected-warning@-2 {{unused variable 'aス'}}
+
+int b\ 
+\ 
+ス = 42;
+// expected-warning@-2 {{backslash and newline separated by space}}
+// expected-warning@-4 {{backslash and newline separated by space}}
+// expected-warning@-5 {{unused variable 'bス'}}
+
+int ス\
+ス = 42;
+// expected-warning@-2 {{unused variable 'スス'}}
+
+int \
+ス = 42;
+// expected-warning@-2 {{unused variable 'ス'}}
+
+}
+
+void gh65156_err(void) {
+
+int \
+❌ = 0;
+// cpp-error@-2 {{expected unqualified-id}}
+// c-error@-3 {{expected identifier}}
+
+
+int a\
+❌ = 0;
+// expected-error@-1 {{character <U+274C> not allowed in an identifier}}
+}