Do not emit diagnostics for invalid unicode characters in preprocessing mode

[aaron.ballman]

LGTM!

clang/test/Lexer/unicode.c

3

This RUN line isn't quite correct because it's not running FileCheck over the output. However, I will correct this when landing it.

[aaron.ballman]

I've landed this in 2715c4da50387ed6dae49a2a17fdcfad8a769a3c, thanks for the quick fix!

clang/lib/Lex/Lexer.cpp

bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
   uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/nullptr);
   if (CodePoint == 0) {
     return false;
   }
 
   if (!isAllowedIDChar(CodePoint, LangOpts)) {
     if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
       return false;
-    if (!isLexingRawMode())
+    if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
+        !PP->isPreprocessedOutput())
       diagnoseInvalidUnicodeCodepointInIdentifier(
           PP->getDiagnostics(), LangOpts, CodePoint,
           makeCharRange(*this, CurPtr, UCNPtr),
           /*IsFirst=*/false);
 
     // We got a unicode codepoint that is neither a space nor a
     // a valid identifier part.
     // Carry on as if the codepoint was valid for recovery purposes.
llvm::ConversionResult Result =
                                 llvm::strictConversion);
   if (Result != llvm::conversionOK)
     return false;
 
   if (!isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) {
     if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
       return false;
 
-    if (!isLexingRawMode())
+    if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
+        !PP->isPreprocessedOutput())
       diagnoseInvalidUnicodeCodepointInIdentifier(
           PP->getDiagnostics(), LangOpts, CodePoint,
           makeCharRange(*this, CurPtr, UnicodePtr), /*IsFirst=*/false);
     // We got a unicode codepoint that is neither a space nor a
     // a valid identifier part. Carry on as if the codepoint was
     // valid for recovery purposes.
   } else if (!isLexingRawMode()) {
     maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,

clang/test/Lexer/unicode.c

 // RUN: %clang_cc1 -fsyntax-only -verify -x c -std=c11 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -x c++ -std=c++11 %s
+// RUN: %clang_cc1 -triple x86_64-scei-ps4 -E -DPP_ONLY=1 %s -o %t
 // RUN: %clang_cc1 -E -DPP_ONLY=1 %s -o %t
 // RUN: FileCheck --strict-whitespace --input-file=%t %s
 
 // This file contains Unicode characters; please do not "fix" them!
 
 extern int!x; // expected-warning {{treating Unicode character as whitespace}}
 extern int　x; // expected-warning {{treating Unicode character as whitespace}}