[clang-format] Fix preprocessor nesting after commit 529aa4b011c4ae808d658022ef643c44dd9b2c9c

[krasimir]

Summary

In https://github.com/llvm/llvm-project/commit/529aa4b011c4ae808d658022ef643c44dd9b2c9c
by setting the identifier info to nullptr, we started to subtly
interfere with the parts in the beginning of the function,
https://github.com/llvm/llvm-project/blob/529aa4b011c4ae808d658022ef643c44dd9b2c9c/clang/lib/Format/UnwrappedLineParser.cpp#L991
causing the preprocessor nesting to change in some cases. E.g., for the
added regression test, clang-format started incorrectly guessing the
language as C++.

This tries to address this by introducing an internal identifier info
element to use instead.

[MyDeveloperDay]

clang/unittests/Format/FormatTest.cpp

22727

Only only

[MyDeveloperDay]

clang/lib/Format/FormatToken.h

1077

I'd be ok with just

kw_internal_ident_after_define

[MyDeveloperDay]

Just a few nits, but LGTM

[curdeius]

LGTM with other comments addressed.

[krasimir]

• address review comments

clang/lib/Format/FormatToken.h

AdditionalKeywords(IdentifierTable &IdentTable) {
     kw_required = &IdentTable.get("required");
     kw_returns = &IdentTable.get("returns");
 
     kw_signals = &IdentTable.get("signals");
     kw_qsignals = &IdentTable.get("Q_SIGNALS");
     kw_slots = &IdentTable.get("slots");
     kw_qslots = &IdentTable.get("Q_SLOTS");
 
+    // For internal clang-format use.
+    kw_internal_ident_after_define =
+        &IdentTable.get("__CLANG_FORMAT_INTERNAL_IDENT_AFTER_DEFINE__");
+
     // C# keywords
     kw_dollar = &IdentTable.get("dollar");
     kw_base = &IdentTable.get("base");
     kw_byte = &IdentTable.get("byte");
     kw_checked = &IdentTable.get("checked");
     kw_decimal = &IdentTable.get("decimal");
     kw_delegate = &IdentTable.get("delegate");
     kw_event = &IdentTable.get("event");
struct AdditionalKeywords {
   IdentifierInfo *kw_returns;
 
   // QT keywords.
   IdentifierInfo *kw_signals;
   IdentifierInfo *kw_qsignals;
   IdentifierInfo *kw_slots;
   IdentifierInfo *kw_qslots;
 
+  // For internal use by clang-format.
+  IdentifierInfo *kw_internal_ident_after_define;
+
   // C# keywords
   IdentifierInfo *kw_dollar;
   IdentifierInfo *kw_base;
   IdentifierInfo *kw_byte;
   IdentifierInfo *kw_checked;
   IdentifierInfo *kw_decimal;
   IdentifierInfo *kw_delegate;
   IdentifierInfo *kw_event;

clang/lib/Format/UnwrappedLineParser.cpp

for (auto &Line : Lines) {
         break;
       }
     }
   }
 
   // In the context of a define, even keywords should be treated as normal
   // identifiers. Setting the kind to identifier is not enough, because we need
   // to treat additional keywords like __except as well, which are already
-  // identifiers.
+  // identifiers. Setting the identifier info to null interferes with include
+  // guard processing above, and changes preprocessing nesting.
   FormatTok->Tok.setKind(tok::identifier);
-  FormatTok->Tok.setIdentifierInfo(nullptr);
+  FormatTok->Tok.setIdentifierInfo(Keywords.kw_internal_ident_after_define);
   nextToken();
   if (FormatTok->Tok.getKind() == tok::l_paren &&
       !FormatTok->hasWhitespaceBefore())
     parseParens();
   if (Style.IndentPPDirectives != FormatStyle::PPDIS_None)
     Line->Level += PPBranchLevel + 1;
   addUnwrappedLine();
   ++Line->Level;

clang/unittests/Format/FormatTest.cpp

EXPECT_EQ(FormatStyle::LK_ObjC,
             guessLanguage("foo.h", "int DoStuff(CGRect rect);\n"));
   EXPECT_EQ(
       FormatStyle::LK_ObjC,
       guessLanguage("foo.h",
                     "#define MY_POINT_MAKE(x, y) CGPointMake((x), (y));\n"));
   EXPECT_EQ(
       FormatStyle::LK_Cpp,
       guessLanguage("foo.h", "#define FOO(...) auto bar = [] __VA_ARGS__;"));
+  // Only one of the two preprocessor regions has ObjC-like code.
+  EXPECT_EQ(FormatStyle::LK_ObjC,
+            guessLanguage("foo.h", "#if A\n"
+                                   "#define B() C\n"
+                                   "#else\n"
+                                   "#define B() [NSString a:@\"\"]\n"
+                                   "#endif\n"));
 }
 
 TEST_F(FormatTest, GuessLanguageWithCpp11AttributeSpecifiers) {
   EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo.h", "[[noreturn]];"));
   EXPECT_EQ(FormatStyle::LK_ObjC,
             guessLanguage("foo.h", "array[[calculator getIndex]];"));
   EXPECT_EQ(FormatStyle::LK_Cpp,
             guessLanguage("foo.h", "[[noreturn, deprecated(\"so sorry\")]];"));