[clang-format] Fix regressions in WhitespaceSensitiveMacros

[owenpan]

Summary

Fixes https://github.com/llvm/llvm-project/issues/54522.
Fixes https://github.com/llvm/llvm-project/issues/57158.

[kuhar]

This seems to have caused a test failure:
https://lab.llvm.org/buildbot/#/builders/109/builds/45138

Failed Tests (1):
  Clang-Unit :: Format/./FormatTests/TokenAnnotatorTest/UnderstandsLBracesInMacroDefinition
Testing Time: 92.30s
  Skipped          :    38
  Unsupported      :   767
  Passed           : 82570
  Expectedly Failed:   194
  Failed           :     1

TokenAnnotatorTest.cpp:292: Failure

[owenpan]

In D132001#3741467, @kuhar wrote:

This seems to have caused a test failure:
https://lab.llvm.org/buildbot/#/builders/109/builds/45138

Failed Tests (1):
  Clang-Unit :: Format/./FormatTests/TokenAnnotatorTest/UnderstandsLBracesInMacroDefinition
Testing Time: 92.30s
  Skipped          :    38
  Unsupported      :   767
  Passed           : 82570
  Expectedly Failed:   194
  Failed           :     1

TokenAnnotatorTest.cpp:292: Failure

Fixed in 47e7056.

clang/lib/Format/UnwrappedLineParser.cpp

case tok::identifier: {
 
         bool FollowedByNewline =
             CommentsBeforeNextToken.empty()
                 ? FormatTok->NewlinesBefore > 0
                 : CommentsBeforeNextToken.front()->NewlinesBefore > 0;
 
         if (FollowedByNewline && (Text.size() >= 5 || FunctionLike) &&
             tokenCanStartNewLine(*FormatTok) && Text == Text.upper()) {
+          if (PreviousToken->isNot(TT_UntouchableMacroFunc))
             PreviousToken->setFinalizedType(TT_FunctionLikeOrFreestandingMacro);
           addUnwrappedLine();
           return;
         }
       }
       break;
     }
     case tok::equal:
       if ((Style.isJavaScript() || Style.isCSharp()) &&

clang/unittests/Format/FormatTest.cpp

TEST_F(FormatTest, OperatorPassedAsAFunctionPtr) {
   verifyFormat("foo(-42, operator, );", Style);
   verifyFormat("foo(operator, , -42);", Style);
 }
 
 TEST_F(FormatTest, WhitespaceSensitiveMacros) {
   FormatStyle Style = getLLVMStyle();
   Style.WhitespaceSensitiveMacros.push_back("FOO");
 
+  // Newlines are important here.
+  verifyFormat("FOO(1+2 )\n", Style);
+  verifyFormat("FOO(a:b:c)\n", Style);
+
   // Don't use the helpers here, since 'mess up' will change the whitespace
   // and these are all whitespace sensitive by definition
   EXPECT_EQ("FOO(String-ized&Messy+But(: :Still)=Intentional);",
             format("FOO(String-ized&Messy+But(: :Still)=Intentional);", Style));
   EXPECT_EQ(
       "FOO(String-ized&Messy+But\\(: :Still)=Intentional);",
       format("FOO(String-ized&Messy+But\\(: :Still)=Intentional);", Style));
   EXPECT_EQ("FOO(String-ized&Messy+But,: :Still=Intentional);",

clang/unittests/Format/TokenAnnotatorTest.cpp

 TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
   auto Tokens =
       annotate("template <typename T> "
                "inline bool var = is_integral_v<int> && is_signed_v<int>;");
   EXPECT_EQ(Tokens.size(), 20u) << Tokens;
   EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
 }
 
+TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
+  FormatStyle Style = getLLVMStyle();
+  Style.WhitespaceSensitiveMacros.push_back("FOO");
+
+  auto Tokens = annotate("FOO(1+2 )\n", Style);
+  EXPECT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
+
+  Tokens = annotate("FOO(a:b:c)\n", Style);
+  EXPECT_EQ(Tokens.size(), 9u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
+}
+
+TEST_F(TokenAnnotatorTest, UnderstandsLBracesInMacroDefinition) {
+  auto Tokens = annotate("#define BEGIN NS {");
+  EXPECT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown);
+}
+
 TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
   auto Tokens = annotate("delete (void *)p;");
   EXPECT_EQ(Tokens.size(), 8u) << Tokens;
   EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
 
   Tokens = annotate("delete[] (void *)p;");
   EXPECT_EQ(Tokens.size(), 10u) << Tokens;
   EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);