[clang-format] Change heuristic for locating lambda template arguments

[rymiel]

Summary

Previously, the heuristic was simply to look for template argument-
specific keywords, such as typename, class, template and auto
that are preceded by a left angle bracket <.

This changes the heuristic to instead look for a left angle bracket <
preceded by a right square bracket ], since according to the C++
grammar, the template arguments must *directly* follow the introducer.
(This sort of check might just end up being *too* aggressive)

This patch also adds a bunch more token annotator tests for lambdas,
specifically for some of the stranger forms of lambdas now allowed as
of C++20 or soon-to-be-allowed as part of C++23.

Fixes https://github.com/llvm/llvm-project/issues/57093

This does NOT resolve the FIXME regarding explicit template lists, but
perhaps it gets closer

[MyDeveloperDay]

can you add some format tests to show examples of how you think it should change

[rymiel]

Add a few format tests in addition to the annotator tests

From some incidental testing, I am pretty sure the formatting of lambdas that were mis-annotated as "array subscript" tended to end up being formatted correct-ish, however, I added a test case for the bug which this patch solves, which I should have done earlier.

[MyDeveloperDay]

If none of the previous tests fail then this looks like it might be a good fix.

[MyDeveloperDay]

LGTM but maybe wait for the others

[owenpan]

clang/unittests/Format/FormatTest.cpp

21738

There should be no terminating newline here.

21744

Ditto.

[rymiel]

Remove trailing newlines in added format tests

[curdeius]

@rymiel, please provide your name and email address for the commit message, so that we can land it for you.

[HazardyKnusperkeks]

Can you please rebase the patch, it doesn't apply anymore.
Sorry for that.

[rymiel]

Rebase on main

clang/lib/Format/UnwrappedLineParser.cpp

while (FormatTok->isNot(tok::l_brace)) {
     case tok::l_brace:
       break;
     case tok::l_paren:
       parseParens();
       break;
     case tok::l_square:
       parseSquare();
       break;
-    case tok::kw_auto:
-    case tok::kw_class:
-    case tok::kw_template:
-    case tok::kw_typename:
+    case tok::less:
       assert(FormatTok->Previous);
-      if (FormatTok->Previous->is(tok::less))
+      if (FormatTok->Previous->is(tok::r_square))
         InTemplateParameterList = true;
       nextToken();
       break;
+    case tok::kw_auto:
+    case tok::kw_class:
+    case tok::kw_template:
+    case tok::kw_typename:
     case tok::amp:
     case tok::star:
     case tok::kw_const:
     case tok::kw_constexpr:
     case tok::comma:
-    case tok::less:
     case tok::greater:
     case tok::identifier:
     case tok::numeric_constant:
     case tok::coloncolon:
     case tok::kw_mutable:
     case tok::kw_noexcept:
       nextToken();
       break;

clang/unittests/Format/FormatTest.cpp

verifyFormat("auto L = []<template <typename...> class T>(T...) {\n"
                "  }\n"
                "};\n");
   verifyFormat("auto L = []</*comment*/ class... T>(T...) {\n"
                "  {\n"
                "    f();\n"
                "    g();\n"
                "  }\n"
                "};\n");
+  verifyFormat("auto L = []<int... T>(T...) {\n"
+               "  {\n"
+               "    f();\n"
+               "    g();\n"
+               "  }\n"
+               "};");
+  verifyFormat("auto L = []<Foo... T>(T...) {\n"
+               "  {\n"
+               "    f();\n"
+               "    g();\n"
+               "  }\n"
+               "};");
 
   // Multiple lambdas in the same parentheses change indentation rules. These
   // lambdas are forced to start on new lines.
   verifyFormat("SomeFunction(\n"
                "    []() {\n"
                "      //\n"
                "    },\n"
                "    []() {\n"

clang/unittests/Format/TokenAnnotatorTest.cpp

 TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
   auto Tokens = annotate("[]() constexpr {}");
   ASSERT_EQ(Tokens.size(), 8u) << Tokens;
   EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
   EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
 
   Tokens = annotate("[]() -> auto {}");
   ASSERT_EQ(Tokens.size(), 9u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
 
   Tokens = annotate("[]() -> auto & {}");
   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
 
   Tokens = annotate("[]() -> auto * {}");
   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
   EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
   EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] {}");
+  ASSERT_EQ(Tokens.size(), 5u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] noexcept {}");
+  ASSERT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] -> auto {}");
+  ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
+  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <typename T> () {}");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <typename T> {}");
+  ASSERT_EQ(Tokens.size(), 9u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <typename... T> () {}");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <typename... T> {}");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <int... T> () {}");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <int... T> {}");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <Foo... T> () {}");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <Foo... T> {}");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
   auto Tokens = annotate("template <typename T>\n"
                          "DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"
                          "string OldFunction(const string &parameter) {}");
   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
   EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);