clang-format: [JS] fix async parsing.

[mprobst]

Summary

Only treat the sequence async function as the start of a function expression,
as opposed to every occurrence of the token async (whoops).

Also refactors AnnotatedLine.startsWith/endsWith by extracting the core functionality
into FormatToken.startsSequence/endsSequence. This allows checking tokens within
the pointered linked list structure with a lookahead, automatically ignoring
comments, which is useful in many places (e.g. see subsequent commit).

[mprobst]

• fix description

[djasper]

Looks good.

lib/Format/FormatToken.h

template <typename A, typename B> bool isOneOf(A K1, B K2) const {
     return is(K1) || is(K2);
   }
   template <typename A, typename B, typename... Ts>
   bool isOneOf(A K1, B K2, Ts... Ks) const {
     return is(K1) || isOneOf(K2, Ks...);
   }
   template <typename T> bool isNot(T Kind) const { return !is(Kind); }
 
+  /// \c true if this token starts a sequence with the given tokens in order,
+  /// following the ``Next`` pointers, ignoring comments.
+  template <typename A, typename... Ts>
+  bool startsSequence(A K1, Ts... Tokens) const {
+    return startsSequenceInternal(K1, Tokens...);
+  }
+
+  /// \c true if this token ends a sequence with the given tokens in order,
+  /// following the ``Previous`` pointers, ignoring comments.
+  template <typename A, typename... Ts>
+  bool endsSequence(A K1, Ts... Tokens) const {
+    return endsSequenceInternal(K1, Tokens...);
+  }
+
   bool isStringLiteral() const { return tok::isStringLiteral(Tok.getKind()); }
 
   bool isObjCAtKeyword(tok::ObjCKeywordKind Kind) const {
     return Tok.isObjCAtKeyword(Kind);
   }
 
   bool isAccessSpecifier(bool ColonRequired = true) const {
     return isOneOf(tok::kw_public, tok::kw_protected, tok::kw_private) &&
struct FormatToken {
   bool closesBlockOrBlockTypeList(const FormatStyle &Style) const {
     return MatchingParen && MatchingParen->opensBlockOrBlockTypeList(Style);
   }
 
 private:
   // Disallow copying.
   FormatToken(const FormatToken &) = delete;
   void operator=(const FormatToken &) = delete;
+
+  template <typename A, typename... Ts>
+  bool startsSequenceInternal(A K1, Ts... Tokens) const {
+    if (is(tok::comment) && Next)
+      return Next->startsSequenceInternal(K1, Tokens...);
+    return is(K1) && Next && Next->startsSequenceInternal(Tokens...);
+  }
+
+  template <typename A>
+  bool startsSequenceInternal(A K1) const {
+    if (is(tok::comment) && Next)
+      return Next->startsSequenceInternal(K1);
+    return is(K1);
+  }
+
+  template <typename A, typename... Ts>
+  bool endsSequenceInternal(A K1) const {
+    if (is(tok::comment) && Previous)
+      return Previous->endsSequenceInternal(K1);
+    return is(K1);
+  }
+
+  template <typename A, typename... Ts>
+  bool endsSequenceInternal(A K1, Ts... Tokens) const {
+    if (is(tok::comment) && Previous)
+      return Previous->endsSequenceInternal(K1, Tokens...);
+    return is(K1) && Previous && Previous->endsSequenceInternal(Tokens...);
+  }
 };
 
 class ContinuationIndenter;
 struct LineState;
 
 class TokenRole {
 public:
   TokenRole(const FormatStyle &Style) : Style(Style) {}

lib/Format/TokenAnnotator.h

while (Current) {
       Current->Role.reset();
       Current = Current->Next;
     }
   }
 
   /// \c true if this line starts with the given tokens in order, ignoring
   /// comments.
   template <typename... Ts> bool startsWith(Ts... Tokens) const {
-    return startsWithInternal(First, Tokens...);
+    return First && First->startsSequence(Tokens...);
   }
 
   /// \c true if this line ends with the given tokens in reversed order,
   /// ignoring comments.
   /// For example, given tokens [T1, T2, T3, ...], the function returns true if
   /// this line is like "... T3 T2 T1".
   template <typename... Ts> bool endsWith(Ts... Tokens) const {
-    return endsWithInternal(Last, Tokens...);
+    return Last && Last->endsSequence(Tokens...);
   }
 
   /// \c true if this line looks like a function definition instead of a
   /// function declaration. Asserts MightBeFunctionDecl.
   bool mightBeFunctionDefinition() const {
     assert(MightBeFunctionDecl);
     // FIXME: Line.Last points to other characters than tok::semi
     // and tok::lbrace.
public:
 
   /// \c True if a one of this line's children intersects with an input range.
   bool ChildrenAffected;
 
 private:
   // Disallow copying.
   AnnotatedLine(const AnnotatedLine &) = delete;
   void operator=(const AnnotatedLine &) = delete;
-
-  template <typename A, typename... Ts>
-  bool startsWithInternal(const FormatToken *Tok, A K1) const {
-    // Even though we skip comments in the outer `startWithInternal` function,
-    // this loop is still necessary if it is invoked by the public interface
-    // `startsWith`.
-    while (Tok && Tok->is(tok::comment))
-      Tok = Tok->Next;
-    return Tok && Tok->is(K1);
-  }
-
-  template <typename A, typename... Ts>
-  bool startsWithInternal(const FormatToken *Tok, A K1, Ts... Tokens) const {
-    // Skip comments before calling `startsWithInternal(Tok, K1)` so that  the
-    // second call to `startsWithInternal` takes the correct `Tok->Next`, which
-    // should be the next token of the token checked in the first call.
-    while (Tok && Tok->is(tok::comment))
-      Tok = Tok->Next;
-    return Tok && startsWithInternal(Tok, K1) &&
-           startsWithInternal(Tok->Next, Tokens...);
-  }
-
-  template <typename A, typename... Ts>
-  bool endsWithInternal(const FormatToken *Tok, A K1) const {
-    // See the comments above in `startsWithInternal(Tok, K1)`.
-    while (Tok && Tok->is(tok::comment))
-      Tok = Tok->Previous;
-    return Tok && Tok->is(K1);
-  }
-
-  template <typename A, typename... Ts>
-  bool endsWithInternal(const FormatToken *Tok, A K1, Ts... Tokens) const {
-    // See the comments above in `startsWithInternal(Tok, K1, Tokens)`.
-    while (Tok && Tok->is(tok::comment))
-      Tok = Tok->Previous;
-    return Tok && endsWithInternal(Tok, K1) &&
-           endsWithInternal(Tok->Previous, Tokens...);
-  }
 };
 
 /// \brief Determines extra information about the tokens comprising an
 /// \c UnwrappedLine.
 class TokenAnnotator {
 public:
   TokenAnnotator(const FormatStyle &Style, const AdditionalKeywords &Keywords)
       : Style(Style), Keywords(Keywords) {}

lib/Format/UnwrappedLineParser.cpp

case tok::identifier: {
       if (FormatTok->is(TT_MacroBlockEnd)) {
         addUnwrappedLine();
         return;
       }
 
       // Parse function literal unless 'function' is the first token in a line
       // in which case this should be treated as a free-standing function.
       if (Style.Language == FormatStyle::LK_JavaScript &&
-          FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function) &&
+          (FormatTok->is(Keywords.kw_function) ||
+           FormatTok->startsSequence(Keywords.kw_async,
+                                     Keywords.kw_function)) &&
           Line->Tokens.size() > 0) {
         tryToParseJSFunction();
         break;
       }
       if ((Style.Language == FormatStyle::LK_JavaScript ||
            Style.Language == FormatStyle::LK_Java) &&
           FormatTok->is(Keywords.kw_interface)) {
         if (Style.Language == FormatStyle::LK_JavaScript) {
do {
     } else {
       return false;
     }
   } while (!eof());
   return false;
 }
 
 void UnwrappedLineParser::tryToParseJSFunction() {
-  assert(FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function));
+  assert(FormatTok->is(Keywords.kw_function) ||
+         FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function));
   if (FormatTok->is(Keywords.kw_async))
     nextToken();
   // Consume "function".
   nextToken();
 
   // Consume * (generator function).
   if (FormatTok->is(tok::star))
     nextToken();

 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
   bool HasError = false;
   nextToken();
 
   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
   // replace this by using parseAssigmentExpression() inside.
   do {
     if (Style.Language == FormatStyle::LK_JavaScript) {
-      if (FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function)) {
+      if (FormatTok->is(Keywords.kw_function) ||
+          FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
         tryToParseJSFunction();
         continue;
       }
       if (FormatTok->is(TT_JsFatArrow)) {
         nextToken();
         // Fat arrows can be followed by simple expressions or by child blocks
         // in curly braces.
         if (FormatTok->is(tok::l_brace)) {
case tok::l_brace:
       break;
     case tok::at:
       nextToken();
       if (FormatTok->Tok.is(tok::l_brace))
         parseBracedList();
       break;
     case tok::identifier:
       if (Style.Language == FormatStyle::LK_JavaScript &&
-          FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function))
+          (FormatTok->is(Keywords.kw_function) ||
+           FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)))
         tryToParseJSFunction();
       else
         nextToken();
       break;
     default:
       nextToken();
       break;
     }

unittests/Format/FormatTestJS.cpp

verifyFormat("async function* f() {\n"
                "  yield fetch(x);\n"
                "}");
   verifyFormat("export async function f() {\n"
                "  return fetch(x);\n"
                "}");
   verifyFormat("class X {\n"
                "  async asyncMethod() { return fetch(1); }\n"
                "}");
+  verifyFormat("function initialize() {\n"
+               "  // Comment.\n"
+               "  return async.then();\n"
+               "}\n");
 }
 
 TEST_F(FormatTestJS, ArrayLiterals) {
   verifyFormat("var aaaaa: List<SomeThing> =\n"
                "    [new SomeThingAAAAAAAAAAAA(), new SomeThingBBBBBBBBB()];");
   verifyFormat("return [\n"
                "  aaaaaaaaaaaaaaaaaaaaaaaaaaa, bbbbbbbbbbbbbbbbbbbbbbbbbbb,\n"
                "  ccccccccccccccccccccccccccc\n"