clang-format: [JS] generator and async functions.

[mprobst]

• fix tests
• add spec links

[djasper]

unittests/Format/FormatTestJS.cpp

339 ↗

(On Diff #54017)

What does the star mean here? Should we actually introduce a token type or do you think it's just going to work? Specifically, what happens if you need to line-wrap?

[mprobst]

• test for wrapping

[mprobst]

unittests/Format/FormatTestJS.cpp

339 ↗

(On Diff #54017)

The star indicates that this is a function that returns a generator, i.e. implicitly returns an iterator in which you shove values with yield. I think you wouldn't want to wrap between function and *. We currently don't, but more by chance - it's a side effect of not breaking before a binary operator.

function* cannot be one token, there may be whitespace and comments in between. We could have a generator star token, but there are places where that'd be hard for us to detect, e.g. the class method above. Having a generator star only occasionally be recognized of course would be rather confusing.

So I'd punt on this for the time beimg. I've added a test to make sure we correctly wrap above. Does that make sense?

[djasper]

unittests/Format/FormatTestJS.cpp

339 ↗

(On Diff #54195)

Fundamentally, I think there is two things we might need to do:

1. Not allow wrapping it. I am ok with keeping that based on the operator wrapping for now, but it seems a bit dodgy.
2. The ExpressionParser in TokenAnnotator.cpp must not treat this as a binary operator or else it might do weird things with indentation. What't the token type at the moment?

[mprobst]

unittests/Format/FormatTestJS.cpp

339 ↗

(On Diff #54195)

For (1) if needed we can detect the immediately preceding function.
For (2), token type is tok::star. Indentation in the wrapping example seems fine.

Overall I find it highly unlikely that this becomes a problem, having generator functions that wrap is unlikely, they tend to be top level functions.

[mprobst]

Friendly ping.

[djasper]

Looks good.

cfe/trunk/lib/Format/ContinuationIndenter.cpp

unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
   //   var loooooong_name =
   //       function() {
   //     // code
   //   }
   //
   // is common and should be formatted like a free-standing function.
   if (Style.Language != FormatStyle::LK_JavaScript ||
       Current.NestingLevel != 0 || !PreviousNonComment->is(tok::equal) ||
-      !Current.is(Keywords.kw_function))
+      !Current.isOneOf(Keywords.kw_async, Keywords.kw_function))
     State.Stack.back().NestedBlockIndent = State.Column;
 
   if (NextNonComment->isMemberAccess()) {
     if (State.Stack.back().CallContinuation == 0)
       State.Stack.back().CallContinuation = State.Column;
   } else if (NextNonComment->is(TT_SelectorName)) {
     if (!State.Stack.back().ObjCSelectorNameFound) {
       if (NextNonComment->LongestObjCSelectorName == 0) {

cfe/trunk/lib/Format/FormatToken.h

AdditionalKeywords(IdentifierTable &IdentTable) {
     kw_override = &IdentTable.get("override");
     kw_in = &IdentTable.get("in");
     kw_of = &IdentTable.get("of");
     kw_CF_ENUM = &IdentTable.get("CF_ENUM");
     kw_CF_OPTIONS = &IdentTable.get("CF_OPTIONS");
     kw_NS_ENUM = &IdentTable.get("NS_ENUM");
     kw_NS_OPTIONS = &IdentTable.get("NS_OPTIONS");
 
+    kw_async = &IdentTable.get("async");
+    kw_await = &IdentTable.get("await");
     kw_finally = &IdentTable.get("finally");
-    kw_function = &IdentTable.get("function");
     kw_from = &IdentTable.get("from");
+    kw_function = &IdentTable.get("function");
     kw_import = &IdentTable.get("import");
     kw_is = &IdentTable.get("is");
     kw_let = &IdentTable.get("let");
     kw_var = &IdentTable.get("var");
+    kw_yield = &IdentTable.get("yield");
 
     kw_abstract = &IdentTable.get("abstract");
     kw_assert = &IdentTable.get("assert");
     kw_extends = &IdentTable.get("extends");
     kw_implements = &IdentTable.get("implements");
     kw_instanceof = &IdentTable.get("instanceof");
     kw_interface = &IdentTable.get("interface");
     kw_native = &IdentTable.get("native");
struct AdditionalKeywords {
   IdentifierInfo *kw_of;
   IdentifierInfo *kw_CF_ENUM;
   IdentifierInfo *kw_CF_OPTIONS;
   IdentifierInfo *kw_NS_ENUM;
   IdentifierInfo *kw_NS_OPTIONS;
   IdentifierInfo *kw___except;
 
   // JavaScript keywords.
+  IdentifierInfo *kw_async;
+  IdentifierInfo *kw_await;
   IdentifierInfo *kw_finally;
-  IdentifierInfo *kw_function;
   IdentifierInfo *kw_from;
+  IdentifierInfo *kw_function;
   IdentifierInfo *kw_import;
   IdentifierInfo *kw_is;
   IdentifierInfo *kw_let;
   IdentifierInfo *kw_var;
+  IdentifierInfo *kw_yield;
 
   // Java keywords.
   IdentifierInfo *kw_abstract;
   IdentifierInfo *kw_assert;
   IdentifierInfo *kw_extends;
   IdentifierInfo *kw_implements;
   IdentifierInfo *kw_instanceof;
   IdentifierInfo *kw_interface;

cfe/trunk/lib/Format/TokenAnnotator.cpp

if (Right.is(tok::period) &&
                      Keywords.kw_repeated, Keywords.kw_extend))
       return true;
     if (Right.is(tok::l_paren) &&
         Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
       return true;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     if (Left.is(TT_JsFatArrow))
       return true;
+    if (Right.is(tok::star) &&
+        Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
+      return false;
     if (Left.isOneOf(Keywords.kw_let, Keywords.kw_var, Keywords.kw_in,
                      Keywords.kw_of) &&
         (!Left.Previous || !Left.Previous->is(tok::period)))
       return true;
     if (Left.is(tok::kw_default) && Left.Previous &&
         Left.Previous->is(tok::kw_export))
       return true;
     if (Left.is(Keywords.kw_is) && Right.is(tok::l_brace))

cfe/trunk/lib/Format/UnwrappedLineParser.cpp

return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
          Tok.isNot(tok::kw_noexcept);
 }
 
 static bool mustBeJSIdent(const AdditionalKeywords &Keywords,
                           const FormatToken *FormatTok) {
   // FIXME: This returns true for C/C++ keywords like 'struct'.
   return FormatTok->is(tok::identifier) &&
          (FormatTok->Tok.getIdentifierInfo() == nullptr ||
-          !FormatTok->isOneOf(Keywords.kw_in, Keywords.kw_of,
+          !FormatTok->isOneOf(Keywords.kw_in, Keywords.kw_of, Keywords.kw_async,
+                              Keywords.kw_await, Keywords.kw_yield,
                               Keywords.kw_finally, Keywords.kw_function,
                               Keywords.kw_import, Keywords.kw_is,
                               Keywords.kw_let, Keywords.kw_var,
                               Keywords.kw_abstract, Keywords.kw_extends,
                               Keywords.kw_implements, Keywords.kw_instanceof,
                               Keywords.kw_interface, Keywords.kw_throws));
 }
 
 static bool mustBeJSIdentOrValue(const AdditionalKeywords &Keywords,
                                  const FormatToken *FormatTok) {
   return FormatTok->Tok.isLiteral() || mustBeJSIdent(Keywords, FormatTok);
 }
 
 // isJSDeclOrStmt returns true if |FormatTok| starts a declaration or statement
 // when encountered after a value (see mustBeJSIdentOrValue).
 static bool isJSDeclOrStmt(const AdditionalKeywords &Keywords,
                            const FormatToken *FormatTok) {
   return FormatTok->isOneOf(
-      tok::kw_return,
+      tok::kw_return, Keywords.kw_yield,
       // conditionals
       tok::kw_if, tok::kw_else,
       // loops
       tok::kw_for, tok::kw_while, tok::kw_do, tok::kw_continue, tok::kw_break,
       // switch/case
       tok::kw_switch, tok::kw_case,
       // exceptions
       tok::kw_throw, tok::kw_try, tok::kw_catch, Keywords.kw_finally,
       // declaration
       tok::kw_const, tok::kw_class, Keywords.kw_var, Keywords.kw_let,
-      Keywords.kw_function);
+      Keywords.kw_async, Keywords.kw_function,
+      // import/export
+      Keywords.kw_import, tok::kw_export);
 }
 
 // readTokenWithJavaScriptASI reads the next token and terminates the current
 // line if JavaScript Automatic Semicolon Insertion must
 // happen between the current token and the next token.
 //
 // This method is conservative - it cannot cover all edge cases of JavaScript,
 // but only aims to correctly handle certain well known cases. It *must not*
case tok::identifier: {
       if (FormatTok->is(TT_MacroBlockEnd)) {
         addUnwrappedLine();
         return;
       }
 
       // Parse function literal unless 'function' is the first token in a line
       // in which case this should be treated as a free-standing function.
       if (Style.Language == FormatStyle::LK_JavaScript &&
-          FormatTok->is(Keywords.kw_function) && Line->Tokens.size() > 0) {
+          FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function) &&
+          Line->Tokens.size() > 0) {
         tryToParseJSFunction();
         break;
       }
       if ((Style.Language == FormatStyle::LK_JavaScript ||
            Style.Language == FormatStyle::LK_Java) &&
           FormatTok->is(Keywords.kw_interface)) {
         if (Style.Language == FormatStyle::LK_JavaScript) {
           // In JavaScript/TypeScript, "interface" can be used as a standalone
do {
     } else {
       return false;
     }
   } while (!eof());
   return false;
 }
 
 void UnwrappedLineParser::tryToParseJSFunction() {
+  assert(FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function));
+  if (FormatTok->is(Keywords.kw_async))
+    nextToken();
+  // Consume "function".
+  nextToken();
+
+  // Consume * (generator function).
+  if (FormatTok->is(tok::star))
     nextToken();
 
   // Consume function name.
   if (FormatTok->is(tok::identifier))
     nextToken();
 
   if (FormatTok->isNot(tok::l_paren))
     return;
 

 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
   bool HasError = false;
   nextToken();
 
   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
   // replace this by using parseAssigmentExpression() inside.
   do {
     if (Style.Language == FormatStyle::LK_JavaScript) {
-      if (FormatTok->is(Keywords.kw_function)) {
+      if (FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function)) {
         tryToParseJSFunction();
         continue;
       }
       if (FormatTok->is(TT_JsFatArrow)) {
         nextToken();
         // Fat arrows can be followed by simple expressions or by child blocks
         // in curly braces.
         if (FormatTok->is(tok::l_brace)) {
case tok::l_brace:
       break;
     case tok::at:
       nextToken();
       if (FormatTok->Tok.is(tok::l_brace))
         parseBracedList();
       break;
     case tok::identifier:
       if (Style.Language == FormatStyle::LK_JavaScript &&
-          FormatTok->is(Keywords.kw_function))
+          FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function))
         tryToParseJSFunction();
       else
         nextToken();
       break;
     default:
       nextToken();
       break;
     }
void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
   bool IsImport = FormatTok->is(Keywords.kw_import);
   assert(IsImport || FormatTok->is(tok::kw_export));
   nextToken();
 
   // Consume the "default" in "export default class/function".
   if (FormatTok->is(tok::kw_default))
     nextToken();
 
-  // Consume "function" and "default function", so that these get parsed as
-  // free-standing JS functions, i.e. do not require a trailing semicolon.
+  // Consume "async function", "function" and "default function", so that these
+  // get parsed as free-standing JS functions, i.e. do not require a trailing
+  // semicolon.
+  if (FormatTok->is(Keywords.kw_async))
+    nextToken();
   if (FormatTok->is(Keywords.kw_function)) {
     nextToken();
     return;
   }
 
   // For imports, `export *`, `export {...}`, consume the rest of the line up
   // to the terminating `;`. For everything else, just return and continue
   // parsing the structural element, i.e. the declaration or expression for

cfe/trunk/unittests/Format/FormatTestJS.cpp

verifyFormat("function outer1(a, b) {\n"
                "}\n"
                "function outer2(a, b) {\n"
                "  function inner2(a, b) { return a; }\n"
                "  inner2(a, b);\n"
                "}");
   verifyFormat("function f() {}");
 }
 
+TEST_F(FormatTestJS, GeneratorFunctions) {
+  verifyFormat("function* f() {\n"
+               "  let x = 1;\n"
+               "  yield x;\n"
+               "  yield* something();\n"
+               "}");
+  verifyFormat("function*\n"
+               "    f() {\n"
+               "}",
+               getGoogleJSStyleWithColumns(8));
+  verifyFormat("export function* f() {\n"
+               "  yield 1;\n"
+               "}\n");
+  verifyFormat("class X {\n"
+               "  * generatorMethod() { yield x; }\n"
+               "}");
+}
+
+TEST_F(FormatTestJS, AsyncFunctions) {
+  verifyFormat("async function f() {\n"
+               "  let x = 1;\n"
+               "  return fetch(x);\n"
+               "}");
+  verifyFormat("async function* f() {\n"
+               "  yield fetch(x);\n"
+               "}");
+  verifyFormat("export async function f() {\n"
+               "  return fetch(x);\n"
+               "}");
+  verifyFormat("class X {\n"
+               "  async asyncMethod() { return fetch(1); }\n"
+               "}");
+}
+
 TEST_F(FormatTestJS, ArrayLiterals) {
   verifyFormat("var aaaaa: List<SomeThing> =\n"
                "    [new SomeThingAAAAAAAAAAAA(), new SomeThingBBBBBBBBB()];");
   verifyFormat("return [\n"
                "  aaaaaaaaaaaaaaaaaaaaaaaaaaa, bbbbbbbbbbbbbbbbbbbbbbbbbbb,\n"
                "  ccccccccccccccccccccccccccc\n"
                "];");
   verifyFormat("return [\n"