clang-format: [JS] space between pseudo keywords and template literals.

[mprobst]

Summary

Before:

yield`foo`;

After:

yield `foo`;

This reinstates commit 71d3b5cd91 / r307023 and fixes the logic by
introducing an explicit table of JavaScript pseudo keywords.

[djasper]

lib/Format/FormatToken.h

753

Nit: Should be "JsKeywords" (http://llvm.org/docs/CodingStandards.html#name-types-functions-variables-and-enumerators-properly). The kw_foo things follow a special convention to be the same as other token types.

[mprobst]

• fix style of field name and rename it to better match what it does (extra keywords)

lib/Format/FormatToken.h

 #ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKEN_H
 #define LLVM_CLANG_LIB_FORMAT_FORMATTOKEN_H
 
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/OperatorPrecedence.h"
 #include "clang/Format/Format.h"
 #include "clang/Lex/Lexer.h"
 #include <memory>
+#include <unordered_set>
 
 namespace clang {
 namespace format {
 
 #define LIST_TOKEN_TYPES \
   TYPE(ArrayInitializerLSquare) \
   TYPE(ArraySubscriptLSquare) \
   TYPE(AttributeParen) \
AdditionalKeywords(IdentifierTable &IdentTable) {
     kw_is = &IdentTable.get("is");
     kw_let = &IdentTable.get("let");
     kw_module = &IdentTable.get("module");
     kw_set = &IdentTable.get("set");
     kw_type = &IdentTable.get("type");
     kw_var = &IdentTable.get("var");
     kw_yield = &IdentTable.get("yield");
 
+    JsExtraKeywords = std::unordered_set<IdentifierInfo *>(
+        {kw_as, kw_async, kw_await, kw_declare, kw_finally, kw_from,
+         kw_function, kw_get, kw_import, kw_is, kw_let, kw_module, kw_set,
+         kw_type, kw_var, kw_yield,
+         // Keywords from the Java section.
+         kw_abstract, kw_extends, kw_implements, kw_instanceof, kw_interface});
+
     kw_abstract = &IdentTable.get("abstract");
     kw_assert = &IdentTable.get("assert");
     kw_extends = &IdentTable.get("extends");
     kw_implements = &IdentTable.get("implements");
     kw_instanceof = &IdentTable.get("instanceof");
     kw_interface = &IdentTable.get("interface");
     kw_native = &IdentTable.get("native");
     kw_package = &IdentTable.get("package");
struct AdditionalKeywords {
   IdentifierInfo *kw_required;
   IdentifierInfo *kw_returns;
 
   // QT keywords.
   IdentifierInfo *kw_signals;
   IdentifierInfo *kw_qsignals;
   IdentifierInfo *kw_slots;
   IdentifierInfo *kw_qslots;
+
+  /// \brief Returns \c true if \p Tok is a true JavaScript identifier, returns
+  /// \c false if it is a keyword or a pseudo keyword.
+  bool IsJavaScriptIdentifier(const FormatToken &Tok) const {
+    return Tok.is(tok::identifier) &&
+           JsExtraKeywords.find(Tok.Tok.getIdentifierInfo()) ==
+               JsExtraKeywords.end();
+  }
+
+private:
+  /// \brief The JavaScript keywords beyond the C++ keyword set.
+  std::unordered_set<IdentifierInfo *> JsExtraKeywords;
 };
 
 } // namespace format
 } // namespace clang
 
 #endif

lib/Format/TokenAnnotator.cpp

if (Left.is(Keywords.kw_async) && Right.is(tok::l_paren) &&
       // An async arrow function, for example: `x = async () => foo();`,
       // as opposed to calling a function called async: `x = async();`
       if (Next && Next->is(TT_JsFatArrow))
         return true;
     }
     if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
         (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
       return false;
-    if (Left.is(tok::identifier) && Right.is(TT_TemplateString))
+    // In tagged template literals ("html`bar baz`"), there is no space between
+    // the tag identifier and the template string. getIdentifierInfo makes sure
+    // that the identifier is not a pseudo keyword like `yield`, either.
+    if (Left.is(tok::identifier) && Keywords.IsJavaScriptIdentifier(Left) &&
+        Right.is(TT_TemplateString))
       return false;
     if (Right.is(tok::star) &&
         Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
       return false;
     if (Right.isOneOf(tok::l_brace, tok::l_square) &&
         Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
       return true;
     // JS methods can use some keywords as names (e.g. `delete()`).

unittests/Format/FormatTestJS.cpp

TEST_F(FormatTestJS, NestedTemplateStrings) {
   verifyFormat("var x = `he${({text: 'll'}.text)}o`;");
 
   // Crashed at some point.
   verifyFormat("}");
 }
 
 TEST_F(FormatTestJS, TaggedTemplateStrings) {
   verifyFormat("var x = html`<ul>`;");
+  verifyFormat("yield `hello`;");
 }
 
 TEST_F(FormatTestJS, CastSyntax) {
   verifyFormat("var x = <type>foo;");
   verifyFormat("var x = foo as type;");
   verifyFormat("let x = (a + b) as\n"
                "    LongTypeIsLong;",
                getGoogleJSStyleWithColumns(20));