clang-format: [JS] nested and tagged template strings.

[mprobst]

lib/Format/TokenAnnotator.cpp

1821

I've chosen 100 by fair dice roll. Please advise on how to pick a reasonable penalty :-)

[djasper]

lib/Format/FormatTokenLexer.cpp

231

I think, this could now use an elaborate comment on what it is actually trying to parse. And part of that should probably make it into the patch description.

236

Indent seems off.

246

Maybe use "Offset[1]" instead of "*(Offset + 1)"?

lib/Format/TokenAnnotator.cpp

1821

In what cases would you actually want to put a line break here? Presumably, if otherwise the column limit is violated. Are there any other cases?

[mprobst]

lib/Format/FormatTokenLexer.cpp

231

Done. Also, it turns out my attempt here was overly optimistic, I actually need a proper state stack to handle this correctly.

246

Is somePointer[1] idiomatic C for "the next element after this pointer"? To me + 1 seemed like a better expression of intent, if a bit more wordy. Done in any case.

lib/Format/TokenAnnotator.cpp

1821

No, I think breaking in the template string should probably be a last resort. It's IMHO conceptually tighter than string concatenation using +. That being said, I have no idea what number that should translate into :-)

[mprobst]

• Use a stack to parse nested template strings.
• move docs

[mprobst]

Friendly ping :-)

[djasper]

lib/Format/FormatTokenLexer.cpp

243

I'd use Offset[0]

lib/Format/FormatTokenLexer.h

69

Should we also use this for GreaterStashed and LessStashed?

[mprobst]

• Use a stack to parse nested template strings.
• move docs
• Fold GreaterStashed, LessStashed into TOKEN_STASHED lexer state.

[mprobst]

• Test escaped dollar sign.

lib/Format/FormatTokenLexer.cpp

245

Question by @ygao in the other diff:

What happens if the '${' is immediately after a backslash (the if statement above), should the '${' get escaped?

A template like foo\${bar} is the literal string 'foo\${bar}', i.e. the \ acts as an escape for the '$' sign. I've added a test to validate that.

[mprobst]

• Fix escaping issue.

[djasper]

Looks good. Sorry for the delay!

lib/Format/FormatTokenLexer.h

60

.. in turn.

[mprobst]

No worries, thanks for the review.

lib/Format/FormatTokenLexer.h

private:
   // regex literals. Changes the current token to a regex literal and updates
   // its text if successful.
   void tryParseJSRegexLiteral();
 
   void tryParseTemplateString();
 
   bool tryMerge_TMacro();
 
   bool tryMergeConflictMarkers();
 
   FormatToken *getStashedToken();
 
   FormatToken *getNextToken();
 
   FormatToken *FormatTok;
   bool IsFirstToken;
   bool GreaterStashed, LessStashed;
+  unsigned TemplateStringDepth;
   unsigned Column;
   unsigned TrailingWhitespace;
   std::unique_ptr<Lexer> Lex;
   const SourceManager &SourceMgr;
   FileID ID;
   const FormatStyle &Style;
   IdentifierTable IdentTable;
   AdditionalKeywords Keywords;

lib/Format/FormatTokenLexer.cpp

 
 namespace clang {
 namespace format {
 
 FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
                                    const FormatStyle &Style,
                                    encoding::Encoding Encoding)
     : FormatTok(nullptr), IsFirstToken(true), GreaterStashed(false),
-      LessStashed(false), Column(0), TrailingWhitespace(0),
-      SourceMgr(SourceMgr), ID(ID), Style(Style),
+      LessStashed(false), TemplateStringDepth(0), Column(0),
+      TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID), Style(Style),
       IdentTable(getFormattingLangOpts(Style)), Keywords(IdentTable),
       Encoding(Encoding), FirstInLineIndex(0), FormattingDisabled(false),
       MacroBlockBeginRegex(Style.MacroBlockBegin),
       MacroBlockEndRegex(Style.MacroBlockEnd) {
   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
                       getFormattingLangOpts(Style)));
   Lex->SetKeepWhitespaceMode(true);
 
void FormatTokenLexer::tryParseJSRegexLiteral() {
   // Treat regex literals like other string_literals.
   RegexToken->Tok.setKind(tok::string_literal);
   RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
   RegexToken->ColumnWidth = RegexToken->TokenText.size();
 
   resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
 }
 
 void FormatTokenLexer::tryParseTemplateString() {
   FormatToken *BacktickToken = Tokens.back();
-  if (!BacktickToken->is(tok::unknown) || BacktickToken->TokenText != "`")
+  if (TemplateStringDepth > 0 && BacktickToken->TokenText == "}")
+    TemplateStringDepth--;
+  else if (!BacktickToken->is(tok::unknown) ||
+             BacktickToken->TokenText != "`")
     return;
 
   // 'Manually' lex ahead in the current file buffer.
   const char *Offset = Lex->getBufferLocation();
   const char *TmplBegin = Offset - BacktickToken->TokenText.size(); // at "`"
   for (; Offset != Lex->getBuffer().end() && *Offset != '`'; ++Offset) {
     if (*Offset == '\\')
       ++Offset; // Skip the escaped character.
+    if (Offset + 1 < Lex->getBuffer().end() && *Offset == '$' &&
+        *(Offset + 1) == '{') {
+      // '${' introduces an expression interpolation in the template string.
+      TemplateStringDepth++;
+      ++Offset;
+      break;
+    }
   }
 
   StringRef LiteralText(TmplBegin, Offset - TmplBegin + 1);
   BacktickToken->Type = TT_TemplateString;
   BacktickToken->Tok.setKind(tok::string_literal);
   BacktickToken->TokenText = LiteralText;
 
   // Adjust width for potentially multiline string literals.

lib/Format/TokenAnnotator.cpp

void resetTokenMetadata(FormatToken *Token) {
     if (!Token)
       return;
 
     // Reset token type in case we have already looked at it and then
     // recovered from an error (e.g. failure to find the matching >).
     if (!CurrentToken->isOneOf(TT_LambdaLSquare, TT_ForEachMacro,
                                TT_FunctionLBrace, TT_ImplicitStringLiteral,
                                TT_InlineASMBrace, TT_JsFatArrow, TT_LambdaArrow,
-                               TT_RegexLiteral))
+                               TT_RegexLiteral, TT_TemplateString))
       CurrentToken->Type = TT_Unknown;
     CurrentToken->Role.reset();
     CurrentToken->MatchingParen = nullptr;
     CurrentToken->FakeLParens.clear();
     CurrentToken->FakeRParens = 0;
   }
 
   void next() {
if (Right.is(Keywords.kw_implements))
       return 2;
     if (Left.is(tok::comma) && Left.NestingLevel == 0)
       return 3;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma))
       return 100;
     if (Left.is(TT_JsTypeColon))
       return 35;
+    if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
+        (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
+      return 100;
   }
 
   if (Left.is(tok::comma) || (Right.is(tok::identifier) && Right.Next &&
                               Right.Next->is(TT_DictLiteral)))
     return 1;
   if (Right.is(tok::l_square)) {
     if (Style.Language == FormatStyle::LK_Proto)
       return 1;
if (Right.is(tok::period) &&
                      Keywords.kw_repeated, Keywords.kw_extend))
       return true;
     if (Right.is(tok::l_paren) &&
         Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
       return true;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     if (Left.is(TT_JsFatArrow))
       return true;
+    if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
+        (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
+      return false;
+    if (Left.is(tok::identifier) && Right.is(TT_TemplateString))
+      return false;
     if (Right.is(tok::star) &&
         Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
       return false;
     if (Left.isOneOf(Keywords.kw_let, Keywords.kw_var, Keywords.kw_in,
                      Keywords.kw_of, tok::kw_const) &&
         (!Left.Previous || !Left.Previous->is(tok::period)))
       return true;
     if (Left.is(tok::kw_default) && Left.Previous &&

unittests/Format/FormatTestJS.cpp

verifyFormat("export {\n"
                "  A,\n"
                "} from 'some/module.js';",
                Style);
 }
 
 TEST_F(FormatTestJS, TemplateStrings) {
   // Keeps any whitespace/indentation within the template string.
   verifyFormat("var x = `hello\n"
-            "     ${  name    }\n"
+            "     ${name}\n"
             "  !`;",
             "var x    =    `hello\n"
                    "     ${  name    }\n"
                    "  !`;");
 
   verifyFormat("var x =\n"
                "    `hello ${world}` >= some();",
                getGoogleJSStyleWithColumns(34)); // Barely doesn't fit.
verifyFormat("var x = `/*a`;\n"
                "var y = ` */ `;",
                "var x =\n `/*a`;\n"
                "var y =\n ` */ `;");
   // Escaped backtick.
   verifyFormat("var x = ` \\` a`;\n"
                "var y;",
                "var x = ` \\` a`;\n"
                "var y;");
+  verifyFormat(
+      "var x = `<ul>${xs.map(x => `<li>${x}</li>`).join('\\n')}</ul>`;");
+}
+
+TEST_F(FormatTestJS, TaggedTemplateStrings) {
+  verifyFormat("var x = html`<ul>`;");
 }
 
 TEST_F(FormatTestJS, CastSyntax) {
   verifyFormat("var x = <type>foo;");
   verifyFormat("var x = foo as type;");
   verifyFormat("let x = (a + b) as\n"
                "    LongTypeIsLong;",
                getGoogleJSStyleWithColumns(20));