clang-format: [JS] nested and tagged template strings.

[mprobst]

lib/Format/TokenAnnotator.cpp

1821

I've chosen 100 by fair dice roll. Please advise on how to pick a reasonable penalty :-)

[djasper]

lib/Format/FormatTokenLexer.cpp

230

I think, this could now use an elaborate comment on what it is actually trying to parse. And part of that should probably make it into the patch description.

235

Indent seems off.

260

Maybe use "Offset[1]" instead of "*(Offset + 1)"?

lib/Format/TokenAnnotator.cpp

1821

In what cases would you actually want to put a line break here? Presumably, if otherwise the column limit is violated. Are there any other cases?

[mprobst]

lib/Format/FormatTokenLexer.cpp

230

Done. Also, it turns out my attempt here was overly optimistic, I actually need a proper state stack to handle this correctly.

260

Is somePointer[1] idiomatic C for "the next element after this pointer"? To me + 1 seemed like a better expression of intent, if a bit more wordy. Done in any case.

lib/Format/TokenAnnotator.cpp

1821

No, I think breaking in the template string should probably be a last resort. It's IMHO conceptually tighter than string concatenation using +. That being said, I have no idea what number that should translate into :-)

[mprobst]

• Use a stack to parse nested template strings.
• move docs

[mprobst]

Friendly ping :-)

[djasper]

lib/Format/FormatTokenLexer.cpp

252

I'd use Offset[0]

lib/Format/FormatTokenLexer.h

86

Should we also use this for GreaterStashed and LessStashed?

[mprobst]

• Use a stack to parse nested template strings.
• move docs
• Fold GreaterStashed, LessStashed into TOKEN_STASHED lexer state.

[mprobst]

• Test escaped dollar sign.

lib/Format/FormatTokenLexer.cpp

259

Question by @ygao in the other diff:

What happens if the '${' is immediately after a backslash (the if statement above), should the '${' get escaped?

A template like foo\${bar} is the literal string 'foo\${bar}', i.e. the \ acts as an escape for the '$' sign. I've added a test to validate that.

[mprobst]

• Fix escaping issue.

[djasper]

Looks good. Sorry for the delay!

lib/Format/FormatTokenLexer.h

68

.. in turn.

[mprobst]

No worries, thanks for the review.

lib/Format/FormatTokenLexer.h

 
 #include "Encoding.h"
 #include "FormatToken.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Format/Format.h"
 #include "llvm/Support/Regex.h"
 
+#include <stack>
+
 namespace clang {
 namespace format {
 
+enum LexerState {
+  NORMAL,
+  TEMPLATE_STRING,
+  TOKEN_STASHED,
+};
+
 class FormatTokenLexer {
 public:
   FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
                    const FormatStyle &Style, encoding::Encoding Encoding);
 
   ArrayRef<FormatToken *> lex();
 
   const AdditionalKeywords &getKeywords() { return Keywords; }
private:
   bool canPrecedeRegexLiteral(FormatToken *Prev);
 
   // Tries to parse a JavaScript Regex literal starting at the current token,
   // if that begins with a slash and is in a location where JavaScript allows
   // regex literals. Changes the current token to a regex literal and updates
   // its text if successful.
   void tryParseJSRegexLiteral();
 
-  void tryParseTemplateString();
+  // Handles JavaScript template strings.
+  //
+  // JavaScript template strings use backticks ('`') as delimiters, and allow
+  // embedding expressions nested in ${expr-here}. Template strings can be
+  // nested recursively, i.e. expressions can contain template strings in turns.
+  //
+  // The code below parses starting from a backtick, up to a closing backtick or
+  // an opening ${. It also maintains a stack of lexing contexts to handle
+  // nested template parts by balancing curly braces.
+  void handleTemplateStrings();
 
   bool tryMerge_TMacro();
 
   bool tryMergeConflictMarkers();
 
   FormatToken *getStashedToken();
 
   FormatToken *getNextToken();
 
   FormatToken *FormatTok;
   bool IsFirstToken;
-  bool GreaterStashed, LessStashed;
+  std::stack<LexerState> StateStack;
   unsigned Column;
   unsigned TrailingWhitespace;
   std::unique_ptr<Lexer> Lex;
   const SourceManager &SourceMgr;
   FileID ID;
   const FormatStyle &Style;
   IdentifierTable IdentTable;
   AdditionalKeywords Keywords;
   encoding::Encoding Encoding;

lib/Format/FormatTokenLexer.cpp

 #include "llvm/Support/Regex.h"
 
 namespace clang {
 namespace format {
 
 FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
                                    const FormatStyle &Style,
                                    encoding::Encoding Encoding)
-    : FormatTok(nullptr), IsFirstToken(true), GreaterStashed(false),
-      LessStashed(false), Column(0), TrailingWhitespace(0),
-      SourceMgr(SourceMgr), ID(ID), Style(Style),
-      IdentTable(getFormattingLangOpts(Style)), Keywords(IdentTable),
-      Encoding(Encoding), FirstInLineIndex(0), FormattingDisabled(false),
-      MacroBlockBeginRegex(Style.MacroBlockBegin),
+    : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
+      Column(0), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
+      Style(Style), IdentTable(getFormattingLangOpts(Style)),
+      Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
+      FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
       MacroBlockEndRegex(Style.MacroBlockEnd) {
   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
                       getFormattingLangOpts(Style)));
   Lex->SetKeepWhitespaceMode(true);
 
   for (const std::string &ForEachMacro : Style.ForEachMacros)
     ForEachMacros.push_back(&IdentTable.get(ForEachMacro));
   std::sort(ForEachMacros.begin(), ForEachMacros.end());
 }
 
 ArrayRef<FormatToken *> FormatTokenLexer::lex() {
   assert(Tokens.empty());
   assert(FirstInLineIndex == 0);
   do {
     Tokens.push_back(getNextToken());
     if (Style.Language == FormatStyle::LK_JavaScript) {
       tryParseJSRegexLiteral();
-      tryParseTemplateString();
+      handleTemplateStrings();
     }
     tryMergePreviousTokens();
     if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
       FirstInLineIndex = Tokens.size() - 1;
   } while (Tokens.back()->Tok.isNot(tok::eof));
   return Tokens;
 }
 
void FormatTokenLexer::tryParseJSRegexLiteral() {
   // Treat regex literals like other string_literals.
   RegexToken->Tok.setKind(tok::string_literal);
   RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
   RegexToken->ColumnWidth = RegexToken->TokenText.size();
 
   resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
 }
 
-void FormatTokenLexer::tryParseTemplateString() {
+void FormatTokenLexer::handleTemplateStrings() {
   FormatToken *BacktickToken = Tokens.back();
-  if (!BacktickToken->is(tok::unknown) || BacktickToken->TokenText != "`")
+
+  if (BacktickToken->is(tok::l_brace)) {
+    StateStack.push(LexerState::NORMAL);
+    return;
+  }
+  if (BacktickToken->is(tok::r_brace)) {
+    StateStack.pop();
+    if (StateStack.top() != LexerState::TEMPLATE_STRING)
       return;
+    // If back in TEMPLATE_STRING, fallthrough and continue parsing the
+  } else if (BacktickToken->is(tok::unknown) &&
+             BacktickToken->TokenText == "`") {
+    StateStack.push(LexerState::TEMPLATE_STRING);
+  } else {
+    return; // Not actually a template
+  }
 
   // 'Manually' lex ahead in the current file buffer.
   const char *Offset = Lex->getBufferLocation();
   const char *TmplBegin = Offset - BacktickToken->TokenText.size(); // at "`"
-  for (; Offset != Lex->getBuffer().end() && *Offset != '`'; ++Offset) {
-    if (*Offset == '\\')
+  for (; Offset != Lex->getBuffer().end(); ++Offset) {
+    if (Offset[0] == '`') {
+      StateStack.pop();
+      break;
+    }
+    if (Offset[0] == '\\')
       ++Offset; // Skip the escaped character.
+    if (Offset + 1 < Lex->getBuffer().end() && Offset[0] == '$' &&
+        Offset[1] == '{') {
+      // '${' introduces an expression interpolation in the template string.
+      StateStack.push(LexerState::NORMAL);
+      ++Offset;
+      break;
+    }
   }
 
   StringRef LiteralText(TmplBegin, Offset - TmplBegin + 1);
   BacktickToken->Type = TT_TemplateString;
   BacktickToken->Tok.setKind(tok::string_literal);
   BacktickToken->TokenText = LiteralText;
 
   // Adjust width for potentially multiline string literals.
   size_t FirstBreak = LiteralText.find('\n');
   StringRef FirstLineText = FirstBreak == StringRef::npos
                                 ? LiteralText
                                 : LiteralText.substr(0, FirstBreak);
   BacktickToken->ColumnWidth = encoding::columnWidthWithTabs(
       FirstLineText, BacktickToken->OriginalColumn, Style.TabWidth, Encoding);
   size_t LastBreak = LiteralText.rfind('\n');
   if (LastBreak != StringRef::npos) {
     BacktickToken->IsMultiline = true;
     unsigned StartColumn = 0; // The template tail spans the entire line.
     BacktickToken->LastLineColumnWidth = encoding::columnWidthWithTabs(
         LiteralText.substr(LastBreak + 1, LiteralText.size()), StartColumn,
         Style.TabWidth, Encoding);
   }
 
-  resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset + 1)));
+  SourceLocation loc = Offset < Lex->getBuffer().end()
+                           ? Lex->getSourceLocation(Offset + 1)
+                           : SourceMgr.getLocForEndOfFile(ID);
+  resetLexer(SourceMgr.getFileOffset(loc));
 }
 
 bool FormatTokenLexer::tryMerge_TMacro() {
   if (Tokens.size() < 4)
     return false;
   FormatToken *Last = Tokens.back();
   if (!Last->is(tok::r_paren))
     return false;
FormatToken *FormatTokenLexer::getStashedToken() {
   FormatTok->TokenText = TokenText;
   FormatTok->ColumnWidth = 1;
   FormatTok->OriginalColumn = OriginalColumn + 1;
 
   return FormatTok;
 }
 
 FormatToken *FormatTokenLexer::getNextToken() {
-  if (GreaterStashed) {
-    GreaterStashed = false;
-    return getStashedToken();
-  }
-  if (LessStashed) {
-    LessStashed = false;
+  if (StateStack.top() == LexerState::TOKEN_STASHED) {
+    StateStack.pop();
     return getStashedToken();
   }
 
   FormatTok = new (Allocator.Allocate()) FormatToken;
   readRawToken(*FormatTok);
   SourceLocation WhitespaceStart =
       FormatTok->Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
   FormatTok->IsFirst = IsFirstToken;
if (Style.Language == FormatStyle::LK_Java &&
                FormatTok->isOneOf(tok::kw_struct, tok::kw_union,
                                   tok::kw_operator)) {
       FormatTok->Tok.setKind(tok::identifier);
       FormatTok->Tok.setIdentifierInfo(nullptr);
     }
   } else if (FormatTok->Tok.is(tok::greatergreater)) {
     FormatTok->Tok.setKind(tok::greater);
     FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
-    GreaterStashed = true;
+    StateStack.push(LexerState::TOKEN_STASHED);
   } else if (FormatTok->Tok.is(tok::lessless)) {
     FormatTok->Tok.setKind(tok::less);
     FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
-    LessStashed = true;
+    StateStack.push(LexerState::TOKEN_STASHED);
   }
 
   // Now FormatTok is the next non-whitespace token.
 
   StringRef Text = FormatTok->TokenText;
   size_t FirstNewlinePos = Text.find('\n');
   if (FirstNewlinePos == StringRef::npos) {
     // FIXME: ColumnWidth actually depends on the start column, we need to

lib/Format/TokenAnnotator.cpp

void resetTokenMetadata(FormatToken *Token) {
     if (!Token)
       return;
 
     // Reset token type in case we have already looked at it and then
     // recovered from an error (e.g. failure to find the matching >).
     if (!CurrentToken->isOneOf(TT_LambdaLSquare, TT_ForEachMacro,
                                TT_FunctionLBrace, TT_ImplicitStringLiteral,
                                TT_InlineASMBrace, TT_JsFatArrow, TT_LambdaArrow,
-                               TT_RegexLiteral))
+                               TT_RegexLiteral, TT_TemplateString))
       CurrentToken->Type = TT_Unknown;
     CurrentToken->Role.reset();
     CurrentToken->MatchingParen = nullptr;
     CurrentToken->FakeLParens.clear();
     CurrentToken->FakeRParens = 0;
   }
 
   void next() {
if (Right.is(Keywords.kw_implements))
       return 2;
     if (Left.is(tok::comma) && Left.NestingLevel == 0)
       return 3;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma))
       return 100;
     if (Left.is(TT_JsTypeColon))
       return 35;
+    if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
+        (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
+      return 100;
   }
 
   if (Left.is(tok::comma) || (Right.is(tok::identifier) && Right.Next &&
                               Right.Next->is(TT_DictLiteral)))
     return 1;
   if (Right.is(tok::l_square)) {
     if (Style.Language == FormatStyle::LK_Proto)
       return 1;
if (Right.is(tok::period) &&
                      Keywords.kw_repeated, Keywords.kw_extend))
       return true;
     if (Right.is(tok::l_paren) &&
         Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
       return true;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     if (Left.is(TT_JsFatArrow))
       return true;
+    if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
+        (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
+      return false;
+    if (Left.is(tok::identifier) && Right.is(TT_TemplateString))
+      return false;
     if (Right.is(tok::star) &&
         Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
       return false;
     if (Left.isOneOf(Keywords.kw_let, Keywords.kw_var, Keywords.kw_in,
                      Keywords.kw_of, tok::kw_const) &&
         (!Left.Previous || !Left.Previous->is(tok::period)))
       return true;
     if (Left.is(tok::kw_default) && Left.Previous &&

unittests/Format/FormatTestJS.cpp

verifyFormat("export {\n"
                "  A,\n"
                "} from 'some/module.js';",
                Style);
 }
 
 TEST_F(FormatTestJS, TemplateStrings) {
   // Keeps any whitespace/indentation within the template string.
   verifyFormat("var x = `hello\n"
-            "     ${  name    }\n"
+            "     ${name}\n"
             "  !`;",
             "var x    =    `hello\n"
                    "     ${  name    }\n"
                    "  !`;");
 
   verifyFormat("var x =\n"
                "    `hello ${world}` >= some();",
                getGoogleJSStyleWithColumns(34)); // Barely doesn't fit.
verifyFormat("var x = `/*a`;\n"
                "var y = ` */ `;",
                "var x =\n `/*a`;\n"
                "var y =\n ` */ `;");
   // Escaped backtick.
   verifyFormat("var x = ` \\` a`;\n"
                "var y;",
                "var x = ` \\` a`;\n"
                "var y;");
+  // Escaped dollar.
+  verifyFormat("var x = ` \\${foo}`;\n");
+}
+
+TEST_F(FormatTestJS, NestedTemplateStrings) {
+  verifyFormat(
+      "var x = `<ul>${xs.map(x => `<li>${x}</li>`).join('\\n')}</ul>`;");
+  verifyFormat("var x = `he${({text: 'll'}.text)}o`;");
+}
+
+TEST_F(FormatTestJS, TaggedTemplateStrings) {
+  verifyFormat("var x = html`<ul>`;");
 }
 
 TEST_F(FormatTestJS, CastSyntax) {
   verifyFormat("var x = <type>foo;");
   verifyFormat("var x = foo as type;");
   verifyFormat("let x = (a + b) as\n"
                "    LongTypeIsLong;",
                getGoogleJSStyleWithColumns(20));