clang-format: [js] Support template strings.

[mprobst]

Summary

Merge template strings (marked by backticks ``).
Do not format any contents of template strings.

[djasper]

lib/Format/Format.cpp

796–797

I think, I'd find this (very slightly) easier to read as:

if (I[0]->IsMultiline || I[0]->NewlinesBefore > 0)
  IsMultiline = true;

811

Reading the comment for ColumnWidth again, it says:

/// \brief The width of the non-whitespace parts of the token (or its first
/// line for multi-line tokens) in columns.

So, if this turns into a multiline token, we need the length of the first line. The reason is this:

var someLongVariableName = `aaaaaaaaa
        bbbbbbb
        cccccccc`;

While we can't do anything to the content of the template string without affecting the runtime behavior, we need to make the decision of whether or not to break after the "=". For that, we need to know the lengths of the first line of the token.

Should be reasonably easy to reproduce in a test :-).

813

I think we need to set LastLineColumnWidth only for MultiLine tokens. It should not have any meaning for single-line tokens. Setting it to LastColumn for non-multiline tokens certainly is confusing.

[djasper]

lib/Format/Format.cpp

798

Can you add a test with two template strings? I think that might do the wrong thing as you need to abort when you find a TT_TemplateString.

[mprobst]

Fix counting of ColumnWidth and LastLineColumnWidth.

Still contains an issue with width counting of tok::unknown tokens.

[mprobst]

lib/Format/Format.cpp

796–797

Done.

798

It's not strictly needed - if there was a preceding template string, it cannot just equals "", it must have at least opener and closer "`". But it's a reasonable optimization and IMHO adds clarity.

811

As discussed offline, this changed quite a bit. PTAL.

813

Done.

[djasper]

Looks good, I'll add the test and commit.

lib/Format/Format.cpp

798

I agree. Still seems useful to have a test ;-).

[mprobst]

lib/Format/Format.cpp

798

Sorry for being unclear, I actually did add one.

[djasper]

Submitted as r230011.

lib/Format/Format.cpp

public:
   }
 
   ArrayRef<FormatToken *> lex() {
     assert(Tokens.empty());
     assert(FirstInLineIndex == 0);
     do {
       Tokens.push_back(getNextToken());
       tryMergePreviousTokens();
-      if (Tokens.back()->NewlinesBefore > 0)
+      if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
         FirstInLineIndex = Tokens.size() - 1;
     } while (Tokens.back()->Tok.isNot(tok::eof));
     return Tokens;
   }
 
   const AdditionalKeywords &getKeywords() { return Keywords; }
 
 private:
   void tryMergePreviousTokens() {
     if (tryMerge_TMacro())
       return;
     if (tryMergeConflictMarkers())
       return;
     if (tryMergeLessLess())
       return;
 
     if (Style.Language == FormatStyle::LK_JavaScript) {
       if (tryMergeJSRegexLiteral())
         return;
       if (tryMergeEscapeSequence())
         return;
+      if (tryMergeTemplateString())
+        return;
 
       static tok::TokenKind JSIdentity[] = {tok::equalequal, tok::equal};
       static tok::TokenKind JSNotIdentity[] = {tok::exclaimequal, tok::equal};
       static tok::TokenKind JSShiftEqual[] = {tok::greater, tok::greater,
                                               tok::greaterequal};
       static tok::TokenKind JSRightArrow[] = {tok::equal, tok::greater};
       // FIXME: We probably need to change token type to mimic operator with the
       // correct priority.
for (auto I = Tokens.rbegin() + 1, E = Tokens.rend(); I != E; ++I) {
 
       // There can't be a newline inside a regex literal.
       if (I[0]->NewlinesBefore > 0)
         return false;
     }
     return false;
   }
 
+  bool tryMergeTemplateString() {
+    if (Tokens.size() < 2)
+      return false;
+
+    FormatToken *EndBacktick = Tokens.back();
+    if (!(EndBacktick->is(tok::unknown) && EndBacktick->TokenText == "`"))
+      return false;
+
+    unsigned TokenCount = 0;
+    bool IsMultiline = false;
+    unsigned EndColumnInFirstLine = 0;
+    for (auto I = Tokens.rbegin() + 1, E = Tokens.rend(); I != E; I++) {
+      ++TokenCount;
+      if (I[0]->NewlinesBefore > 0 || I[0]->IsMultiline)
+        IsMultiline = true;
+
+      // If there was a preceding template string, this must be the start of a
+      // template string, not the end.
+      if (I[0]->is(TT_TemplateString))
+        return false;
+
+      if (I[0]->isNot(tok::unknown) || I[0]->TokenText != "`") {
+        // Keep track of the rhs offset of the last token to wrap across lines -
+        // its the rhs offset of the first line of the template string, used to
+        // determine its width.
+        if (I[0]->IsMultiline)
+          EndColumnInFirstLine = I[0]->OriginalColumn + I[0]->ColumnWidth;
+        // If the token has newlines, the token before it (if it exists) is the
+        // rhs end of the previous line.
+        if (I[0]->NewlinesBefore > 0 && (I + 1 != E))
+          EndColumnInFirstLine = I[1]->OriginalColumn + I[1]->ColumnWidth;
+
+        continue;
+      }
+
+      Tokens.resize(Tokens.size() - TokenCount);
+      Tokens.back()->Type = TT_TemplateString;
+      const char *EndOffset = EndBacktick->TokenText.data() + 1;
+      Tokens.back()->TokenText =
+          StringRef(Tokens.back()->TokenText.data(),
+                    EndOffset - Tokens.back()->TokenText.data());
+      if (IsMultiline) {
+        // ColumnWidth is from backtick to last token in line.
+        // LastLineColumnWidth is 0 to backtick.
+        // x = `some content
+        //     until here`;
+        Tokens.back()->ColumnWidth =
+            EndColumnInFirstLine - Tokens.back()->OriginalColumn;
+        Tokens.back()->LastLineColumnWidth = EndBacktick->OriginalColumn;
+        Tokens.back()->IsMultiline = true;
+      } else {
+        // Token simply spans from start to end, +1 for the ` itself.
+        Tokens.back()->ColumnWidth =
+            EndBacktick->OriginalColumn - Tokens.back()->OriginalColumn + 1;
+      }
+      return true;
+    }
+    return false;
+  }
+
   bool tryMerge_TMacro() {
     if (Tokens.size() < 4)
       return false;
     FormatToken *Last = Tokens.back();
     if (!Last->is(tok::r_paren))
       return false;
 
     FormatToken *String = Tokens[Tokens.size() - 2];
FormatToken *getNextToken() {
     SourceLocation WhitespaceStart =
         FormatTok->Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
     FormatTok->IsFirst = IsFirstToken;
     IsFirstToken = false;
 
     // Consume and record whitespace until we find a significant token.
     unsigned WhitespaceLength = TrailingWhitespace;
     while (FormatTok->Tok.is(tok::unknown)) {
+      // FIXME: This miscounts tok:unknown tokens that are not just
+      // whitespace, e.g. a '`' character.
       for (int i = 0, e = FormatTok->TokenText.size(); i != e; ++i) {
         switch (FormatTok->TokenText[i]) {
         case '\n':
           ++FormatTok->NewlinesBefore;
           // FIXME: This is technically incorrect, as it could also
           // be a literal backslash at the end of the line.
           if (i == 0 || (FormatTok->TokenText[i - 1] != '\\' &&
                          (FormatTok->TokenText[i - 1] != '\r' || i == 1 ||

lib/Format/FormatToken.h

enum TokenType {
   TT_PointerOrReference,
   TT_PureVirtualSpecifier,
   TT_RangeBasedForLoopColon,
   TT_RegexLiteral,
   TT_SelectorName,
   TT_StartOfName,
   TT_TemplateCloser,
   TT_TemplateOpener,
+  TT_TemplateString,
   TT_TrailingAnnotation,
   TT_TrailingReturnArrow,
   TT_TrailingUnaryOperator,
   TT_UnaryOperator,
   TT_Unknown
 };
 
 // Represents what type of block a set of braces open.

unittests/Format/FormatTestJS.cpp

TEST_F(FormatTestJS, Modules) {
   verifyFormat("export var x = 12;");
   verifyFormat("export var x: number = 12;");
   verifyFormat("export const y = {\n"
                "  a: 1,\n"
                "  b: 2\n"
                "};");
 }
 
+TEST_F(FormatTestJS, TemplateStrings) {
+  // Keeps any whitespace/indentation within the template string.
+  EXPECT_EQ("var x = `hello\n"
+            "     ${  name    }\n"
+            "  !`;",
+            format("var x    =    `hello\n"
+                   "     ${  name    }\n"
+                   "  !`;"));
+
+  // FIXME: +1 / -1 offsets are to work around clang-format miscalculating
+  // widths for unknown tokens that are not whitespace (e.g. '`'). Remove when
+  // the code is corrected.
+
+  verifyFormat("var x =\n"
+               "    `hello ${world}` >= some();",
+               getGoogleJSStyleWithColumns(34)); // Barely doesn't fit.
+  verifyFormat("var x = `hello ${world}` >= some();",
+               getGoogleJSStyleWithColumns(35 + 1)); // Barely fits.
+  EXPECT_EQ("var x = `hello\n"
+            "  ${world}` >=\n"
+            "        some();",
+            format("var x =\n"
+                   "    `hello\n"
+                   "  ${world}` >= some();",
+                   getGoogleJSStyleWithColumns(21))); // Barely doesn't fit.
+  EXPECT_EQ("var x = `hello\n"
+            "  ${world}` >= some();",
+            format("var x =\n"
+                   "    `hello\n"
+                   "  ${world}` >= some();",
+                   getGoogleJSStyleWithColumns(22))); // Barely fits.
+
+  verifyFormat("var x =\n    `h`;", getGoogleJSStyleWithColumns(13 - 1));
+  EXPECT_EQ(
+      "var x =\n    `multi\n  line`;",
+      format("var x = `multi\n  line`;", getGoogleJSStyleWithColumns(14 - 1)));
+
+  // Two template strings.
+  verifyFormat("var x = `hello` == `hello`;");
+}
+
 } // end namespace tooling
 } // end namespace clang