clang-format: [JS] handle comments in template strings.

[djasper]

unittests/Format/FormatTestJS.cpp

653

I'd be slightly hesitant to use verifyFormat for any of these as that simply cops out when finding "//". Thus, tests might pass for the wrong reasons. It is probably better to use EXPECT_EQ with a manually messed up "before".

[mprobst]

Using EXPECT_EQ and manually messed up strings.

[mprobst]

unittests/Format/FormatTestJS.cpp

653

Done, wasn't aware of that. clang-format would actually misformat these snippets (indenting the second var) before the change, so the tests did move from red to green during development.

[djasper]

Submitted as r235078.

lib/Format/Format.cpp

bool tryMergeJSRegexLiteral() {
     return false;
   }
 
   bool tryMergeTemplateString() {
     if (Tokens.size() < 2)
       return false;
 
     FormatToken *EndBacktick = Tokens.back();
-    if (!(EndBacktick->is(tok::unknown) && EndBacktick->TokenText == "`"))
+    // Backticks get lexed as tok:unknown tokens. If a template string contains
+    // a comment start, it gets lexed as a tok::comment, or tok::unknown if
+    // unterminated.
+    if (!EndBacktick->isOneOf(tok::comment, tok::unknown))
+      return false;
+    size_t CommentBacktickPos = EndBacktick->TokenText.find('`');
+    // Unknown token that's not actually a backtick, or a comment that doesn't
+    // contain a backtick.
+    if (CommentBacktickPos == StringRef::npos)
       return false;
 
     unsigned TokenCount = 0;
     bool IsMultiline = false;
     unsigned EndColumnInFirstLine = 0;
     for (auto I = Tokens.rbegin() + 1, E = Tokens.rend(); I != E; I++) {
       ++TokenCount;
       if (I[0]->NewlinesBefore > 0 || I[0]->IsMultiline)
for (auto I = Tokens.rbegin() + 1, E = Tokens.rend(); I != E; I++) {
         if (I[0]->NewlinesBefore > 0 && (I + 1 != E))
           EndColumnInFirstLine = I[1]->OriginalColumn + I[1]->ColumnWidth;
 
         continue;
       }
 
       Tokens.resize(Tokens.size() - TokenCount);
       Tokens.back()->Type = TT_TemplateString;
-      const char *EndOffset = EndBacktick->TokenText.data() + 1;
+      const char *EndOffset =
+          EndBacktick->TokenText.data() + 1 + CommentBacktickPos;
+      if (CommentBacktickPos != 0) {
+        // If the backtick was not the first character (e.g. in a comment),
+        // re-lex after the backtick position.
+        SourceLocation Loc = EndBacktick->Tok.getLocation();
+        resetLexer(SourceMgr.getFileOffset(Loc) + CommentBacktickPos + 1);
+      }
       Tokens.back()->TokenText =
           StringRef(Tokens.back()->TokenText.data(),
                     EndOffset - Tokens.back()->TokenText.data());
       if (IsMultiline) {
         // ColumnWidth is from backtick to last token in line.
         // LastLineColumnWidth is 0 to backtick.
         // x = `some content
         //     until here`;

unittests/Format/FormatTestJS.cpp

TEST_F(FormatTestJS, TemplateStrings) {
 
   verifyFormat("var x =\n    `h`;", getGoogleJSStyleWithColumns(13 - 1));
   EXPECT_EQ(
       "var x =\n    `multi\n  line`;",
       format("var x = `multi\n  line`;", getGoogleJSStyleWithColumns(14 - 1)));
 
   // Two template strings.
   verifyFormat("var x = `hello` == `hello`;");
+
+  // Comments in template strings.
+  EXPECT_EQ("var x = `//a`;\n"
+            "var y;",
+            format("var x =\n `//a`;\n"
+                   "var y  ;"));
+  EXPECT_EQ("var x = `/*a`;\n"
+            "var y;",
+            format("var x =\n `/*a`;\n"
+                   "var y;"));
+  // Backticks in a comment - not a template string.
+  EXPECT_EQ("var x = 1  // `/*a`;\n"
+            "    ;",
+            format("var x =\n 1  // `/*a`;\n"
+                   "    ;"));
+  EXPECT_EQ("/* ` */ var x = 1; /* ` */",
+            format("/* ` */ var x\n= 1; /* ` */"));
+  // Comment spans multiple template strings.
+  EXPECT_EQ("var x = `/*a`;\n"
+            "var y = ` */ `;",
+            format("var x =\n `/*a`;\n"
+                   "var y =\n ` */ `;"));
+  // Escaped backtick.
+  EXPECT_EQ("var x = ` \\` a`;\n"
+            "var y;",
+            format("var x = ` \\` a`;\n"
+                   "var y;"));
 }
 
 TEST_F(FormatTestJS, CastSyntax) {
   verifyFormat("var x = <type>foo;");
 }
 
 TEST_F(FormatTestJS, TypeArguments) {
   verifyFormat("class X<Y> {}");