[clang-format] Improve support for multiline C# strings

[jbcoe]

Fix failing test by bypassing verifyFormat which does not understand C# multiline strings.

[krasimir]

This is pretty cool! Just a couple of nits.

clang/lib/Format/FormatTokenLexer.cpp

187

nit: please add a comment that some of the string merging is also handled in handleCSharpVerbatimAndInterpolatedStrings.

544

isn't this a no-op?

clang/unittests/Format/FormatTestCSharp.cpp

435

consider adding a test case with a multiline string in the middle of the body of a function with statements after it to verify that the indentation used after the string literal is correct.

[jbcoe]

Add additional comment as requested in review.

[jbcoe]

Address review comments - add tests and remove no-op.

[MyDeveloperDay]

these are great patches you are making...

clang/lib/Format/ContinuationIndenter.cpp

ContinuationIndenter::getRawStringStyle(const FormatToken &Current,
   return RawStringStyle;
 }
 
 std::unique_ptr<BreakableToken>
 ContinuationIndenter::createBreakableToken(const FormatToken &Current,
                                            LineState &State, bool AllowBreak) {
   unsigned StartColumn = State.Column - Current.ColumnWidth;
   if (Current.isStringLiteral()) {
-    // FIXME: String literal breaking is currently disabled for C#,Java and
+    // FIXME: String literal breaking is currently disabled for C#, Java and
     // JavaScript, as it requires strings to be merged using "+" which we
     // don't support.
     if (Style.Language == FormatStyle::LK_Java ||
         Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp() ||
         !Style.BreakStringLiterals || !AllowBreak)
       return nullptr;
 
     // Don't break string literals inside preprocessor directives (except for

clang/lib/Format/FormatTokenLexer.h

private:
   // embedding expressions nested in ${expr-here}. Template strings can be
   // nested recursively, i.e. expressions can contain template strings in turn.
   //
   // The code below parses starting from a backtick, up to a closing backtick or
   // an opening ${. It also maintains a stack of lexing contexts to handle
   // nested template parts by balancing curly braces.
   void handleTemplateStrings();
 
+  void handleCSharpVerbatimAndInterpolatedStrings();
+
   void tryParsePythonComment();
 
   bool tryMerge_TMacro();
 
   bool tryMergeConflictMarkers();
 
   FormatToken *getStashedToken();
 

clang/lib/Format/FormatTokenLexer.cpp

do {
     Tokens.push_back(getNextToken());
     if (Style.Language == FormatStyle::LK_JavaScript) {
       tryParseJSRegexLiteral();
       handleTemplateStrings();
     }
     if (Style.Language == FormatStyle::LK_TextProto)
       tryParsePythonComment();
     tryMergePreviousTokens();
+    if (Style.isCSharp())
+      // This needs to come after tokens have been merged so that C#
+      // string literals are correctly identified.
+      handleCSharpVerbatimAndInterpolatedStrings();
     if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
       FirstInLineIndex = Tokens.size() - 1;
   } while (Tokens.back()->Tok.isNot(tok::eof));
   return Tokens;
 }
 
 void FormatTokenLexer::tryMergePreviousTokens() {
   if (tryMerge_TMacro())
bool FormatTokenLexer::tryMergeJSPrivateIdentifier() {
   Hash->ColumnWidth += Identifier->ColumnWidth;
   Hash->Type = TT_JsPrivateIdentifier;
   Tokens.erase(Tokens.end() - 1);
   return true;
 }
 
 // Search for verbatim or interpolated string literals @"ABC" or
 // $"aaaaa{abc}aaaaa" i and mark the token as TT_CSharpStringLiteral, and to
 // prevent splitting of @, $ and ".
 bool FormatTokenLexer::tryMergeCSharpStringLiteral() {
   if (Tokens.size() < 2)
     return false;
 
-  auto &CSharpStringLiteral = *(Tokens.end() - 2);
-
   // Interpolated strings could contain { } with " characters inside.
   // $"{x ?? "null"}"
   // should not be split into $"{x ?? ", null, "}" but should treated as a
   // single string-literal.
   //
   // We opt not to try and format expressions inside {} within a C#
   // interpolated string. Formatting expressions within an interpolated string
   // would require similar work as that done for JavaScript template strings
if (UnmatchedOpeningBraceCount > 0) {
                     NextToken->TokenText.end() -
                         CSharpInterpolatedString->TokenText.begin());
       CSharpInterpolatedString->ColumnWidth += NextToken->ColumnWidth;
       Tokens.erase(Tokens.end() - 1);
       return true;
     }
   }
 
-  // verbatim strings could contain "" which C# sees as an escaped ".
-  // @"""Hello""" will have been tokenized as @"" "Hello" "" and needs
-  // merging into a single string literal.
+  // Look for @"aaaaaa" or $"aaaaaa".
   auto &String = *(Tokens.end() - 1);
   if (!String->is(tok::string_literal))
     return false;
 
-  if (CSharpStringLiteral->Type == TT_CSharpStringLiteral &&
-      (CSharpStringLiteral->TokenText.startswith(R"(@")") ||
-       CSharpStringLiteral->TokenText.startswith(R"($@")"))) {
-    CSharpStringLiteral->TokenText = StringRef(
-        CSharpStringLiteral->TokenText.begin(),
-        String->TokenText.end() - CSharpStringLiteral->TokenText.begin());
-    CSharpStringLiteral->ColumnWidth += String->ColumnWidth;
-    Tokens.erase(Tokens.end() - 1);
-    return true;
-  }
-
   auto &At = *(Tokens.end() - 2);
-
-  // Look for @"aaaaaa" or $"aaaaaa".
   if (!(At->is(tok::at) || At->TokenText == "$"))
     return false;
 
   if (Tokens.size() > 2 && At->is(tok::at)) {
     auto &Dollar = *(Tokens.end() - 3);
     if (Dollar->TokenText == "$") {
       // This looks like $@"aaaaa" so we need to combine all 3 tokens.
       Dollar->Tok.setKind(tok::string_literal);
void FormatTokenLexer::tryParseJSRegexLiteral() {
   // Treat regex literals like other string_literals.
   RegexToken->Tok.setKind(tok::string_literal);
   RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
   RegexToken->ColumnWidth = RegexToken->TokenText.size();
 
   resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
 }
 
+void FormatTokenLexer::handleCSharpVerbatimAndInterpolatedStrings() {
+  FormatToken *CSharpStringLiteral = Tokens.back();
+
+  if (CSharpStringLiteral->Type != TT_CSharpStringLiteral)
+    return;
+
+  // Deal with multiline strings.
+  if (!(CSharpStringLiteral->TokenText.startswith(R"(@")") ||
+        CSharpStringLiteral->TokenText.startswith(R"($@")")))
+    return;
+
+  const char *StrBegin =
+      Lex->getBufferLocation() - CSharpStringLiteral->TokenText.size();
+  const char *Offset = StrBegin;
+  if (CSharpStringLiteral->TokenText.startswith(R"(@")"))
+    Offset += 2;
+  else // CSharpStringLiteral->TokenText.startswith(R"($@")")
+    Offset += 3;
+
+  // Look for a terminating '"' in the current file buffer.
+  // Make no effort to format code within an interpolated or verbatim string.
+  for (; Offset != Lex->getBuffer().end(); ++Offset) {
+    if (Offset[0] == '"') {
+      // "" within a verbatim string is an escaped double quote: skip it.
+      if (Offset + 1 < Lex->getBuffer().end() && Offset[1] == '"')
+        ++Offset;
+      else
+        break;
+    }
+  }
+
+  // Make no attempt to format code properly if a verbatim string is
+  // unterminated.
+  if (Offset == Lex->getBuffer().end())
+    return;
+
+  StringRef LiteralText(StrBegin, Offset - StrBegin + 1);
+  CSharpStringLiteral->TokenText = LiteralText;
+  CSharpStringLiteral->ColumnWidth = LiteralText.size();
+  size_t LastBreak = LiteralText.rfind('\n');
+  if (LastBreak != StringRef::npos) {
+    CSharpStringLiteral->IsMultiline = true;
+    unsigned StartColumn = 0; // The template tail spans the entire line.
+    CSharpStringLiteral->LastLineColumnWidth = encoding::columnWidthWithTabs(
+        LiteralText.substr(LastBreak + 1, LiteralText.size()), StartColumn,
+        Style.TabWidth, Encoding);
+  }
+
+  Lex->getSourceLocation(Lex->getBuffer().end());
+
+  SourceLocation loc = Offset < Lex->getBuffer().end()
+                           ? Lex->getSourceLocation(Offset + 1)
+                           : SourceMgr.getLocForEndOfFile(ID);
+  resetLexer(SourceMgr.getFileOffset(loc));
+}
+
 void FormatTokenLexer::handleTemplateStrings() {
   FormatToken *BacktickToken = Tokens.back();
 
   if (BacktickToken->is(tok::l_brace)) {
     StateStack.push(LexerState::NORMAL);
     return;
   }
   if (BacktickToken->is(tok::r_brace)) {
     if (StateStack.size() == 1)

clang/unittests/Format/FormatTestCSharp.cpp

TEST_F(FormatTestCSharp, CSharpSpaceAfterCStyleCast) {
 
   Style.SpaceAfterCStyleCast = true;
   verifyFormat("(int) x / y;", Style);
 }
 
 TEST_F(FormatTestCSharp, CSharpEscapedQuotesInVerbatimStrings) {
   FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
 
-  verifyFormat(R"(string str = @"""")", Style);
-  verifyFormat(R"(string str = @"""Hello world""")", Style);
-  verifyFormat(R"(string str = $@"""Hello {friend}""")", Style);
+  verifyFormat(R"(string str = @"""";)", Style);
+  verifyFormat(R"(string str = @"""Hello world""";)", Style);
+  verifyFormat(R"(string str = $@"""Hello {friend}""";)", Style);
 }
 
 TEST_F(FormatTestCSharp, CSharpQuotesInInterpolatedStrings) {
   FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
 
   verifyFormat(R"(string str1 = $"{null ?? "null"}";)", Style);
   verifyFormat(R"(string str2 = $"{{{braceCount} braces";)", Style);
   verifyFormat(R"(string str3 = $"{braceCount}}} braces";)", Style);
 }
 
+TEST_F(FormatTestCSharp, CSharpNewlinesInVerbatimStrings) {
+  FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
+
+  verifyFormat(R"(string s1 = $@"some code:
+  class {className} {{
+    {className}() {{}}
+  }}";)", Style);
+}
+
 } // namespace format
 } // end namespace clang