[lldb] Fix that trailing backslashes in source lines break the Clang highlighter

[teemperor]

Summary

Clang's raw Lexer doesn't produce any tokens for trailing backslashes in a line. This doesn't work with
LLDB's Clang highlighter which builds the source code to display from the list of tokens the Lexer returns.
This causes that lines with trailing backslashes are lacking the backslash and the following newline when
rendering source code in LLDB.

This patch removes the trailing newline from the current line we are highlighting. This way Clang doesn't
drop the backslash token and we just restore the newline after tokenising.

Fixes rdar://57091487

[labath]

lldb/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp

150

technically, this trim might remove additional spurious carriage return characters that will not be restored correctly afterwards. What I'd do is replace endswith with consume_back which will also remove the suffix while testing for it's presence.

[teemperor]

• Moved to consume_back

[labath]

lldb/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp

149

= line_had_lf && consume_back(\r)

[JDevlieghere]

LGTM with Pavel's comment. Thanks for the quick turnaround on this!

[teemperor]

• Turns out \r is also a new line that can be escaped, so that is now also handled. Also moved to a variable storing the line ending instead of three bools.

lldb/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp

void ClangHighlighter::Highlight(const HighlightStyle &options,
                                  llvm::StringRef previous_lines,
                                  Stream &result) const {
   using namespace clang;
 
   FileSystemOptions file_opts;
   FileManager file_mgr(file_opts,
                        FileSystem::Instance().GetVirtualFileSystem());
 
+  // The line might end in a backslash which would cause Clang to drop the
+  // backslash and the terminating new line. This makes sense when parsing C++,
+  // but when highlighting we care about preserving the backslash/newline. To
+  // not lose this information we remove the new line here so that Clang knows
+  // this is just a single line we are highlighting. We add back the newline
+  // after tokenizing.
+  llvm::StringRef line_ending = "";
+  // There are a few legal line endings Clang recognizes and we need to
+  // temporarily remove from the string.
+  if (line.consume_back("\r\n"))
+    line_ending = "\r\n";
+  else if (line.consume_back("\n"))
+    line_ending = "\n";
+  else if (line.consume_back("\r"))
+    line_ending = "\r";
+
   unsigned line_number = previous_lines.count('\n') + 1U;
 
   // Let's build the actual source code Clang needs and setup some utility
   // objects.
   std::string full_source = previous_lines.str() + line.str();
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> diag_ids(new DiagnosticIDs());
   llvm::IntrusiveRefCntPtr<DiagnosticOptions> diags_opts(
       new DiagnosticOptions());
while (!exit) {
 
     // See how we are supposed to highlight this token.
     HighlightStyle::ColorStyle color =
         determineClangStyle(*this, token, tok_str, options, in_pp_directive);
 
     color.Apply(result, to_print);
   }
 
+  // Add the line ending we trimmed before tokenizing.
+  result << line_ending;
+
   // If we went over the whole file but couldn't find our own file, then
   // somehow our setup was wrong. When we're in release mode we just give the
   // user the normal line and pretend we don't know how to highlight it. In
   // debug mode we bail out with an assert as this should never happen.
   if (!found_user_line) {
     result << line;
     assert(false && "We couldn't find the user line in the input file?");
   }
 }

lldb/unittests/Language/Highlighting/HighlighterTest.cpp

 TEST_F(HighlighterTest, ClangPPDirectives) {
   HighlightStyle s;
   s.pp_directive.Set("<pp>", "</pp>");
 
   EXPECT_EQ("<pp>#</pp><pp>include</pp><pp> </pp><pp>\"foo\"</pp><pp> </pp>//c",
             highlightC("#include \"foo\" //c", s));
 }
 
+TEST_F(HighlighterTest, ClangPreserveNewLine) {
+  HighlightStyle s;
+  s.comment.Set("<cc>", "</cc>");
+
+  EXPECT_EQ("<cc>//</cc>\n", highlightC("//\n", s));
+}
+
+TEST_F(HighlighterTest, ClangTrailingBackslashBeforeNewline) {
+  HighlightStyle s;
+
+  EXPECT_EQ("\\\n", highlightC("\\\n", s));
+  EXPECT_EQ("\\\r\n", highlightC("\\\r\n", s));
+
+  EXPECT_EQ("#define a \\\n", highlightC("#define a \\\n", s));
+  EXPECT_EQ("#define a \\\r\n", highlightC("#define a \\\r\n", s));
+  EXPECT_EQ("#define a \\\r", highlightC("#define a \\\r", s));
+}
+
+TEST_F(HighlighterTest, ClangTrailingBackslashWithWhitespace) {
+  HighlightStyle s;
+
+  EXPECT_EQ("\\  \n", highlightC("\\  \n", s));
+  EXPECT_EQ("\\ \t\n", highlightC("\\ \t\n", s));
+  EXPECT_EQ("\\ \n", highlightC("\\ \n", s));
+  EXPECT_EQ("\\\t\n", highlightC("\\\t\n", s));
+
+  EXPECT_EQ("#define a \\  \n", highlightC("#define a \\  \n", s));
+  EXPECT_EQ("#define a \\ \t\n", highlightC("#define a \\ \t\n", s));
+  EXPECT_EQ("#define a \\ \n", highlightC("#define a \\ \n", s));
+  EXPECT_EQ("#define a \\\t\n", highlightC("#define a \\\t\n", s));
+}
+
+TEST_F(HighlighterTest, ClangTrailingBackslashMissingNewLine) {
+  HighlightStyle s;
+  EXPECT_EQ("\\", highlightC("\\", s));
+  EXPECT_EQ("#define a\\", highlightC("#define a\\", s));
+}
+
 TEST_F(HighlighterTest, ClangComments) {
   HighlightStyle s;
   s.comment.Set("<cc>", "</cc>");
 
   EXPECT_EQ(" <cc>/*com */</cc> <cc>// com /*n*/</cc>",
             highlightC(" /*com */ // com /*n*/", s));
 }