Move the column marking functionality to the Highlighter framework

[teemperor]

Summary

The syntax highlighting feature so far is mutually exclusive with the lldb feature
that marks the current column in the line by underlining it via an ANSI color code.
Meaning that if you enable one, the other is automatically disabled by LLDB.

This was caused by the fact that both features inserted color codes into the the
source code and were likely to interfere with each other (which would result
in a broken source code printout to the user).

This patch moves the cursor code into the highlighting framework, which provides
the same feature to the user in normal non-C source code. For any source code
that is highlighted by Clang, we now also have cursor marking for the whole token
that is under the current source location. E.g., before we underlined only the '!' in the
expression '1 != 2', but now the whole token '!=' is underlined. The same for function
calls and so on. Below you can see two examples where we before only underlined
the first character of the token, but now underline the whole token.

It also simplifies the DisplaySourceLines method in the SourceManager as most of
the code in there was essentially just for getting this column marker to work as
a FormatEntity.

[aprantl]

Thank you, this looks great!

lldb/trunk/include/lldb/Core/Debugger.h

public:
   bool GetUseColor() const;
 
   bool SetUseColor(bool use_color);
 
   bool GetHighlightSource() const;
 
   lldb::StopShowColumn GetStopShowColumn() const;
 
-  const FormatEntity::Entry *GetStopShowColumnAnsiPrefix() const;
+  llvm::StringRef GetStopShowColumnAnsiPrefix() const;
 
-  const FormatEntity::Entry *GetStopShowColumnAnsiSuffix() const;
+  llvm::StringRef GetStopShowColumnAnsiSuffix() const;
 
   uint32_t GetStopSourceLineCount(bool before) const;
 
   StopDisassemblyType GetStopDisassemblyDisplay() const;
 
   uint32_t GetDisassemblyLineCount() const;
 
   bool GetAutoOneLineSummaries() const;

lldb/trunk/include/lldb/Core/Highlighter.h

public:
     void Apply(Stream &s, llvm::StringRef value) const;
 
     /// Sets the prefix and suffix strings.
     /// @param prefix
     /// @param suffix
     void Set(llvm::StringRef prefix, llvm::StringRef suffix);
   };
 
+  /// The style for the token which is below the cursor of the user. Note that
+  /// this style is overwritten by the SourceManager with the values of
+  /// stop-show-column-ansi-prefix/stop-show-column-ansi-suffix.
+  ColorStyle selected;
+
   /// Matches identifiers to variable or functions.
   ColorStyle identifier;
   /// Matches any string or character literals in the language: "foo" or 'f'
   ColorStyle string_literal;
   /// Matches scalar value literals like '42' or '0.1'.
   ColorStyle scalar_literal;
   /// Matches all reserved keywords in the language.
   ColorStyle keyword;
public:
 
   /// Returns a human readable name for the selected highlighter.
   virtual llvm::StringRef GetName() const = 0;
 
   /// Highlights the given line
   /// \param options
   /// \param line
   ///     The user supplied line that needs to be highlighted.
+  /// \param cursor_pos
+  ///     The cursor position of the user in this line, starting at 0 (which
+  ///     means the cursor is on the first character in 'line').
   /// \param previous_lines
   ///     Any previous lines the user has written which we should only use
   ///     for getting the context of the Highlighting right.
   /// \param s
   ///     The stream to which the highlighted version of the user string should
   ///     be written.
   virtual void Highlight(const HighlightStyle &options, llvm::StringRef line,
+                         llvm::Optional<size_t> cursor_pos,
                          llvm::StringRef previous_lines, Stream &s) const = 0;
 
   /// Utility method for calling Highlight without a stream.
   std::string Highlight(const HighlightStyle &options, llvm::StringRef line,
+                        llvm::Optional<size_t> cursor_pos,
                         llvm::StringRef previous_lines = "") const;
 };
 
-/// A default highlighter that does nothing. Used as a fallback.
-class NoHighlighter : public Highlighter {
+/// A default highlighter that only highlights the user cursor, but doesn't
+/// do any other highlighting.
+class DefaultHighlighter : public Highlighter {
 public:
   llvm::StringRef GetName() const override { return "none"; }
 
   void Highlight(const HighlightStyle &options, llvm::StringRef line,
+                 llvm::Optional<size_t> cursor_pos,
                  llvm::StringRef previous_lines, Stream &s) const override;
 };
 
 /// Manages the available highlighters.
 class HighlighterManager {
-  NoHighlighter m_no_highlighter;
+  DefaultHighlighter m_default;
 
 public:
   /// Queries all known highlighter for one that can highlight some source code.
   /// \param language_type
   ///     The language type that the caller thinks the source code was given in.
   /// \param path
   ///     The path to the file the source code is from. Used as a fallback when
   ///     the user can't provide a language.
   /// \return
   ///     The highlighter that wants to highlight the source code. Could be an
   ///     empty highlighter that does nothing.
   const Highlighter &getHighlighterFor(lldb::LanguageType language_type,
                                        llvm::StringRef path) const;
+  const Highlighter &getDefaultHighlighter() const { return m_default; }
 };
 
 } // namespace lldb_private
 
 #endif // liblldb_Highlighter_h_

lldb/trunk/include/lldb/Core/SourceManager.h

#ifndef SWIG
 
   public:
     File(const FileSpec &file_spec, Target *target);
     File(const FileSpec &file_spec, lldb::DebuggerSP debugger_sp);
     ~File() = default;
 
     void UpdateIfNeeded();
 
-    size_t DisplaySourceLines(uint32_t line, uint32_t column,
+    size_t DisplaySourceLines(uint32_t line, llvm::Optional<size_t> column,
                               uint32_t context_before, uint32_t context_after,
                               Stream *s);
     void FindLinesMatchingRegex(RegularExpression &regex, uint32_t start_line,
                                 uint32_t end_line,
                                 std::vector<uint32_t> &match_lines);
 
     bool GetLine(uint32_t line_no, std::string &buffer);
 

lldb/trunk/packages/Python/lldbsuite/test/source-manager/TestSourceManager.py

def test_display_source_python_dumb_terminal(self):
         use_color = False
         self.do_display_source_python_api(use_color, r"\s+\^")
 
     @add_test_categories(['pyapi'])
     def test_display_source_python_ansi_terminal(self):
         """Test display of source using the SBSourceManager API, using a
         dumb terminal and thus no color support (the default)."""
         use_color = True
-        underline_regex = ansi_underline_surround_regex(r".")
+        underline_regex = ansi_underline_surround_regex(r"printf")
         self.do_display_source_python_api(use_color, underline_regex)
 
     @add_test_categories(['pyapi'])
     def test_display_source_python_ansi_terminal_syntax_highlighting(self):
         """Test display of source using the SBSourceManager API and check for
         the syntax highlighted output"""
         use_color = True
         syntax_highlighting = True;
 
         # Just pick 'int' as something that should be colored.
         color_regex = ansi_color_surround_regex("int")
         self.do_display_source_python_api(use_color, color_regex, syntax_highlighting)
 
         # Same for 'char'.
         color_regex = ansi_color_surround_regex("char")
         self.do_display_source_python_api(use_color, color_regex, syntax_highlighting)
 
         # Test that we didn't color unrelated identifiers.
-        self.do_display_source_python_api(use_color, r" printf\(", syntax_highlighting)
+        self.do_display_source_python_api(use_color, r" main\(", syntax_highlighting)
+        self.do_display_source_python_api(use_color, r"\);", syntax_highlighting)
 
     def test_move_and_then_display_source(self):
         """Test that target.source-map settings work by moving main.c to hidden/main.c."""
         self.build()
         exe = self.getBuildArtifact("a.out")
         self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
 
         # Move main.c to hidden/main.c.

lldb/trunk/source/Core/Debugger.cpp

 // offset-no-padding}>}:
 
 // lldb's original format for disassembly would look like this format string -
 // {${function.initial-function}{${module.file.basename}`}{${function.name-
 // without-
 // args}}:\n}{${function.changed}\n{${module.file.basename}`}{${function.name-
 // without-args}}:\n}{${current-pc-arrow} }{${addr-file-or-load}}:
 
-#define DEFAULT_STOP_SHOW_COLUMN_ANSI_PREFIX "${ansi.underline}"
-#define DEFAULT_STOP_SHOW_COLUMN_ANSI_SUFFIX "${ansi.normal}"
-
 static OptionEnumValueElement s_stop_show_column_values[] = {
     {eStopShowColumnAnsiOrCaret, "ansi-or-caret",
      "Highlight the stop column with ANSI terminal codes when color/ANSI mode "
      "is enabled; otherwise, fall back to using a text-only caret (^) as if "
      "\"caret-only\" mode was selected."},
     {eStopShowColumnAnsi, "ansi", "Highlight the stop column with ANSI "
                                   "terminal codes when running LLDB with "
                                   "color/ANSI enabled."},
static PropertyDefinition g_properties[] = {
      "The number of sources lines to display that come before the "
      "current source line when displaying a stopped context."},
     {"highlight-source", OptionValue::eTypeBoolean, true, true, nullptr,
      nullptr, "If true, LLDB will highlight the displayed source code."},
     {"stop-show-column", OptionValue::eTypeEnum, false,
      eStopShowColumnAnsiOrCaret, nullptr, s_stop_show_column_values,
      "If true, LLDB will use the column information from the debug info to "
      "mark the current position when displaying a stopped context."},
-    {"stop-show-column-ansi-prefix", OptionValue::eTypeFormatEntity, true, 0,
-     DEFAULT_STOP_SHOW_COLUMN_ANSI_PREFIX, nullptr,
+    {"stop-show-column-ansi-prefix", OptionValue::eTypeString, true, 0,
+     "${ansi.underline}", nullptr,
      "When displaying the column marker in a color-enabled (i.e. ANSI) "
      "terminal, use the ANSI terminal code specified in this format at the "
      "immediately before the column to be marked."},
-    {"stop-show-column-ansi-suffix", OptionValue::eTypeFormatEntity, true, 0,
-     DEFAULT_STOP_SHOW_COLUMN_ANSI_SUFFIX, nullptr,
+    {"stop-show-column-ansi-suffix", OptionValue::eTypeString, true, 0,
+     "${ansi.normal}", nullptr,
      "When displaying the column marker in a color-enabled (i.e. ANSI) "
      "terminal, use the ANSI terminal code specified in this format "
      "immediately after the column to be marked."},
     {"term-width", OptionValue::eTypeSInt64, true, 80, nullptr, nullptr,
      "The maximum number of columns to use for displaying text."},
     {"thread-format", OptionValue::eTypeFormatEntity, true, 0,
      DEFAULT_THREAD_FORMAT, nullptr,
      "The default thread format string to use "

 }
 
 StopShowColumn Debugger::GetStopShowColumn() const {
   const uint32_t idx = ePropertyStopShowColumn;
   return (lldb::StopShowColumn)m_collection_sp->GetPropertyAtIndexAsEnumeration(
       nullptr, idx, g_properties[idx].default_uint_value);
 }
 
-const FormatEntity::Entry *Debugger::GetStopShowColumnAnsiPrefix() const {
+llvm::StringRef Debugger::GetStopShowColumnAnsiPrefix() const {
   const uint32_t idx = ePropertyStopShowColumnAnsiPrefix;
-  return m_collection_sp->GetPropertyAtIndexAsFormatEntity(nullptr, idx);
+  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
 }
 
-const FormatEntity::Entry *Debugger::GetStopShowColumnAnsiSuffix() const {
+llvm::StringRef Debugger::GetStopShowColumnAnsiSuffix() const {
   const uint32_t idx = ePropertyStopShowColumnAnsiSuffix;
-  return m_collection_sp->GetPropertyAtIndexAsFormatEntity(nullptr, idx);
+  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
 }
 
 uint32_t Debugger::GetStopSourceLineCount(bool before) const {
   const uint32_t idx =
       before ? ePropertyStopLineCountBefore : ePropertyStopLineCountAfter;
   return m_collection_sp->GetPropertyAtIndexAsSInt64(
       nullptr, idx, g_properties[idx].default_uint_value);
 }

lldb/trunk/source/Core/Highlighter.cpp

 }
 
 void HighlightStyle::ColorStyle::Set(llvm::StringRef prefix,
                                      llvm::StringRef suffix) {
   m_prefix = lldb_utility::ansi::FormatAnsiTerminalCodes(prefix);
   m_suffix = lldb_utility::ansi::FormatAnsiTerminalCodes(suffix);
 }
 
-void NoHighlighter::Highlight(const HighlightStyle &options,
+void DefaultHighlighter::Highlight(const HighlightStyle &options,
                                    llvm::StringRef line,
-                              llvm::StringRef previous_lines, Stream &s) const {
-  // We just forward the input to the output and do no highlighting.
+                                   llvm::Optional<size_t> cursor_pos,
+                                   llvm::StringRef previous_lines,
+                                   Stream &s) const {
+  // If we don't have a valid cursor, then we just print the line as-is.
+  if (!cursor_pos || *cursor_pos >= line.size()) {
     s << line;
+    return;
+  }
+
+  // If we have a valid cursor, we have to apply the 'selected' style around
+  // the character below the cursor.
+
+  // Split the line around the character which is below the cursor.
+  size_t column = *cursor_pos;
+  // Print the characters before the cursor.
+  s << line.substr(0, column);
+  // Print the selected character with the defined color codes.
+  options.selected.Apply(s, line.substr(column, 1));
+  // Print the rest of the line.
+  s << line.substr(column + 1U);
 }
 
 static HighlightStyle::ColorStyle GetColor(const char *c) {
   return HighlightStyle::ColorStyle(c, "${ansi.normal}");
 }
 
 HighlightStyle HighlightStyle::MakeVimStyle() {
   HighlightStyle result;
   result.comment = GetColor("${ansi.fg.purple}");
   result.scalar_literal = GetColor("${ansi.fg.red}");
   result.keyword = GetColor("${ansi.fg.green}");
   return result;
 }
 
 const Highlighter &
 HighlighterManager::getHighlighterFor(lldb::LanguageType language_type,
                                       llvm::StringRef path) const {
   Language *language = lldb_private::Language::FindPlugin(language_type, path);
   if (language && language->GetHighlighter())
     return *language->GetHighlighter();
-  return m_no_highlighter;
+  return m_default;
 }
 
 std::string Highlighter::Highlight(const HighlightStyle &options,
                                    llvm::StringRef line,
+                                   llvm::Optional<size_t> cursor_pos,
                                    llvm::StringRef previous_lines) const {
   StreamString s;
-  Highlight(options, line, previous_lines, s);
+  Highlight(options, line, cursor_pos, previous_lines, s);
   s.Flush();
   return s.GetString().str();
 }

lldb/trunk/source/Core/SourceManager.cpp

static bool should_show_stop_column_with_ansi(DebuggerSP debugger_sp) {
   if (!debugger_sp)
     return false;
 
   // We don't use ANSI stop column formatting if the debugger doesn't think it
   // should be using color.
   if (!debugger_sp->GetUseColor())
     return false;
 
-  // Don't use terminal attributes when we have highlighting enabled. This
-  // can mess up the command line.
-  if (debugger_sp->GetHighlightSource())
-    return false;
-
   // We only use ANSI stop column formatting if we're either supposed to show
   // ANSI where available (which we know we have when we get to this point), or
   // if we're only supposed to use ANSI.
   const auto value = debugger_sp->GetStopShowColumn();
   return ((value == eStopShowColumnAnsiOrCaret) ||
           (value == eStopShowColumnAnsi));
 }
 
for (uint32_t line = start_line; line <= end_line; ++line) {
           ::snprintf(prefix, sizeof(prefix), "[%u] ", bp_count);
         else
           ::snprintf(prefix, sizeof(prefix), "    ");
       }
 
       return_value +=
           s->Printf("%s%2.2s %-4u\t", prefix,
                     line == curr_line ? current_line_cstr : "", line);
-      size_t this_line_size = m_last_file_sp->DisplaySourceLines(
-          line, line == curr_line ? column : 0, 0, 0, s);
+
+      // So far we treated column 0 as a special 'no column value', but
+      // DisplaySourceLines starts counting columns from 0 (and no column is
+      // expressed by passing an empty optional).
+      llvm::Optional<size_t> columnToHighlight;
+      if (line == curr_line && column)
+        columnToHighlight = column - 1;
+
+      size_t this_line_size =
+          m_last_file_sp->DisplaySourceLines(line, columnToHighlight, 0, 0, s);
       if (column != 0 && line == curr_line &&
           should_show_stop_column_with_caret(m_debugger_wp.lock())) {
         // Display caret cursor.
         std::string src_line;
         m_last_file_sp->GetLine(line, src_line);
         return_value += s->Printf("    \t");
         // Insert a space for every non-tab character in the source line.
         for (size_t i = 0; i + 1 < column && i < src_line.length(); ++i)
void SourceManager::File::UpdateIfNeeded() {
   if (curr_mod_time != llvm::sys::TimePoint<>() &&
       m_mod_time != curr_mod_time) {
     m_mod_time = curr_mod_time;
     m_data_sp = DataBufferLLVM::CreateFromPath(m_file_spec.GetPath());
     m_offsets.clear();
   }
 }
 
-size_t SourceManager::File::DisplaySourceLines(uint32_t line, uint32_t column,
+size_t SourceManager::File::DisplaySourceLines(uint32_t line,
+                                               llvm::Optional<size_t> column,
                                                uint32_t context_before,
                                                uint32_t context_after,
                                                Stream *s) {
   // Nothing to write if there's no stream.
   if (!s)
     return 0;
 
   // Sanity check m_data_sp before proceeding.
   if (!m_data_sp)
     return 0;
 
   size_t bytes_written = s->GetWrittenBytes();
 
-  std::string previous_content;
+  auto debugger_sp = m_debugger_wp.lock();
+
+  HighlightStyle style;
+  // Use the default Vim style if source highlighting is enabled.
+  if (should_highlight_source(debugger_sp))
+    style = HighlightStyle::MakeVimStyle();
+
+  // If we should mark the stop column with color codes, then copy the prefix
+  // and suffix to our color style.
+  if (should_show_stop_column_with_ansi(debugger_sp))
+    style.selected.Set(debugger_sp->GetStopShowColumnAnsiPrefix(),
+                       debugger_sp->GetStopShowColumnAnsiSuffix());
 
-  HighlightStyle style = HighlightStyle::MakeVimStyle();
   HighlighterManager mgr;
   std::string path = GetFileSpec().GetPath(/*denormalize*/ false);
   // FIXME: Find a way to get the definitive language this file was written in
   // and pass it to the highlighter.
-  auto &highlighter =
-      mgr.getHighlighterFor(lldb::LanguageType::eLanguageTypeUnknown, path);
+  const auto &h = mgr.getHighlighterFor(lldb::eLanguageTypeUnknown, path);
 
   const uint32_t start_line =
       line <= context_before ? 1 : line - context_before;
   const uint32_t start_line_offset = GetLineOffset(start_line);
   if (start_line_offset != UINT32_MAX) {
     const uint32_t end_line = line + context_after;
     uint32_t end_line_offset = GetLineOffset(end_line + 1);
     if (end_line_offset == UINT32_MAX)
       end_line_offset = m_data_sp->GetByteSize();
 
     assert(start_line_offset <= end_line_offset);
     if (start_line_offset < end_line_offset) {
       size_t count = end_line_offset - start_line_offset;
       const uint8_t *cstr = m_data_sp->GetBytes() + start_line_offset;
 
       auto ref = llvm::StringRef(reinterpret_cast<const char *>(cstr), count);
-      bool displayed_line = false;
-
-      auto debugger_sp = m_debugger_wp.lock();
-      if (should_highlight_source(debugger_sp)) {
-        highlighter.Highlight(style, ref, previous_content, *s);
-        displayed_line = true;
-        // Add the new line to the previous lines.
-        previous_content += ref.str();
-      }
-
-      if (!displayed_line && column && (column < count)) {
-        if (should_show_stop_column_with_ansi(debugger_sp) && debugger_sp) {
-          // Check if we have any ANSI codes with which to mark this column. If
-          // not, no need to do this work.
-          auto ansi_prefix_entry = debugger_sp->GetStopShowColumnAnsiPrefix();
-          auto ansi_suffix_entry = debugger_sp->GetStopShowColumnAnsiSuffix();
-
-          // We only bother breaking up the line to format the marked column if
-          // there is any marking specified on both sides of the marked column.
-          // In ANSI-terminal-sequence land, there must be a post if there is a
-          // pre format, and vice versa.
-          if (ansi_prefix_entry && ansi_suffix_entry) {
-            // Mark the current column with the desired escape sequence for
-            // formatting the column (e.g. underline, inverse, etc.)
-
-            // First print the part before the column to mark.
-            s->Write(cstr, column - 1);
-
-            // Write the pre escape sequence.
-            const SymbolContext *sc = nullptr;
-            const ExecutionContext *exe_ctx = nullptr;
-            const Address addr = LLDB_INVALID_ADDRESS;
-            ValueObject *valobj = nullptr;
-            const bool function_changed = false;
-            const bool initial_function = false;
-
-            FormatEntity::Format(*ansi_prefix_entry, *s, sc, exe_ctx, &addr,
-                                 valobj, function_changed, initial_function);
-
-            s->Write(cstr + column - 1, 1);
-
-            // Write the post escape sequence.
-            FormatEntity::Format(*ansi_suffix_entry, *s, sc, exe_ctx, &addr,
-                                 valobj, function_changed, initial_function);
-
-            // And finish up with the rest of the line.
-            s->Write(cstr + column, count - column);
-
-            // Keep track of the fact that we just wrote the line.
-            displayed_line = true;
-          }
-        }
-      }
 
-      // If we didn't end up displaying the line with ANSI codes for whatever
-      // reason, display it now sans codes.
-      if (!displayed_line)
-        s->PutCString(ref);
+      h.Highlight(style, ref, column, "", *s);
 
       // Ensure we get an end of line character one way or another.
       if (!is_newline_char(ref.back()))
         s->EOL();
     }
   }
   return s->GetWrittenBytes() - bytes_written;
 }

lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.h

 class ClangHighlighter : public Highlighter {
   llvm::StringSet<> keywords;
 
 public:
   ClangHighlighter();
   llvm::StringRef GetName() const override { return "clang"; }
 
   void Highlight(const HighlightStyle &options, llvm::StringRef line,
+                 llvm::Optional<size_t> cursor_pos,
                  llvm::StringRef previous_lines, Stream &s) const override;
 
   /// Returns true if the given string represents a keywords in any Clang
   /// supported language.
   bool isKeyword(llvm::StringRef token) const;
 };
 
 } // namespace lldb_private
 
 #endif // liblldb_ClangHighlighter_h_

lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp

else
     default:
       break;
     }
   return HighlightStyle::ColorStyle();
 }
 
 void ClangHighlighter::Highlight(const HighlightStyle &options,
                                  llvm::StringRef line,
+                                 llvm::Optional<size_t> cursor_pos,
                                  llvm::StringRef previous_lines,
                                  Stream &result) const {
   using namespace clang;
 
   FileSystemOptions file_opts;
   FileManager file_mgr(file_opts);
 
   unsigned line_number = previous_lines.count('\n') + 1U;
void ClangHighlighter::Highlight(const HighlightStyle &options,
   lex.SetKeepWhitespaceMode(true);
 
   // Keeps track if we have entered a PP directive.
   bool in_pp_directive = false;
 
   // True once we actually lexed the user provided line.
   bool found_user_line = false;
 
+  // True if we already highlighted the token under the cursor, false otherwise.
+  bool highlighted_cursor = false;
   Token token;
   bool exit = false;
   while (!exit) {
     // Returns true if this is the last token we get from the lexer.
     exit = lex.LexFromRawLexer(token);
 
     bool invalid = false;
     unsigned current_line_number =
while (!exit) {
 
     // Extract the token string from our source code.
     llvm::StringRef tok_str = line.substr(start, token.getLength());
 
     // If the token is just an empty string, we can skip all the work below.
     if (tok_str.empty())
       continue;
 
+    // If the cursor is inside this token, we have to apply the 'selected'
+    // highlight style before applying the actual token color.
+    llvm::StringRef to_print = tok_str;
+    StreamString storage;
+    auto end = start + token.getLength();
+    if (cursor_pos && end > *cursor_pos && !highlighted_cursor) {
+      highlighted_cursor = true;
+      options.selected.Apply(storage, tok_str);
+      to_print = storage.GetString();
+    }
+
     // See how we are supposed to highlight this token.
     HighlightStyle::ColorStyle color =
         determineClangStyle(*this, token, tok_str, options, in_pp_directive);
 
-    color.Apply(result, tok_str);
+    color.Apply(result, to_print);
   }
 
   // If we went over the whole file but couldn't find our own file, then
   // somehow our setup was wrong. When we're in release mode we just give the
   // user the normal line and pretend we don't know how to highlight it. In
   // debug mode we bail out with an assert as this should never happen.
   if (!found_user_line) {
     result << line;
     assert(false && "We couldn't find the user line in the input file?");
   }
 }

lldb/trunk/unittests/Language/Highlighting/HighlighterTest.cpp

TEST_F(HighlighterTest, FallbackHighlighter) {
   const Highlighter &h =
       mgr.getHighlighterFor(lldb::eLanguageTypePascal83, "foo.pas");
 
   HighlightStyle style;
   style.identifier.Set("[", "]");
   style.semicolons.Set("<", ">");
 
   const char *code = "program Hello;";
-  std::string output = h.Highlight(style, code);
+  std::string output = h.Highlight(style, code, llvm::Optional<size_t>());
 
   EXPECT_STREQ(output.c_str(), code);
 }
 
-TEST_F(HighlighterTest, DefaultHighlighter) {
+static std::string
+highlightDefault(llvm::StringRef code, HighlightStyle style,
+                 llvm::Optional<size_t> cursor = llvm::Optional<size_t>()) {
   HighlighterManager mgr;
-  const Highlighter &h = mgr.getHighlighterFor(lldb::eLanguageTypeC, "main.c");
-
-  HighlightStyle style;
+  return mgr.getDefaultHighlighter().Highlight(style, code, cursor);
+}
 
+TEST_F(HighlighterTest, DefaultHighlighter) {
   const char *code = "int my_main() { return 22; } \n";
-  std::string output = h.Highlight(style, code);
 
-  EXPECT_STREQ(output.c_str(), code);
+  HighlightStyle style;
+  EXPECT_EQ(code, highlightDefault(code, style));
+}
+
+TEST_F(HighlighterTest, DefaultHighlighterWithCursor) {
+  HighlightStyle style;
+  style.selected.Set("<c>", "</c>");
+  EXPECT_EQ("<c>a</c> bc", highlightDefault("a bc", style, 0));
+  EXPECT_EQ("a<c> </c>bc", highlightDefault("a bc", style, 1));
+  EXPECT_EQ("a <c>b</c>c", highlightDefault("a bc", style, 2));
+  EXPECT_EQ("a b<c>c</c>", highlightDefault("a bc", style, 3));
 }
 
+TEST_F(HighlighterTest, DefaultHighlighterWithCursorOutOfBounds) {
+  HighlightStyle style;
+  style.selected.Set("<c>", "</c>");
+  EXPECT_EQ("a bc", highlightDefault("a bc", style, 4));
+}
 //------------------------------------------------------------------------------
 // Tests highlighting with the Clang highlighter.
 //------------------------------------------------------------------------------
 
-static std::string highlightC(llvm::StringRef code, HighlightStyle style) {
+static std::string
+highlightC(llvm::StringRef code, HighlightStyle style,
+           llvm::Optional<size_t> cursor = llvm::Optional<size_t>()) {
   HighlighterManager mgr;
   const Highlighter &h = mgr.getHighlighterFor(lldb::eLanguageTypeC, "main.c");
-  return h.Highlight(style, code);
+  return h.Highlight(style, code, cursor);
 }
 
 TEST_F(HighlighterTest, ClangEmptyInput) {
   HighlightStyle s;
   EXPECT_EQ("", highlightC("", s));
 }
 
 TEST_F(HighlighterTest, ClangScalarLiterals) {

 
 TEST_F(HighlighterTest, ClangIdentifiers) {
   HighlightStyle s;
   s.identifier.Set("<id>", "</id>");
 
   EXPECT_EQ(" <id>foo</id> <id>c</id> = <id>bar</id>(); return 1;",
             highlightC(" foo c = bar(); return 1;", s));
 }
+
+TEST_F(HighlighterTest, ClangCursorPos) {
+  HighlightStyle s;
+  s.selected.Set("<c>", "</c>");
+
+  EXPECT_EQ("<c> </c>foo c = bar(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 0));
+  EXPECT_EQ(" <c>foo</c> c = bar(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 1));
+  EXPECT_EQ(" <c>foo</c> c = bar(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 2));
+  EXPECT_EQ(" <c>foo</c> c = bar(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 3));
+  EXPECT_EQ(" foo<c> </c>c = bar(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 4));
+  EXPECT_EQ(" foo <c>c</c> = bar(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 5));
+}
+
+TEST_F(HighlighterTest, ClangCursorPosEndOfLine) {
+  HighlightStyle s;
+  s.selected.Set("<c>", "</c>");
+
+  EXPECT_EQ("f", highlightC("f", s, 1));
+}
+
+TEST_F(HighlighterTest, ClangCursorOutOfBounds) {
+  HighlightStyle s;
+  s.selected.Set("<c>", "</c>");
+  EXPECT_EQ("f", highlightC("f", s, 2));
+  EXPECT_EQ("f", highlightC("f", s, 3));
+  EXPECT_EQ("f", highlightC("f", s, 4));
+}
+
+TEST_F(HighlighterTest, ClangCursorPosBeforeOtherToken) {
+  HighlightStyle s;
+  s.selected.Set("<c>", "</c>");
+  s.identifier.Set("<id>", "</id>");
+
+  EXPECT_EQ("<c> </c><id>foo</id> <id>c</id> = <id>bar</id>(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 0));
+}
+
+TEST_F(HighlighterTest, ClangCursorPosAfterOtherToken) {
+  HighlightStyle s;
+  s.selected.Set("<c>", "</c>");
+  s.identifier.Set("<id>", "</id>");
+
+  EXPECT_EQ(" <id>foo</id><c> </c><id>c</id> = <id>bar</id>(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 4));
+}
+
+TEST_F(HighlighterTest, ClangCursorPosInOtherToken) {
+  HighlightStyle s;
+  s.selected.Set("<c>", "</c>");
+  s.identifier.Set("<id>", "</id>");
+
+  EXPECT_EQ(" <id><c>foo</c></id> <id>c</id> = <id>bar</id>(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 1));
+  EXPECT_EQ(" <id><c>foo</c></id> <id>c</id> = <id>bar</id>(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 2));
+  EXPECT_EQ(" <id><c>foo</c></id> <id>c</id> = <id>bar</id>(); return 1;",
+            highlightC(" foo c = bar(); return 1;", s, 3));
+}