Index: lldb/trunk/include/lldb/Core/Debugger.h =================================================================== --- lldb/trunk/include/lldb/Core/Debugger.h +++ lldb/trunk/include/lldb/Core/Debugger.h @@ -272,6 +272,8 @@ bool SetUseColor(bool use_color); + bool GetHighlightSource() const; + lldb::StopShowColumn GetStopShowColumn() const; const FormatEntity::Entry *GetStopShowColumnAnsiPrefix() const; Index: lldb/trunk/include/lldb/Core/Highlighter.h =================================================================== --- lldb/trunk/include/lldb/Core/Highlighter.h +++ lldb/trunk/include/lldb/Core/Highlighter.h @@ -0,0 +1,159 @@ +//===-- Highlighter.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef liblldb_Highlighter_h_ +#define liblldb_Highlighter_h_ + +#include +#include + +#include "lldb/Utility/Stream.h" +#include "lldb/lldb-enumerations.h" +#include "llvm/ADT/StringRef.h" + +namespace lldb_private { + +//---------------------------------------------------------------------- +/// Represents style that the highlighter should apply to the given source code. +/// Stores information about how every kind of token should be annotated. +//---------------------------------------------------------------------- +struct HighlightStyle { + + //---------------------------------------------------------------------- + /// A pair of strings that should be placed around a certain token. Usually + /// stores color codes in these strings (the suffix string is often used for + /// resetting the terminal attributes back to normal). + //---------------------------------------------------------------------- + class ColorStyle { + std::string m_prefix; + std::string m_suffix; + + public: + ColorStyle() = default; + ColorStyle(llvm::StringRef prefix, llvm::StringRef suffix) { + Set(prefix, suffix); + } + + /// Applies this style to the given value. + /// \param s + /// The stream to which the result should be appended. + /// \param value + /// The value that we should place our strings around. + /// \return + /// The number of bytes that have been written to the given stream. + std::size_t Apply(Stream &s, llvm::StringRef value) const; + + /// Sets the prefix and suffix strings. + /// @param prefix + /// @param suffix + void Set(llvm::StringRef prefix, llvm::StringRef suffix); + }; + + /// Matches identifiers to variable or functions. + ColorStyle identifier; + /// Matches any string or character literals in the language: "foo" or 'f' + ColorStyle string_literal; + /// Matches scalar value literals like '42' or '0.1'. + ColorStyle scalar_literal; + /// Matches all reserved keywords in the language. + ColorStyle keyword; + /// Matches any comments in the language. + ColorStyle comment; + /// Matches commas: ',' + ColorStyle comma; + /// Matches one colon: ':' + ColorStyle colon; + /// Matches any semicolon: ';' + ColorStyle semicolons; + /// Matches operators like '+', '-', '%', '&', '=' + ColorStyle operators; + + /// Matches '{' or '}' + ColorStyle braces; + /// Matches '[' or ']' + ColorStyle square_brackets; + /// Matches '(' or ')' + ColorStyle parentheses; + + //----------------------------------------------------------------------- + // C language specific options + //----------------------------------------------------------------------- + + /// Matches directives to a preprocessor (if the language has any). + ColorStyle pp_directive; + + /// Returns a HighlightStyle that is based on vim's default highlight style. + static HighlightStyle MakeVimStyle(); +}; + +//---------------------------------------------------------------------- +/// Annotates source code with color attributes. +//---------------------------------------------------------------------- +class Highlighter { +public: + Highlighter() = default; + virtual ~Highlighter() = default; + DISALLOW_COPY_AND_ASSIGN(Highlighter); + + /// Returns a human readable name for the selected highlighter. + virtual llvm::StringRef GetName() const = 0; + + /// Highlights the given line + /// \param options + /// \param line + /// The user supplied line that needs to be highlighted. + /// \param previous_lines + /// Any previous lines the user has written which we should only use + /// for getting the context of the Highlighting right. + /// \param s + /// The stream to which the highlighted version of the user string should + /// be written. + /// \return + /// The number of bytes that have been written to the stream. + virtual std::size_t Highlight(const HighlightStyle &options, + llvm::StringRef line, + llvm::StringRef previous_lines, + Stream &s) const = 0; + + /// Utility method for calling Highlight without a stream. + std::string Highlight(const HighlightStyle &options, llvm::StringRef line, + llvm::StringRef previous_lines = "") const; +}; + +/// A default highlighter that does nothing. Used as a fallback. +class NoHighlighter : public Highlighter { +public: + llvm::StringRef GetName() const override { return "none"; } + + std::size_t Highlight(const HighlightStyle &options, llvm::StringRef line, + llvm::StringRef previous_lines, + Stream &s) const override; +}; + +/// Manages the available highlighters. +class HighlighterManager { + NoHighlighter m_no_highlighter; + +public: + /// Queries all known highlighter for one that can highlight some source code. + /// \param language_type + /// The language type that the caller thinks the source code was given in. + /// \param path + /// The path to the file the source code is from. Used as a fallback when + /// the user can't provide a language. + /// \return + /// The highlighter that wants to highlight the source code. Could be an + /// empty highlighter that does nothing. + const Highlighter &getHighlighterFor(lldb::LanguageType language_type, + llvm::StringRef path) const; +}; + +} // namespace lldb_private + +#endif // liblldb_Highlighter_h_ Index: lldb/trunk/include/lldb/Target/Language.h =================================================================== --- lldb/trunk/include/lldb/Target/Language.h +++ lldb/trunk/include/lldb/Target/Language.h @@ -20,6 +20,7 @@ // Other libraries and framework includes // Project includes +#include "lldb/Core/Highlighter.h" #include "lldb/Core/PluginInterface.h" #include "lldb/DataFormatters/DumpValueObjectOptions.h" #include "lldb/DataFormatters/FormatClasses.h" @@ -152,6 +153,13 @@ static Language *FindPlugin(lldb::LanguageType language); + /// Returns the Language associated with the given file path or a nullptr + /// if there is no known language. + static Language *FindPlugin(llvm::StringRef file_path); + + static Language *FindPlugin(lldb::LanguageType language, + llvm::StringRef file_path); + // return false from callback to stop iterating static void ForEach(std::function callback); @@ -159,6 +167,10 @@ virtual bool IsTopLevelFunction(Function &function); + virtual bool IsSourceFile(llvm::StringRef file_path) const = 0; + + virtual const Highlighter *GetHighlighter() const { return nullptr; } + virtual lldb::TypeCategoryImplSP GetFormatters(); virtual HardcodedFormatters::HardcodedFormatFinder GetHardcodedFormats(); Index: lldb/trunk/packages/Python/lldbsuite/test/source-manager/TestSourceManager.py =================================================================== --- lldb/trunk/packages/Python/lldbsuite/test/source-manager/TestSourceManager.py +++ lldb/trunk/packages/Python/lldbsuite/test/source-manager/TestSourceManager.py @@ -22,6 +22,8 @@ # return re.compile(r"\[4m%s\[0m" % inner_regex_text) return "4.+\033\\[4m%s\033\\[0m" % inner_regex_text +def ansi_color_surround_regex(inner_regex_text): + return "\033\\[3[0-7]m%s\033\\[0m" % inner_regex_text class SourceManagerTestCase(TestBase): @@ -47,7 +49,7 @@ # the character column after the initial whitespace. return len(stop_line) - len(stop_line.lstrip()) + 1 - def do_display_source_python_api(self, use_color, column_marker_regex): + def do_display_source_python_api(self, use_color, needle_regex, highlight_source=False): self.build() exe = self.getBuildArtifact("a.out") self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET) @@ -69,6 +71,9 @@ # Setup whether we should use ansi escape sequences, including color # and styles such as underline. self.dbg.SetUseColor(use_color) + # Disable syntax highlighting if needed. + + self.runCmd("settings set highlight-source " + str(highlight_source).lower()) filespec = lldb.SBFileSpec(self.file, False) source_mgr = self.dbg.GetSourceManager() @@ -87,10 +92,10 @@ # => 4 printf("Hello world.\n"); // Set break point at this line. # 5 return 0; # 6 } - self.expect(stream.GetData(), "Source code displayed correctly", + self.expect(stream.GetData(), "Source code displayed correctly:\n" + stream.GetData(), exe=False, patterns=['=> %d.*Hello world' % self.line, - column_marker_regex]) + needle_regex]) # Boundary condition testings for SBStream(). LLDB should not crash! stream.Print(None) @@ -111,6 +116,24 @@ underline_regex = ansi_underline_surround_regex(r".") self.do_display_source_python_api(use_color, underline_regex) + @add_test_categories(['pyapi']) + def test_display_source_python_ansi_terminal_syntax_highlighting(self): + """Test display of source using the SBSourceManager API and check for + the syntax highlighted output""" + use_color = True + syntax_highlighting = True; + + # Just pick 'int' as something that should be colored. + color_regex = ansi_color_surround_regex("int") + self.do_display_source_python_api(use_color, color_regex, syntax_highlighting) + + # Same for 'char'. + color_regex = ansi_color_surround_regex("char") + self.do_display_source_python_api(use_color, color_regex, syntax_highlighting) + + # Test that we didn't color unrelated identifiers. + self.do_display_source_python_api(use_color, r" printf\(", syntax_highlighting) + def test_move_and_then_display_source(self): """Test that target.source-map settings work by moving main.c to hidden/main.c.""" self.build() Index: lldb/trunk/source/Core/CMakeLists.txt =================================================================== --- lldb/trunk/source/Core/CMakeLists.txt +++ lldb/trunk/source/Core/CMakeLists.txt @@ -25,6 +25,7 @@ FileLineResolver.cpp FileSpecList.cpp FormatEntity.cpp + Highlighter.cpp IOHandler.cpp Listener.cpp Mangled.cpp Index: lldb/trunk/source/Core/Debugger.cpp =================================================================== --- lldb/trunk/source/Core/Debugger.cpp +++ lldb/trunk/source/Core/Debugger.cpp @@ -233,6 +233,8 @@ nullptr, "The number of sources lines to display that come before the " "current source line when displaying a stopped context."}, + {"highlight-source", OptionValue::eTypeBoolean, true, true, nullptr, + nullptr, "If true, LLDB will highlight the displayed source code."}, {"stop-show-column", OptionValue::eTypeEnum, false, eStopShowColumnAnsiOrCaret, nullptr, s_stop_show_column_values, "If true, LLDB will use the column information from the debug info to " @@ -296,6 +298,7 @@ ePropertyStopDisassemblyDisplay, ePropertyStopLineCountAfter, ePropertyStopLineCountBefore, + ePropertyHighlightSource, ePropertyStopShowColumn, ePropertyStopShowColumnAnsiPrefix, ePropertyStopShowColumnAnsiSuffix, @@ -470,6 +473,12 @@ return ret; } +bool Debugger::GetHighlightSource() const { + const uint32_t idx = ePropertyHighlightSource; + return m_collection_sp->GetPropertyAtIndexAsBoolean( + nullptr, idx, g_properties[idx].default_uint_value); +} + StopShowColumn Debugger::GetStopShowColumn() const { const uint32_t idx = ePropertyStopShowColumn; return (lldb::StopShowColumn)m_collection_sp->GetPropertyAtIndexAsEnumeration( Index: lldb/trunk/source/Core/Highlighter.cpp =================================================================== --- lldb/trunk/source/Core/Highlighter.cpp +++ lldb/trunk/source/Core/Highlighter.cpp @@ -0,0 +1,68 @@ +//===-- Highlighter.cpp -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "lldb/Core/Highlighter.h" + +#include "lldb/Target/Language.h" +#include "lldb/Utility/AnsiTerminal.h" +#include "lldb/Utility/StreamString.h" + +using namespace lldb_private; + +std::size_t HighlightStyle::ColorStyle::Apply(Stream &s, + llvm::StringRef value) const { + s << m_prefix << value << m_suffix; + // Calculate how many bytes we have written. + return m_prefix.size() + value.size() + m_suffix.size(); +} + +void HighlightStyle::ColorStyle::Set(llvm::StringRef prefix, + llvm::StringRef suffix) { + m_prefix = lldb_utility::ansi::FormatAnsiTerminalCodes(prefix); + m_suffix = lldb_utility::ansi::FormatAnsiTerminalCodes(suffix); +} + +std::size_t NoHighlighter::Highlight(const HighlightStyle &options, + llvm::StringRef line, + llvm::StringRef previous_lines, + Stream &s) const { + // We just forward the input to the output and do no highlighting. + s << line; + return line.size(); +} + +static HighlightStyle::ColorStyle GetColor(const char *c) { + return HighlightStyle::ColorStyle(c, "${ansi.normal}"); +} + +HighlightStyle HighlightStyle::MakeVimStyle() { + HighlightStyle result; + result.comment = GetColor("${ansi.fg.purple}"); + result.scalar_literal = GetColor("${ansi.fg.red}"); + result.keyword = GetColor("${ansi.fg.green}"); + return result; +} + +const Highlighter & +HighlighterManager::getHighlighterFor(lldb::LanguageType language_type, + llvm::StringRef path) const { + Language *language = lldb_private::Language::FindPlugin(language_type, path); + if (language && language->GetHighlighter()) + return *language->GetHighlighter(); + return m_no_highlighter; +} + +std::string Highlighter::Highlight(const HighlightStyle &options, + llvm::StringRef line, + llvm::StringRef previous_lines) const { + StreamString s; + Highlight(options, line, previous_lines, s); + s.Flush(); + return s.GetString().str(); +} Index: lldb/trunk/source/Core/SourceManager.cpp =================================================================== --- lldb/trunk/source/Core/SourceManager.cpp +++ lldb/trunk/source/Core/SourceManager.cpp @@ -13,6 +13,7 @@ #include "lldb/Core/AddressRange.h" // for AddressRange #include "lldb/Core/Debugger.h" #include "lldb/Core/FormatEntity.h" // for FormatEntity +#include "lldb/Core/Highlighter.h" #include "lldb/Core/Module.h" #include "lldb/Core/ModuleList.h" // for ModuleList #include "lldb/Host/FileSystem.h" @@ -103,6 +104,18 @@ return file_sp; } +static bool should_highlight_source(DebuggerSP debugger_sp) { + if (!debugger_sp) + return false; + + // We don't use ANSI stop column formatting if the debugger doesn't think it + // should be using color. + if (!debugger_sp->GetUseColor()) + return false; + + return debugger_sp->GetHighlightSource(); +} + static bool should_show_stop_column_with_ansi(DebuggerSP debugger_sp) { // We don't use ANSI stop column formatting if we can't lookup values from // the debugger. @@ -114,6 +127,11 @@ if (!debugger_sp->GetUseColor()) return false; + // Don't use terminal attributes when we have highlighting enabled. This + // can mess up the command line. + if (debugger_sp->GetHighlightSource()) + return false; + // We only use ANSI stop column formatting if we're either supposed to show // ANSI where available (which we know we have when we get to this point), or // if we're only supposed to use ANSI. @@ -515,6 +533,16 @@ if (!m_data_sp) return 0; + std::string previous_content; + + HighlightStyle style = HighlightStyle::MakeVimStyle(); + HighlighterManager mgr; + std::string path = GetFileSpec().GetPath(/*denormalize*/ false); + // FIXME: Find a way to get the definitive language this file was written in + // and pass it to the highlighter. + auto &highlighter = + mgr.getHighlighterFor(lldb::LanguageType::eLanguageTypeUnknown, path); + const uint32_t start_line = line <= context_before ? 1 : line - context_before; const uint32_t start_line_offset = GetLineOffset(start_line); @@ -530,10 +558,19 @@ size_t count = end_line_offset - start_line_offset; const uint8_t *cstr = m_data_sp->GetBytes() + start_line_offset; + auto ref = llvm::StringRef(reinterpret_cast(cstr), count); bool displayed_line = false; - if (column && (column < count)) { - auto debugger_sp = m_debugger_wp.lock(); + auto debugger_sp = m_debugger_wp.lock(); + if (should_highlight_source(debugger_sp)) { + bytes_written += + highlighter.Highlight(style, ref, previous_content, *s); + displayed_line = true; + // Add the new line to the previous lines. + previous_content += ref.str(); + } + + if (!displayed_line && column && (column < count)) { if (should_show_stop_column_with_ansi(debugger_sp) && debugger_sp) { // Check if we have any ANSI codes with which to mark this column. If // not, no need to do this work. @@ -581,10 +618,10 @@ // If we didn't end up displaying the line with ANSI codes for whatever // reason, display it now sans codes. if (!displayed_line) - bytes_written = s->Write(cstr, count); + bytes_written = s->PutCString(ref); // Ensure we get an end of line character one way or another. - if (!is_newline_char(cstr[count - 1])) + if (!is_newline_char(ref.back())) bytes_written += s->EOL(); } return bytes_written; Index: lldb/trunk/source/Plugins/Language/CMakeLists.txt =================================================================== --- lldb/trunk/source/Plugins/Language/CMakeLists.txt +++ lldb/trunk/source/Plugins/Language/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(ClangCommon) add_subdirectory(CPlusPlus) add_subdirectory(Go) add_subdirectory(Java) Index: lldb/trunk/source/Plugins/Language/CPlusPlus/CMakeLists.txt =================================================================== --- lldb/trunk/source/Plugins/Language/CPlusPlus/CMakeLists.txt +++ lldb/trunk/source/Plugins/Language/CPlusPlus/CMakeLists.txt @@ -24,6 +24,8 @@ lldbSymbol lldbTarget lldbUtility + lldbPluginClangCommon + LINK_COMPONENTS Support ) Index: lldb/trunk/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h =================================================================== --- lldb/trunk/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h +++ lldb/trunk/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h @@ -19,6 +19,7 @@ #include "llvm/ADT/StringRef.h" // Project includes +#include "Plugins/Language/ClangCommon/ClangHighlighter.h" #include "lldb/Target/Language.h" #include "lldb/Utility/ConstString.h" #include "lldb/lldb-private.h" @@ -26,6 +27,8 @@ namespace lldb_private { class CPlusPlusLanguage : public Language { + ClangHighlighter m_highlighter; + public: class MethodName { public: @@ -90,6 +93,10 @@ HardcodedFormatters::HardcodedSyntheticFinder GetHardcodedSynthetics() override; + bool IsSourceFile(llvm::StringRef file_path) const override; + + const Highlighter *GetHighlighter() const override { return &m_highlighter; } + //------------------------------------------------------------------ // Static Functions //------------------------------------------------------------------ Index: lldb/trunk/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp =================================================================== --- lldb/trunk/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp +++ lldb/trunk/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp @@ -1002,3 +1002,16 @@ return g_formatters; } + +bool CPlusPlusLanguage::IsSourceFile(llvm::StringRef file_path) const { + const auto suffixes = {".cpp", ".cxx", ".c++", ".cc", ".c", + ".h", ".hh", ".hpp", ".hxx", ".h++"}; + for (auto suffix : suffixes) { + if (file_path.endswith_lower(suffix)) + return true; + } + + // Check if we're in a STL path (where the files usually have no extension + // that we could check for. + return file_path.contains("/usr/include/c++/"); +} Index: lldb/trunk/source/Plugins/Language/ClangCommon/CMakeLists.txt =================================================================== --- lldb/trunk/source/Plugins/Language/ClangCommon/CMakeLists.txt +++ lldb/trunk/source/Plugins/Language/ClangCommon/CMakeLists.txt @@ -0,0 +1,9 @@ +add_lldb_library(lldbPluginClangCommon PLUGIN + ClangHighlighter.cpp + + LINK_LIBS + lldbCore + lldbUtility + LINK_COMPONENTS + Support +) Index: lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.h =================================================================== --- lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.h +++ lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.h @@ -0,0 +1,42 @@ +//===-- ClangHighlighter.h --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef liblldb_ClangHighlighter_h_ +#define liblldb_ClangHighlighter_h_ + +// C Includes +// C++ Includes +// Other libraries and framework includes +#include "lldb/Utility/Stream.h" +#include "llvm/ADT/StringSet.h" + +// Project includes +#include "lldb/Core/Highlighter.h" + +namespace lldb_private { + +class ClangHighlighter : public Highlighter { + llvm::StringSet<> keywords; + +public: + ClangHighlighter(); + llvm::StringRef GetName() const override { return "clang"; } + + std::size_t Highlight(const HighlightStyle &options, llvm::StringRef line, + llvm::StringRef previous_lines, + Stream &s) const override; + + /// Returns true if the given string represents a keywords in any Clang + /// supported language. + bool isKeyword(llvm::StringRef token) const; +}; + +} // namespace lldb_private + +#endif // liblldb_ClangHighlighter_h_ Index: lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp =================================================================== --- lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp +++ lldb/trunk/source/Plugins/Language/ClangCommon/ClangHighlighter.cpp @@ -0,0 +1,227 @@ +//===-- ClangHighlighter.cpp ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ClangHighlighter.h" + +#include "lldb/Target/Language.h" +#include "lldb/Utility/AnsiTerminal.h" +#include "lldb/Utility/StreamString.h" + +#include "clang/Basic/SourceManager.h" +#include "clang/Lex/Lexer.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace lldb_private; + +bool ClangHighlighter::isKeyword(llvm::StringRef token) const { + return keywords.find(token) != keywords.end(); +} + +ClangHighlighter::ClangHighlighter() { +#define KEYWORD(X, N) keywords.insert(#X); +#include "clang/Basic/TokenKinds.def" +} + +/// Determines which style should be applied to the given token. +/// \param highlighter +/// The current highlighter that should use the style. +/// \param token +/// The current token. +/// \param tok_str +/// The string in the source code the token represents. +/// \param options +/// The style we use for coloring the source code. +/// \param in_pp_directive +/// If we are currently in a preprocessor directive. NOTE: This is +/// passed by reference and will be updated if the current token starts +/// or ends a preprocessor directive. +/// \return +/// The ColorStyle that should be applied to the token. +static HighlightStyle::ColorStyle +determineClangStyle(const ClangHighlighter &highlighter, + const clang::Token &token, llvm::StringRef tok_str, + const HighlightStyle &options, bool &in_pp_directive) { + using namespace clang; + + if (token.is(tok::comment)) { + // If we were in a preprocessor directive before, we now left it. + in_pp_directive = false; + return options.comment; + } else if (in_pp_directive || token.getKind() == tok::hash) { + // Let's assume that the rest of the line is a PP directive. + in_pp_directive = true; + // Preprocessor directives are hard to match, so we have to hack this in. + return options.pp_directive; + } else if (tok::isStringLiteral(token.getKind())) + return options.string_literal; + else if (tok::isLiteral(token.getKind())) + return options.scalar_literal; + else if (highlighter.isKeyword(tok_str)) + return options.keyword; + else + switch (token.getKind()) { + case tok::raw_identifier: + case tok::identifier: + return options.identifier; + case tok::l_brace: + case tok::r_brace: + return options.braces; + case tok::l_square: + case tok::r_square: + return options.square_brackets; + case tok::l_paren: + case tok::r_paren: + return options.parentheses; + case tok::comma: + return options.comma; + case tok::coloncolon: + case tok::colon: + return options.colon; + + case tok::amp: + case tok::ampamp: + case tok::ampequal: + case tok::star: + case tok::starequal: + case tok::plus: + case tok::plusplus: + case tok::plusequal: + case tok::minus: + case tok::arrow: + case tok::minusminus: + case tok::minusequal: + case tok::tilde: + case tok::exclaim: + case tok::exclaimequal: + case tok::slash: + case tok::slashequal: + case tok::percent: + case tok::percentequal: + case tok::less: + case tok::lessless: + case tok::lessequal: + case tok::lesslessequal: + case tok::spaceship: + case tok::greater: + case tok::greatergreater: + case tok::greaterequal: + case tok::greatergreaterequal: + case tok::caret: + case tok::caretequal: + case tok::pipe: + case tok::pipepipe: + case tok::pipeequal: + case tok::question: + case tok::equal: + case tok::equalequal: + return options.operators; + default: + break; + } + return HighlightStyle::ColorStyle(); +} + +std::size_t ClangHighlighter::Highlight(const HighlightStyle &options, + llvm::StringRef line, + llvm::StringRef previous_lines, + Stream &result) const { + using namespace clang; + + std::size_t written_bytes = 0; + + FileSystemOptions file_opts; + FileManager file_mgr(file_opts); + + unsigned line_number = previous_lines.count('\n') + 1U; + + // Let's build the actual source code Clang needs and setup some utility + // objects. + std::string full_source = previous_lines.str() + line.str(); + llvm::IntrusiveRefCntPtr diag_ids(new DiagnosticIDs()); + llvm::IntrusiveRefCntPtr diags_opts( + new DiagnosticOptions()); + DiagnosticsEngine diags(diag_ids, diags_opts); + clang::SourceManager SM(diags, file_mgr); + auto buf = llvm::MemoryBuffer::getMemBuffer(full_source); + + FileID FID = SM.createFileID(clang::SourceManager::Unowned, buf.get()); + + // Let's just enable the latest ObjC and C++ which should get most tokens + // right. + LangOptions Opts; + Opts.ObjC2 = true; + Opts.CPlusPlus17 = true; + Opts.LineComment = true; + + Lexer lex(FID, buf.get(), SM, Opts); + // The lexer should keep whitespace around. + lex.SetKeepWhitespaceMode(true); + + // Keeps track if we have entered a PP directive. + bool in_pp_directive = false; + + // True once we actually lexed the user provided line. + bool found_user_line = false; + + Token token; + bool exit = false; + while (!exit) { + // Returns true if this is the last token we get from the lexer. + exit = lex.LexFromRawLexer(token); + + bool invalid = false; + unsigned current_line_number = + SM.getSpellingLineNumber(token.getLocation(), &invalid); + if (current_line_number != line_number) + continue; + found_user_line = true; + + // We don't need to print any tokens without a spelling line number. + if (invalid) + continue; + + // Same as above but with the column number. + invalid = false; + unsigned start = SM.getSpellingColumnNumber(token.getLocation(), &invalid); + if (invalid) + continue; + // Column numbers start at 1, but indexes in our string start at 0. + --start; + + // Annotations don't have a length, so let's skip them. + if (token.isAnnotation()) + continue; + + // Extract the token string from our source code. + llvm::StringRef tok_str = line.substr(start, token.getLength()); + + // If the token is just an empty string, we can skip all the work below. + if (tok_str.empty()) + continue; + + // See how we are supposed to highlight this token. + HighlightStyle::ColorStyle color = + determineClangStyle(*this, token, tok_str, options, in_pp_directive); + + written_bytes += color.Apply(result, tok_str); + } + + // If we went over the whole file but couldn't find our own file, then + // somehow our setup was wrong. When we're in release mode we just give the + // user the normal line and pretend we don't know how to highlight it. In + // debug mode we bail out with an assert as this should never happen. + if (!found_user_line) { + result << line; + written_bytes += line.size(); + assert(false && "We couldn't find the user line in the input file?"); + } + + return written_bytes; +} Index: lldb/trunk/source/Plugins/Language/Go/GoLanguage.h =================================================================== --- lldb/trunk/source/Plugins/Language/Go/GoLanguage.h +++ lldb/trunk/source/Plugins/Language/Go/GoLanguage.h @@ -39,6 +39,8 @@ HardcodedFormatters::HardcodedSyntheticFinder GetHardcodedSynthetics() override; + bool IsSourceFile(llvm::StringRef file_path) const override; + //------------------------------------------------------------------ // Static Functions //------------------------------------------------------------------ Index: lldb/trunk/source/Plugins/Language/Go/GoLanguage.cpp =================================================================== --- lldb/trunk/source/Plugins/Language/Go/GoLanguage.cpp +++ lldb/trunk/source/Plugins/Language/Go/GoLanguage.cpp @@ -125,3 +125,7 @@ return g_formatters; } + +bool GoLanguage::IsSourceFile(llvm::StringRef file_path) const { + return file_path.endswith(".go"); +} Index: lldb/trunk/source/Plugins/Language/Java/JavaLanguage.h =================================================================== --- lldb/trunk/source/Plugins/Language/Java/JavaLanguage.h +++ lldb/trunk/source/Plugins/Language/Java/JavaLanguage.h @@ -45,6 +45,8 @@ bool IsNilReference(ValueObject &valobj) override; lldb::TypeCategoryImplSP GetFormatters() override; + + bool IsSourceFile(llvm::StringRef file_path) const override; }; } // namespace lldb_private Index: lldb/trunk/source/Plugins/Language/Java/JavaLanguage.cpp =================================================================== --- lldb/trunk/source/Plugins/Language/Java/JavaLanguage.cpp +++ lldb/trunk/source/Plugins/Language/Java/JavaLanguage.cpp @@ -99,3 +99,7 @@ }); return g_category; } + +bool JavaLanguage::IsSourceFile(llvm::StringRef file_path) const { + return file_path.endswith(".java"); +} Index: lldb/trunk/source/Plugins/Language/OCaml/OCamlLanguage.h =================================================================== --- lldb/trunk/source/Plugins/Language/OCaml/OCamlLanguage.h +++ lldb/trunk/source/Plugins/Language/OCaml/OCamlLanguage.h @@ -31,6 +31,8 @@ return lldb::eLanguageTypeOCaml; } + bool IsSourceFile(llvm::StringRef file_path) const override; + static void Initialize(); static void Terminate(); Index: lldb/trunk/source/Plugins/Language/OCaml/OCamlLanguage.cpp =================================================================== --- lldb/trunk/source/Plugins/Language/OCaml/OCamlLanguage.cpp +++ lldb/trunk/source/Plugins/Language/OCaml/OCamlLanguage.cpp @@ -28,6 +28,15 @@ using namespace lldb; using namespace lldb_private; +bool OCamlLanguage::IsSourceFile(llvm::StringRef file_path) const { + const auto suffixes = {".ml", ".mli"}; + for (auto suffix : suffixes) { + if (file_path.endswith_lower(suffix)) + return true; + } + return false; +} + void OCamlLanguage::Initialize() { PluginManager::RegisterPlugin(GetPluginNameStatic(), "OCaml Language", CreateInstance); Index: lldb/trunk/source/Plugins/Language/ObjC/CMakeLists.txt =================================================================== --- lldb/trunk/source/Plugins/Language/ObjC/CMakeLists.txt +++ lldb/trunk/source/Plugins/Language/ObjC/CMakeLists.txt @@ -31,6 +31,7 @@ lldbTarget lldbUtility lldbPluginAppleObjCRuntime + lldbPluginClangCommon EXTRA_CXXFLAGS ${EXTRA_CXXFLAGS} ) Index: lldb/trunk/source/Plugins/Language/ObjC/ObjCLanguage.h =================================================================== --- lldb/trunk/source/Plugins/Language/ObjC/ObjCLanguage.h +++ lldb/trunk/source/Plugins/Language/ObjC/ObjCLanguage.h @@ -17,6 +17,7 @@ // Other libraries and framework includes // Project includes +#include "Plugins/Language/ClangCommon/ClangHighlighter.h" #include "lldb/Target/Language.h" #include "lldb/Utility/ConstString.h" #include "lldb/lldb-private.h" @@ -24,6 +25,8 @@ namespace lldb_private { class ObjCLanguage : public Language { + ClangHighlighter m_highlighter; + public: class MethodName { public: @@ -121,6 +124,10 @@ bool IsNilReference(ValueObject &valobj) override; + bool IsSourceFile(llvm::StringRef file_path) const override; + + const Highlighter *GetHighlighter() const override { return &m_highlighter; } + //------------------------------------------------------------------ // Static Functions //------------------------------------------------------------------ Index: lldb/trunk/source/Plugins/Language/ObjC/ObjCLanguage.cpp =================================================================== --- lldb/trunk/source/Plugins/Language/ObjC/ObjCLanguage.cpp +++ lldb/trunk/source/Plugins/Language/ObjC/ObjCLanguage.cpp @@ -1102,3 +1102,12 @@ bool isZero = valobj.GetValueAsUnsigned(0, &canReadValue) == 0; return canReadValue && isZero; } + +bool ObjCLanguage::IsSourceFile(llvm::StringRef file_path) const { + const auto suffixes = {".h", ".m", ".M"}; + for (auto suffix : suffixes) { + if (file_path.endswith_lower(suffix)) + return true; + } + return false; +} Index: lldb/trunk/source/Plugins/Language/ObjCPlusPlus/CMakeLists.txt =================================================================== --- lldb/trunk/source/Plugins/Language/ObjCPlusPlus/CMakeLists.txt +++ lldb/trunk/source/Plugins/Language/ObjCPlusPlus/CMakeLists.txt @@ -1,7 +1,8 @@ add_lldb_library(lldbPluginObjCPlusPlusLanguage PLUGIN ObjCPlusPlusLanguage.cpp - + LINK_LIBS lldbCore lldbTarget + lldbPluginClangCommon ) Index: lldb/trunk/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.h =================================================================== --- lldb/trunk/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.h +++ lldb/trunk/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.h @@ -14,12 +14,15 @@ // C++ Includes // Other libraries and framework includes // Project includes +#include "Plugins/Language/ClangCommon/ClangHighlighter.h" #include "lldb/Target/Language.h" #include "lldb/lldb-private.h" namespace lldb_private { class ObjCPlusPlusLanguage : public Language { + ClangHighlighter m_highlighter; + public: ObjCPlusPlusLanguage() = default; @@ -29,6 +32,10 @@ return lldb::eLanguageTypeObjC_plus_plus; } + bool IsSourceFile(llvm::StringRef file_path) const override; + + const Highlighter *GetHighlighter() const override { return &m_highlighter; } + //------------------------------------------------------------------ // Static Functions //------------------------------------------------------------------ Index: lldb/trunk/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.cpp =================================================================== --- lldb/trunk/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.cpp +++ lldb/trunk/source/Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.cpp @@ -16,6 +16,15 @@ using namespace lldb; using namespace lldb_private; +bool ObjCPlusPlusLanguage::IsSourceFile(llvm::StringRef file_path) const { + const auto suffixes = {".h", ".mm"}; + for (auto suffix : suffixes) { + if (file_path.endswith_lower(suffix)) + return true; + } + return false; +} + void ObjCPlusPlusLanguage::Initialize() { PluginManager::RegisterPlugin(GetPluginNameStatic(), "Objective-C++ Language", CreateInstance); Index: lldb/trunk/source/Target/Language.cpp =================================================================== --- lldb/trunk/source/Target/Language.cpp +++ lldb/trunk/source/Target/Language.cpp @@ -77,7 +77,39 @@ return nullptr; } +Language *Language::FindPlugin(llvm::StringRef file_path) { + Language *result = nullptr; + ForEach([&result, file_path](Language *language) { + if (language->IsSourceFile(file_path)) { + result = language; + return false; + } + return true; + }); + return result; +} + +Language *Language::FindPlugin(LanguageType language, + llvm::StringRef file_path) { + Language *result = FindPlugin(language); + // Finding a language by file path is slower, we so we use this as the + // fallback. + if (!result) + result = FindPlugin(file_path); + return result; +} + void Language::ForEach(std::function callback) { + // If we want to iterate over all languages, we first have to complete the + // LanguagesMap. + static llvm::once_flag g_initialize; + llvm::call_once(g_initialize, [] { + for (unsigned lang = eLanguageTypeUnknown; lang < eNumLanguageTypes; + ++lang) { + FindPlugin(static_cast(lang)); + } + }); + std::lock_guard guard(GetLanguagesMutex()); LanguagesMap &map(GetLanguagesMap()); for (const auto &entry : map) { Index: lldb/trunk/unittests/Language/CMakeLists.txt =================================================================== --- lldb/trunk/unittests/Language/CMakeLists.txt +++ lldb/trunk/unittests/Language/CMakeLists.txt @@ -1 +1,2 @@ add_subdirectory(CPlusPlus) +add_subdirectory(Highlighting) Index: lldb/trunk/unittests/Language/Highlighting/CMakeLists.txt =================================================================== --- lldb/trunk/unittests/Language/Highlighting/CMakeLists.txt +++ lldb/trunk/unittests/Language/Highlighting/CMakeLists.txt @@ -0,0 +1,11 @@ +add_lldb_unittest(HighlighterTests + HighlighterTest.cpp + + LINK_LIBS + lldbPluginCPlusPlusLanguage + lldbPluginObjCLanguage + lldbPluginObjCPlusPlusLanguage + lldbPluginJavaLanguage + lldbPluginOCamlLanguage + lldbPluginGoLanguage + ) Index: lldb/trunk/unittests/Language/Highlighting/HighlighterTest.cpp =================================================================== --- lldb/trunk/unittests/Language/Highlighting/HighlighterTest.cpp +++ lldb/trunk/unittests/Language/Highlighting/HighlighterTest.cpp @@ -0,0 +1,221 @@ +//===-- HighlighterTest.cpp -------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "gtest/gtest.h" + +#include "lldb/Core/Highlighter.h" + +#include "Plugins/Language/CPlusPlus/CPlusPlusLanguage.h" +#include "Plugins/Language/Go/GoLanguage.h" +#include "Plugins/Language/Java/JavaLanguage.h" +#include "Plugins/Language/OCaml/OCamlLanguage.h" +#include "Plugins/Language/ObjC/ObjCLanguage.h" +#include "Plugins/Language/ObjCPlusPlus/ObjCPlusPlusLanguage.h" + +using namespace lldb_private; + +namespace { +class HighlighterTest : public testing::Test { +public: + static void SetUpTestCase(); + static void TearDownTestCase(); +}; +} // namespace + +void HighlighterTest::SetUpTestCase() { + // The HighlighterManager uses the language plugins under the hood, so we + // have to initialize them here for our test process. + CPlusPlusLanguage::Initialize(); + GoLanguage::Initialize(); + JavaLanguage::Initialize(); + ObjCLanguage::Initialize(); + ObjCPlusPlusLanguage::Initialize(); + OCamlLanguage::Initialize(); +} + +void HighlighterTest::TearDownTestCase() { + CPlusPlusLanguage::Terminate(); + GoLanguage::Terminate(); + JavaLanguage::Terminate(); + ObjCLanguage::Terminate(); + ObjCPlusPlusLanguage::Terminate(); + OCamlLanguage::Terminate(); +} + +static std::string getName(lldb::LanguageType type) { + HighlighterManager m; + return m.getHighlighterFor(type, "").GetName().str(); +} + +static std::string getName(llvm::StringRef path) { + HighlighterManager m; + return m.getHighlighterFor(lldb::eLanguageTypeUnknown, path).GetName().str(); +} + +TEST_F(HighlighterTest, HighlighterSelectionType) { + EXPECT_EQ(getName(lldb::eLanguageTypeC_plus_plus), "clang"); + EXPECT_EQ(getName(lldb::eLanguageTypeC_plus_plus_03), "clang"); + EXPECT_EQ(getName(lldb::eLanguageTypeC_plus_plus_11), "clang"); + EXPECT_EQ(getName(lldb::eLanguageTypeC_plus_plus_14), "clang"); + EXPECT_EQ(getName(lldb::eLanguageTypeObjC), "clang"); + EXPECT_EQ(getName(lldb::eLanguageTypeObjC_plus_plus), "clang"); + + EXPECT_EQ(getName(lldb::eLanguageTypeUnknown), "none"); + EXPECT_EQ(getName(lldb::eLanguageTypeJulia), "none"); + EXPECT_EQ(getName(lldb::eLanguageTypeJava), "none"); + EXPECT_EQ(getName(lldb::eLanguageTypeHaskell), "none"); +} + +TEST_F(HighlighterTest, HighlighterSelectionPath) { + EXPECT_EQ(getName("myfile.cc"), "clang"); + EXPECT_EQ(getName("moo.cpp"), "clang"); + EXPECT_EQ(getName("mar.cxx"), "clang"); + EXPECT_EQ(getName("foo.C"), "clang"); + EXPECT_EQ(getName("bar.CC"), "clang"); + EXPECT_EQ(getName("a/dir.CC"), "clang"); + EXPECT_EQ(getName("/a/dir.hpp"), "clang"); + EXPECT_EQ(getName("header.h"), "clang"); + + EXPECT_EQ(getName(""), "none"); + EXPECT_EQ(getName("/dev/null"), "none"); + EXPECT_EQ(getName("Factory.java"), "none"); + EXPECT_EQ(getName("poll.py"), "none"); + EXPECT_EQ(getName("reducer.hs"), "none"); +} + +TEST_F(HighlighterTest, FallbackHighlighter) { + HighlighterManager mgr; + const Highlighter &h = + mgr.getHighlighterFor(lldb::eLanguageTypePascal83, "foo.pas"); + + HighlightStyle style; + style.identifier.Set("[", "]"); + style.semicolons.Set("<", ">"); + + const char *code = "program Hello;"; + std::string output = h.Highlight(style, code); + + EXPECT_STREQ(output.c_str(), code); +} + +TEST_F(HighlighterTest, DefaultHighlighter) { + HighlighterManager mgr; + const Highlighter &h = mgr.getHighlighterFor(lldb::eLanguageTypeC, "main.c"); + + HighlightStyle style; + + const char *code = "int my_main() { return 22; } \n"; + std::string output = h.Highlight(style, code); + + EXPECT_STREQ(output.c_str(), code); +} + +//------------------------------------------------------------------------------ +// Tests highlighting with the Clang highlighter. +//------------------------------------------------------------------------------ + +static std::string highlightC(llvm::StringRef code, HighlightStyle style) { + HighlighterManager mgr; + const Highlighter &h = mgr.getHighlighterFor(lldb::eLanguageTypeC, "main.c"); + return h.Highlight(style, code); +} + +TEST_F(HighlighterTest, ClangEmptyInput) { + HighlightStyle s; + EXPECT_EQ("", highlightC("", s)); +} + +TEST_F(HighlighterTest, ClangScalarLiterals) { + HighlightStyle s; + s.scalar_literal.Set("", ""); + + EXPECT_EQ(" int i = 22;", highlightC(" int i = 22;", s)); +} + +TEST_F(HighlighterTest, ClangStringLiterals) { + HighlightStyle s; + s.string_literal.Set("", ""); + + EXPECT_EQ("const char *f = 22 + \"foo\";", + highlightC("const char *f = 22 + \"foo\";", s)); +} + +TEST_F(HighlighterTest, ClangUnterminatedString) { + HighlightStyle s; + s.string_literal.Set("", ""); + + EXPECT_EQ(" f = \"", highlightC(" f = \"", s)); +} + +TEST_F(HighlighterTest, Keywords) { + HighlightStyle s; + s.keyword.Set("", ""); + + EXPECT_EQ(" return 1; ", highlightC(" return 1; ", s)); +} + +TEST_F(HighlighterTest, Colons) { + HighlightStyle s; + s.colon.Set("", ""); + + EXPECT_EQ("foo::bar:", highlightC("foo::bar:", s)); +} + +TEST_F(HighlighterTest, ClangBraces) { + HighlightStyle s; + s.braces.Set("", ""); + + EXPECT_EQ("a{}", highlightC("a{}", s)); +} + +TEST_F(HighlighterTest, ClangSquareBrackets) { + HighlightStyle s; + s.square_brackets.Set("", ""); + + EXPECT_EQ("a[]", highlightC("a[]", s)); +} + +TEST_F(HighlighterTest, ClangCommas) { + HighlightStyle s; + s.comma.Set("", ""); + + EXPECT_EQ(" bool f = foo(), 1;", + highlightC(" bool f = foo(), 1;", s)); +} + +TEST_F(HighlighterTest, ClangPPDirectives) { + HighlightStyle s; + s.pp_directive.Set("", ""); + + EXPECT_EQ("#include \"foo\" //c", + highlightC("#include \"foo\" //c", s)); +} + +TEST_F(HighlighterTest, ClangComments) { + HighlightStyle s; + s.comment.Set("", ""); + + EXPECT_EQ(" /*com */ // com /*n*/", + highlightC(" /*com */ // com /*n*/", s)); +} + +TEST_F(HighlighterTest, ClangOperators) { + HighlightStyle s; + s.operators.Set("[", "]"); + + EXPECT_EQ(" 1[+]2[/]a[*]f[&]x[|][~]l", highlightC(" 1+2/a*f&x|~l", s)); +} + +TEST_F(HighlighterTest, ClangIdentifiers) { + HighlightStyle s; + s.identifier.Set("", ""); + + EXPECT_EQ(" foo c = bar(); return 1;", + highlightC(" foo c = bar(); return 1;", s)); +}