diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -55,6 +55,8 @@ // Implement DiagnosticsConsumer. void onDiagnosticsReady(PathRef File, std::vector Diagnostics) override; void onFileUpdated(PathRef File, const TUStatus &Status) override; + void onHighlightingsReady(PathRef File, + std::vector Highlightings) override; // LSP methods. Notifications have signature void(const Params&). // Calls have signature void(const Params&, Callback). @@ -115,6 +117,9 @@ void reparseOpenedFiles(); void applyConfiguration(const ConfigurationSettings &Settings); + /// Sends a "publishSemanticHighlighting" notification to the LSP client. + void publishSemanticHighlighting(SemanticHighlightingParams Params); + /// Sends a "publishDiagnostics" notification to the LSP client. void publishDiagnostics(const URIForFile &File, std::vector Diagnostics); diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -11,6 +11,7 @@ #include "FormattedString.h" #include "GlobalCompilationDatabase.h" #include "Protocol.h" +#include "SemanticHighlighting.h" #include "SourceCode.h" #include "Trace.h" #include "URI.h" @@ -328,6 +329,8 @@ WithOffsetEncoding.emplace(kCurrentOffsetEncoding, *NegotiatedOffsetEncoding); + ClangdServerOpts.SemanticHighlighting = + Params.capabilities.SemanticHighlighting; if (Params.rootUri && *Params.rootUri) ClangdServerOpts.WorkspaceRoot = Params.rootUri->file(); else if (Params.rootPath && !Params.rootPath->empty()) @@ -407,6 +410,11 @@ }}}}; if (NegotiatedOffsetEncoding) Result["offsetEncoding"] = *NegotiatedOffsetEncoding; + if (Params.capabilities.SemanticHighlighting) + Result.getObject("capabilities") + ->insert( + {"semanticHighlighting", + llvm::json::Object{{"scopes", getTextMateScopeLookupTable()}}}); Reply(std::move(Result)); } @@ -927,6 +935,11 @@ reparseOpenedFiles(); } +void ClangdLSPServer::publishSemanticHighlighting( + SemanticHighlightingParams Params) { + notify("textDocument/semanticHighlighting", Params); +} + void ClangdLSPServer::publishDiagnostics( const URIForFile &File, std::vector Diagnostics) { // Publish diagnostics. @@ -1063,6 +1076,13 @@ return true; } +void ClangdLSPServer::onHighlightingsReady( + PathRef File, std::vector Highlightings) { + publishSemanticHighlighting( + {{URIForFile::canonicalize(File, /*TUPath=*/File)}, + toSemanticHighlightingInformation(Highlightings)}); +} + void ClangdLSPServer::onDiagnosticsReady(PathRef File, std::vector Diagnostics) { auto URI = URIForFile::canonicalize(File, /*TUPath=*/File); diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h --- a/clang-tools-extra/clangd/Protocol.h +++ b/clang-tools-extra/clangd/Protocol.h @@ -406,6 +406,9 @@ /// textDocument.codeAction.codeActionLiteralSupport. bool CodeActionStructure = false; + /// Client supports semantic highlighting. + bool SemanticHighlighting = false; + /// Supported encodings for LSP character offsets. (clangd extension). llvm::Optional> offsetEncoding; @@ -1173,6 +1176,29 @@ }; llvm::json::Value toJSON(const FileStatus &FStatus); +/// Represents a semantic highlighting information that has to be applied on a +/// specific line of the text document. +struct SemanticHighlightingInformation { + /// The line these highlightings belong to. + int Line; + /// The base64 encoded string of highlighting tokens. + std::string Tokens; +}; + +bool operator==(const SemanticHighlightingInformation &Lhs, + const SemanticHighlightingInformation &Rhs); +llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting); + +/// Parameters for the semantic highlighting (server-side) push notification. +struct SemanticHighlightingParams { + /// The textdocument these highlightings belong to. + TextDocumentIdentifier TextDocument; + /// The lines of highlightings that should be sent. + std::vector Lines; +}; + +llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting); + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp --- a/clang-tools-extra/clangd/Protocol.cpp +++ b/clang-tools-extra/clangd/Protocol.cpp @@ -273,6 +273,12 @@ if (!O) return false; if (auto *TextDocument = O->getObject("textDocument")) { + if (auto *SemanticHighlighting = + TextDocument->getObject("semanticHighlightingCapabilities")) { + if (auto SemanticHighlightingSupport = + SemanticHighlighting->getBoolean("semanticHighlighting")) + R.SemanticHighlighting = *SemanticHighlightingSupport; + } if (auto *Diagnostics = TextDocument->getObject("publishDiagnostics")) { if (auto CategorySupport = Diagnostics->getBoolean("categorySupport")) R.DiagnosticCategory = *CategorySupport; @@ -1027,5 +1033,22 @@ return OS << toString(Enc); } +bool operator==(const SemanticHighlightingInformation &Lhs, + const SemanticHighlightingInformation &Rhs) { + return Lhs.Line == Rhs.Line && Lhs.Tokens == Rhs.Tokens; +} + +llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting) { + return llvm::json::Object{{"line", Highlighting.Line}, + {"tokens", Highlighting.Tokens}}; +} + +llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting) { + return llvm::json::Object{ + {"textDocument", Highlighting.TextDocument}, + {"lines", std::move(Highlighting.Lines)}, + }; +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h --- a/clang-tools-extra/clangd/SemanticHighlighting.h +++ b/clang-tools-extra/clangd/SemanticHighlighting.h @@ -4,19 +4,25 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // +// An implementation of semantic highlighting based on this proposal: +// https://github.com/microsoft/vscode-languageserver-node/pull/367 in clangd. +// Semantic highlightings are calculated for an AST by visiting every AST node +// and classifying nodes that are interesting to highlight (variables/function +// calls etc.). //===----------------------------------------------------------------------===// #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H #include "ClangdUnit.h" +#include "Protocol.h" namespace clang { namespace clangd { enum class HighlightingKind { - Variable, - Function, + Variable = 0, + Function = 1, }; // Contains all information needed for the highlighting a token. @@ -31,6 +37,14 @@ // main AST. std::vector getSemanticHighlightings(ParsedAST &AST); +// Gets the TextMate scopes as a double nested array where the +// SemanticHighlightKind indexes correctly into this vector. +std::vector> getTextMateScopeLookupTable(); + +// Convert to LSP's semantic highlighting information. +std::vector +toSemanticHighlightingInformation(llvm::ArrayRef Tokens); + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -8,6 +8,7 @@ #include "SemanticHighlighting.h" #include "Logger.h" +#include "Protocol.h" #include "SourceCode.h" #include "clang/AST/ASTContext.h" #include "clang/AST/RecursiveASTVisitor.h" @@ -63,6 +64,48 @@ } }; +// Encode binary data into base64. +// This was copied from compiler-rt/lib/fuzzer/FuzzerUtil.cpp. +// FIXME: Factor this out into llvm/Support? +std::string encodeBase64(const llvm::SmallVectorImpl &Bytes) { + static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + std::string Res; + size_t I; + for (I = 0; I + 2 < Bytes.size(); I += 3) { + uint32_t X = (Bytes[I] << 16) + (Bytes[I + 1] << 8) + Bytes[I + 2]; + Res += Table[(X >> 18) & 63]; + Res += Table[(X >> 12) & 63]; + Res += Table[(X >> 6) & 63]; + Res += Table[X & 63]; + } + if (I + 1 == Bytes.size()) { + uint32_t X = (Bytes[I] << 16); + Res += Table[(X >> 18) & 63]; + Res += Table[(X >> 12) & 63]; + Res += "=="; + } else if (I + 2 == Bytes.size()) { + uint32_t X = (Bytes[I] << 16) + (Bytes[I + 1] << 8); + Res += Table[(X >> 18) & 63]; + Res += Table[(X >> 12) & 63]; + Res += Table[(X >> 6) & 63]; + Res += "="; + } + return Res; +} + +void write32be(uint32_t I, llvm::raw_ostream &OS) { + std::array Buf; + llvm::support::endian::write32be(Buf.data(), I); + OS.write(Buf.data(), Buf.size()); +} + +void write16be(uint16_t I, llvm::raw_ostream &OS) { + std::array Buf; + llvm::support::endian::write16be(Buf.data(), I); + OS.write(Buf.data(), Buf.size()); +} } // namespace bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs) { @@ -73,5 +116,49 @@ return HighlightingTokenCollector(AST).collectTokens(); } +std::vector +toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { + if (Tokens.size() == 0) { + return {}; + } + + // FIXME: Tokens might be multiple lines long (block comments) in this case + // this needs to add multiple lines for those tokens. + std::map> TokenLines; + for (const HighlightingToken &Token : Tokens) + TokenLines[Token.R.start.line].push_back(Token); + + std::vector Lines; + Lines.reserve(TokenLines.size()); + for (const auto &Line : TokenLines) { + llvm::SmallVector LineByteTokens; + llvm::raw_svector_ostream OS(LineByteTokens); + for (const auto &Token : Line.second) { + // Writes the token to LineByteTokens in the byte format specified by the + // LSP proposal. + write32be(Token.R.start.character, OS); + write16be(Token.R.end.character - Token.R.start.character, OS); + write16be(static_cast(Token.Kind), OS); + } + + Lines.push_back({Line.first, encodeBase64(LineByteTokens)}); + } + + return Lines; +} + +std::vector> getTextMateScopeLookupTable() { + // FIXME: Add scopes for C and Objective C. + std::map> Scopes = { + {HighlightingKind::Variable, {"variable.cpp"}}, + {HighlightingKind::Function, {"entity.name.function.cpp"}}}; + std::vector> NestedScopes(Scopes.size()); + for (const auto &Scope : Scopes) { + NestedScopes[static_cast(Scope.first)] = Scope.second; + } + + return NestedScopes; +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/test/initialize-params.test b/clang-tools-extra/clangd/test/initialize-params.test --- a/clang-tools-extra/clangd/test/initialize-params.test +++ b/clang-tools-extra/clangd/test/initialize-params.test @@ -1,6 +1,6 @@ # RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s # Test initialize request parameters with rootUri -{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootUri":"test:///workspace","capabilities":{},"trace":"off"}} +{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootUri":"test:///workspace","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}} # CHECK: "id": 0, # CHECK-NEXT: "jsonrpc": "2.0", # CHECK-NEXT: "result": { @@ -33,6 +33,16 @@ # CHECK-NEXT: "hoverProvider": true, # CHECK-NEXT: "referencesProvider": true, # CHECK-NEXT: "renameProvider": true, +# CHECK-NEXT: "semanticHighlighting": { +# CHECK-NEXT: "scopes": [ +# CHECK-NEXT: [ +# CHECK-NEXT: "variable.cpp" +# CHECK-NEXT: ], +# CHECK-NEXT: [ +# CHECK-NEXT: "entity.name.function.cpp" +# CHECK-NEXT: ] +# CHECK-NEXT: ] +# CHECK-NEXT: }, # CHECK-NEXT: "signatureHelpProvider": { # CHECK-NEXT: "triggerCharacters": [ # CHECK-NEXT: "(", diff --git a/clang-tools-extra/clangd/test/semantic-highlighting.test b/clang-tools-extra/clangd/test/semantic-highlighting.test new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/test/semantic-highlighting.test @@ -0,0 +1,21 @@ +# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s +{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}} +--- +{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","version":1,"text":"int x = 2;"}}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 0, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- +{"jsonrpc":"2.0","id":3,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -8,6 +8,7 @@ #include "Annotations.h" #include "ClangdServer.h" +#include "Protocol.h" #include "SemanticHighlighting.h" #include "TestFS.h" #include "TestTU.h" @@ -66,7 +67,7 @@ } } -TEST(ClangdSemanticHighlightingTest, GeneratesHighlightsWhenFileChange) { +TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) { class HighlightingsCounterDiagConsumer : public DiagnosticsConsumer { public: std::atomic Count = {0}; @@ -90,6 +91,29 @@ ASSERT_EQ(DiagConsumer.Count, 1); } +TEST(SemanticHighlighting, toSemanticHighlightingInformation) { + auto CreatePosition = [](int Line, int Character) -> Position { + Position Pos; + Pos.line = Line; + Pos.character = Character; + return Pos; + }; + + std::vector Tokens{ + {HighlightingKind::Variable, + Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, + {HighlightingKind::Function, + Range{CreatePosition(3, 4), CreatePosition(3, 7)}}, + {HighlightingKind::Variable, + Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}; + std::vector ActualResults = + toSemanticHighlightingInformation(Tokens); + std::vector ExpectedResults = { + {1, "AAAAAQAEAAA="}, + {3, "AAAACAAEAAAAAAAEAAMAAQ=="}}; + EXPECT_EQ(ActualResults, ExpectedResults); +} + } // namespace } // namespace clangd } // namespace clang