diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -119,6 +119,7 @@ Callback>); void onDocumentLink(const DocumentLinkParams &, Callback>); + void onSemanticTokens(const SemanticTokensParams &, Callback); std::vector getFixes(StringRef File, const clangd::Diagnostic &D); diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -458,6 +458,14 @@ Transp.notify(Method, std::move(Params)); } +static std::vector semanticTokenTypes() { + std::vector Types; + for (unsigned I = 0; I <= static_cast(HighlightingKind::LastKind); + ++I) + Types.push_back(toSemanticTokenType(static_cast(I))); + return Types; +} + void ClangdLSPServer::onInitialize(const InitializeParams &Params, Callback Reply) { // Determine character encoding first as it affects constructed ClangdServer. @@ -569,6 +577,14 @@ // trigger on '->' and '::'. {"triggerCharacters", {".", ">", ":"}}, }}, + {"semanticTokensProvider", + llvm::json::Object{ + {"documentProvider", true}, + {"rangeProvider", false}, + {"legend", + llvm::json::Object{{"tokenTypes", semanticTokenTypes()}, + {"tokenModifiers", llvm::json::Array()}}}, + }}, {"signatureHelpProvider", llvm::json::Object{ {"triggerCharacters", {"(", ","}}, @@ -1220,6 +1236,20 @@ }); } +void ClangdLSPServer::onSemanticTokens(const SemanticTokensParams &Params, + Callback CB) { + Server->semanticHighlights( + Params.textDocument.uri.file(), + [CB(std::move(CB))]( + llvm::Expected> Toks) mutable { + if (!Toks) + return CB(Toks.takeError()); + SemanticTokens Result; + Result.data = toSemanticTokens(*Toks); + CB(std::move(Result)); + }); +} + ClangdLSPServer::ClangdLSPServer( class Transport &Transp, const FileSystemProvider &FSProvider, const clangd::CodeCompleteOptions &CCOpts, @@ -1267,6 +1297,7 @@ MsgHandler->bind("typeHierarchy/resolve", &ClangdLSPServer::onResolveTypeHierarchy); MsgHandler->bind("textDocument/selectionRange", &ClangdLSPServer::onSelectionRange); MsgHandler->bind("textDocument/documentLink", &ClangdLSPServer::onDocumentLink); + MsgHandler->bind("textDocument/semanticTokens", &ClangdLSPServer::onSemanticTokens); // clang-format on } diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -297,7 +297,10 @@ /// Get all document links in a file. void documentLinks(PathRef File, Callback> CB); - + + void semanticHighlights(PathRef File, + Callback>); + /// Returns estimated memory usage for each of the currently open files. /// The order of results is unspecified. /// Overall memory usage of clangd may be significantly more than reported diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -685,6 +685,18 @@ TUScheduler::InvalidateOnUpdate); } +void ClangdServer::semanticHighlights( + PathRef File, Callback> CB) { + auto Action = + [CB = std::move(CB)](llvm::Expected InpAST) mutable { + if (!InpAST) + return CB(InpAST.takeError()); + CB(clangd::getSemanticHighlightings(InpAST->AST)); + }; + WorkScheduler.runWithAST("SemanticHighlights", File, std::move(Action), + TUScheduler::InvalidateOnUpdate); +} + std::vector> ClangdServer::getUsedBytesPerFile() const { return WorkScheduler.getUsedBytesPerFile(); diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h --- a/clang-tools-extra/clangd/Protocol.h +++ b/clang-tools-extra/clangd/Protocol.h @@ -1340,7 +1340,47 @@ std::string state; // FIXME: add detail messages. }; -llvm::json::Value toJSON(const FileStatus &FStatus); +llvm::json::Value toJSON(const FileStatus &); + +/// Specifies a single semantic token in the document. +/// This struct is not part of LSP, which just encodes lists of tokens as +/// arrays of numbers directly. +struct SemanticToken { + /// token line number, relative to the previous token + unsigned deltaLine = 0; + /// token start character, relative to the previous token + /// (relative to 0 or the previous token's start if they are on the same line) + unsigned deltaStart = 0; + /// the length of the token. A token cannot be multiline + unsigned length = 0; + /// will be looked up in `SemanticTokensLegend.tokenTypes` + unsigned tokenType = 0; + /// each set bit will be looked up in `SemanticTokensLegend.tokenModifiers` + unsigned tokenModifiers = 0; + + void encode(std::vector &Out) const; +}; + +/// A versioned set of tokens. +struct SemanticTokens { + // An optional result id. If provided and clients support delta updating + // the client will include the result id in the next semantic token request. + // A server can then instead of computing all semantic tokens again simply + // send a delta. + llvm::Optional resultId; + + /// The actual tokens. For a detailed description about how the data is + /// structured pls see + /// https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71 + std::vector data; +}; +llvm::json::Value toJSON(const SemanticTokens &); + +struct SemanticTokensParams { + /// The text document. + TextDocumentIdentifier textDocument; +}; +bool fromJSON(const llvm::json::Value &, SemanticTokensParams &); /// Represents a semantic highlighting information that has to be applied on a /// specific line of the text document. diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp --- a/clang-tools-extra/clangd/Protocol.cpp +++ b/clang-tools-extra/clangd/Protocol.cpp @@ -984,6 +984,29 @@ }; } +void SemanticToken::encode(std::vector &Out) const { + Out.push_back(deltaLine); + Out.push_back(deltaStart); + Out.push_back(length); + Out.push_back(tokenType); + Out.push_back(tokenModifiers); +} + +llvm::json::Value toJSON(const SemanticTokens &Tokens) { + std::vector Data; + for (const auto &Tok : Tokens.data) + Tok.encode(Data); + llvm::json::Object Result{{"data", std::move(Data)}}; + if (Tokens.resultId) + Result["resultId"] = *Tokens.resultId; + return Result; +} + +bool fromJSON(const llvm::json::Value &Params, SemanticTokensParams &R) { + llvm::json::ObjectMapper O(Params); + return O && O.map("textDocument", R.textDocument); +} + llvm::raw_ostream &operator<<(llvm::raw_ostream &O, const DocumentHighlight &V) { O << V.range; diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h --- a/clang-tools-extra/clangd/SemanticHighlighting.h +++ b/clang-tools-extra/clangd/SemanticHighlighting.h @@ -6,8 +6,21 @@ // //===----------------------------------------------------------------------===// // -// An implementation of semantic highlighting based on this proposal: -// https://github.com/microsoft/vscode-languageserver-node/pull/367 in clangd. +// This file supports semantic highlighting: categorizing tokens in the file so +// that the editor can color/style them differently. +// +// This is particularly valuable for C++: its complex and context-dependent +// grammar is a challenge for simple syntax-highlighting techniques. +// +// We support two protocols for providing highlights to the client: +// - the `textDocument/semanticTokens` request from LSP 3.16 +// https://github.com/microsoft/vscode-languageserver-node/blob/release/protocol/3.16.0-next.1/protocol/src/protocol.semanticTokens.proposed.ts +// - the earlier proposed `textDocument/semanticHighlighting` notification +// https://github.com/microsoft/vscode-languageserver-node/pull/367 +// This is referred to as "Theia" semantic highlighting in the code. +// It was supported from clangd 9 but should be considered deprecated as of +// clangd 11 and eventually removed. +// // Semantic highlightings are calculated for an AST by visiting every AST node // and classifying nodes that are interesting to highlight (variables/function // calls etc.). @@ -75,6 +88,9 @@ // main AST. std::vector getSemanticHighlightings(ParsedAST &AST); +std::vector toSemanticTokens(llvm::ArrayRef); +llvm::StringRef toSemanticTokenType(HighlightingKind Kind); + /// Converts a HighlightingKind to a corresponding TextMate scope /// (https://manual.macromates.com/en/language_grammars). llvm::StringRef toTextMateScope(HighlightingKind Kind); diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -445,6 +445,83 @@ return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens); } +std::vector +toSemanticTokens(llvm::ArrayRef Tokens) { + assert(std::is_sorted(Tokens.begin(), Tokens.end())); + std::vector Result; + const HighlightingToken *Last = nullptr; + for (const HighlightingToken &Tok : Tokens) { + // FIXME: support inactive code - we need to provide the actual bounds. + if (Tok.Kind == HighlightingKind::InactiveCode) + continue; + Result.emplace_back(); + SemanticToken &Out = Result.back(); + // deltaStart/deltaLine are relative if possible. + if (Last) { + assert(Tok.R.start.line >= Last->R.start.line); + Out.deltaLine = Tok.R.start.line - Last->R.start.line; + if (Out.deltaLine == 0) { + assert(Tok.R.start.character >= Last->R.start.character); + Out.deltaStart = Tok.R.start.character - Last->R.start.character; + } else { + Out.deltaStart = Tok.R.start.character; + } + } else { + Out.deltaLine = Tok.R.start.line; + Out.deltaStart = Tok.R.start.character; + } + assert(Tok.R.end.line == Tok.R.start.line); + Out.length = Tok.R.end.character - Tok.R.start.character; + Out.tokenType = static_cast(Tok.Kind); + + Last = &Tok; + } + return Result; +} +llvm::StringRef toSemanticTokenType(HighlightingKind Kind) { + switch (Kind) { + case HighlightingKind::Variable: + case HighlightingKind::LocalVariable: + case HighlightingKind::StaticField: + return "variable"; + case HighlightingKind::Parameter: + return "parameter"; + case HighlightingKind::Function: + return "function"; + case HighlightingKind::Method: + return "member"; + case HighlightingKind::StaticMethod: + // FIXME: better function/member with static modifier? + return "function"; + case HighlightingKind::Field: + return "member"; + case HighlightingKind::Class: + return "class"; + case HighlightingKind::Enum: + return "enum"; + case HighlightingKind::EnumConstant: + return "enumConstant"; // nonstandard + case HighlightingKind::Typedef: + return "type"; + case HighlightingKind::DependentType: + return "dependent"; // nonstandard + case HighlightingKind::DependentName: + return "dependent"; // nonstandard + case HighlightingKind::Namespace: + return "namespace"; + case HighlightingKind::TemplateParameter: + return "typeParameter"; + case HighlightingKind::Concept: + return "concept"; // nonstandard + case HighlightingKind::Primitive: + return "type"; + case HighlightingKind::Macro: + return "macro"; + case HighlightingKind::InactiveCode: + return "comment"; + } +} + std::vector toTheiaSemanticHighlightingInformation( llvm::ArrayRef Tokens) { diff --git a/clang-tools-extra/clangd/test/initialize-params.test b/clang-tools-extra/clangd/test/initialize-params.test --- a/clang-tools-extra/clangd/test/initialize-params.test +++ b/clang-tools-extra/clangd/test/initialize-params.test @@ -38,6 +38,16 @@ # CHECK-NEXT: "referencesProvider": true, # CHECK-NEXT: "renameProvider": true, # CHECK-NEXT: "selectionRangeProvider": true, +# CHECK-NEXT: "semanticTokensProvider": { +# CHECK-NEXT: "documentProvider": true, +# CHECK-NEXT: "legend": { +# CHECK-NEXT: "tokenModifiers": [], +# CHECK-NEXT: "tokenTypes": [ +# CHECK-NEXT: "variable", +# CHECK: ] +# CHECK-NEXT: }, +# CHECK-NEXT: "rangeProvider": false +# CHECK-NEXT: }, # CHECK-NEXT: "signatureHelpProvider": { # CHECK-NEXT: "triggerCharacters": [ # CHECK-NEXT: "(", diff --git a/clang-tools-extra/clangd/test/semantic-tokens.test b/clang-tools-extra/clangd/test/semantic-tokens.test new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/test/semantic-tokens.test @@ -0,0 +1,22 @@ +# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s +{"jsonrpc":"2.0","id":0,"method":"initialize","params":{}} +--- +{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","text":"int x = 2;"}}} +--- +{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens","params":{"textDocument":{"uri":"test:///foo.cpp"}}} +# CHECK: "id": 1, +# CHECK-NEXT: "jsonrpc": "2.0", +# CHECK-NEXT: "result": { +# CHECK-NEXT: "data": [ +# First line, char 5, variable, no modifiers. +# CHECK-NEXT: 0, +# CHECK-NEXT: 4, +# CHECK-NEXT: 1, +# CHECK-NEXT: 0, +# CHECK-NEXT: 0 +# CHECK-NEXT: ] +# CHECK-NEXT: } +--- +{"jsonrpc":"2.0","id":2,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -720,6 +720,41 @@ ASSERT_EQ(Counter.Count, 1); } +TEST(SemanticHighlighting, toSemanticTokens) { + auto CreatePosition = [](int Line, int Character) -> Position { + Position Pos; + Pos.line = Line; + Pos.character = Character; + return Pos; + }; + + std::vector Tokens = { + {HighlightingKind::Variable, + Range{CreatePosition(1, 1), CreatePosition(1, 5)}}, + {HighlightingKind::Function, + Range{CreatePosition(3, 4), CreatePosition(3, 7)}}, + {HighlightingKind::Variable, + Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, + }; + + std::vector Results = toSemanticTokens(Tokens); + EXPECT_EQ(Tokens.size(), Results.size()); + EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable)); + EXPECT_EQ(Results[0].deltaLine, 1u); + EXPECT_EQ(Results[0].deltaStart, 1u); + EXPECT_EQ(Results[0].length, 4u); + + EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function)); + EXPECT_EQ(Results[1].deltaLine, 2u); + EXPECT_EQ(Results[1].deltaStart, 4u); + EXPECT_EQ(Results[1].length, 3u); + + EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable)); + EXPECT_EQ(Results[2].deltaLine, 0u); + EXPECT_EQ(Results[2].deltaStart, 4u); + EXPECT_EQ(Results[2].length, 4u); +} + TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) { auto CreatePosition = [](int Line, int Character) -> Position { Position Pos;