[clangd] Update semanticTokens support to reflect latest LSP draft

[sammccall]

Summary

Mostly a few methods and message names have been renamed.

[hokein]

thanks!

clang-tools-extra/clangd/ClangdLSPServer.h

private:
   void onChangeConfiguration(const DidChangeConfigurationParams &);
   void onSymbolInfo(const TextDocumentPositionParams &,
                     Callback<std::vector<SymbolDetails>>);
   void onSelectionRange(const SelectionRangeParams &,
                         Callback<std::vector<SelectionRange>>);
   void onDocumentLink(const DocumentLinkParams &,
                       Callback<std::vector<DocumentLink>>);
   void onSemanticTokens(const SemanticTokensParams &, Callback<SemanticTokens>);
-  void onSemanticTokensEdits(const SemanticTokensEditsParams &,
-                             Callback<SemanticTokensOrEdits>);
+  void onSemanticTokensDelta(const SemanticTokensDeltaParams &,
+                             Callback<SemanticTokensOrDelta>);
 
   std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
 
   /// Checks if completion request should be ignored. We need this due to the
   /// limitation of the LSP. Per LSP, a client sends requests for all "trigger
   /// character" we specify, but for '>' and ':' we need to check they actually
   /// produce '->' and '::', respectively.
   bool shouldRunCompletion(const CompletionParams &Params) const;

clang-tools-extra/clangd/ClangdLSPServer.cpp

llvm::json::Object Result{
              llvm::json::Object{
                  {"allCommitCharacters", " \t()[]{}<>:;,+-/*%^&#?.=\"'|"},
                  {"resolveProvider", false},
                  // We do extra checks, e.g. that > is part of ->.
                  {"triggerCharacters", {".", "<", ">", ":", "\"", "/"}},
              }},
             {"semanticTokensProvider",
              llvm::json::Object{
-                 {"documentProvider", llvm::json::Object{{"edits", true}}},
-                 {"rangeProvider", false},
+                 {"full", llvm::json::Object{{"delta", true}}},
+                 {"range", false},
                  {"legend",
                   llvm::json::Object{{"tokenTypes", semanticTokenTypes()},
                                      {"tokenModifiers", llvm::json::Array()}}},
              }},
             {"signatureHelpProvider",
              llvm::json::Object{
                  {"triggerCharacters", {"(", ","}},
              }},
Server->semanticHighlights(
           Last.tokens = Result.tokens;
           increment(Last.resultId);
           Result.resultId = Last.resultId;
         }
         CB(std::move(Result));
       });
 }
 
-void ClangdLSPServer::onSemanticTokensEdits(
-    const SemanticTokensEditsParams &Params,
-    Callback<SemanticTokensOrEdits> CB) {
+void ClangdLSPServer::onSemanticTokensDelta(
+    const SemanticTokensDeltaParams &Params,
+    Callback<SemanticTokensOrDelta> CB) {
   Server->semanticHighlights(
       Params.textDocument.uri.file(),
       [this, PrevResultID(Params.previousResultId),
        File(Params.textDocument.uri.file().str()), CB(std::move(CB))](
           llvm::Expected<std::vector<HighlightingToken>> HT) mutable {
         if (!HT)
           return CB(HT.takeError());
         std::vector<SemanticToken> Toks = toSemanticTokens(*HT);
 
-        SemanticTokensOrEdits Result;
+        SemanticTokensOrDelta Result;
         {
           std::lock_guard<std::mutex> Lock(SemanticTokensMutex);
           auto &Last = LastSemanticTokens[File];
 
           if (PrevResultID == Last.resultId) {
             Result.edits = diffTokens(Last.tokens, Toks);
           } else {
-            vlog("semanticTokens/edits: wanted edits vs {0} but last result "
-                 "had ID {1}. Returning full token list.",
+            vlog("semanticTokens/full/delta: wanted edits vs {0} but last "
+                 "result had ID {1}. Returning full token list.",
                  PrevResultID, Last.resultId);
             Result.tokens = Toks;
           }
 
           Last.tokens = std::move(Toks);
           increment(Last.resultId);
           Result.resultId = Last.resultId;
         }
ClangdLSPServer::ClangdLSPServer(
   MsgHandler->bind("textDocument/didSave", &ClangdLSPServer::onDocumentDidSave);
   MsgHandler->bind("workspace/didChangeWatchedFiles", &ClangdLSPServer::onFileEvent);
   MsgHandler->bind("workspace/didChangeConfiguration", &ClangdLSPServer::onChangeConfiguration);
   MsgHandler->bind("textDocument/symbolInfo", &ClangdLSPServer::onSymbolInfo);
   MsgHandler->bind("textDocument/typeHierarchy", &ClangdLSPServer::onTypeHierarchy);
   MsgHandler->bind("typeHierarchy/resolve", &ClangdLSPServer::onResolveTypeHierarchy);
   MsgHandler->bind("textDocument/selectionRange", &ClangdLSPServer::onSelectionRange);
   MsgHandler->bind("textDocument/documentLink", &ClangdLSPServer::onDocumentLink);
-  MsgHandler->bind("textDocument/semanticTokens", &ClangdLSPServer::onSemanticTokens);
-  MsgHandler->bind("textDocument/semanticTokens/edits", &ClangdLSPServer::onSemanticTokensEdits);
+  MsgHandler->bind("textDocument/semanticTokens/full", &ClangdLSPServer::onSemanticTokens);
+  MsgHandler->bind("textDocument/semanticTokens/full/delta", &ClangdLSPServer::onSemanticTokensDelta);
   // clang-format on
 }
 
 ClangdLSPServer::~ClangdLSPServer() {
   IsBeingDestroyed = true;
   // Explicitly destroy ClangdServer first, blocking on threads it owns.
   // This ensures they don't access any other members.
   Server.reset();

clang-tools-extra/clangd/Protocol.h

 /// A versioned set of tokens.
 struct SemanticTokens {
   // An optional result id. If provided and clients support delta updating
   // the client will include the result id in the next semantic token request.
   // A server can then instead of computing all semantic tokens again simply
   // send a delta.
   std::string resultId;
 
-  /// The actual tokens. For a detailed description about how the data is
-  /// structured pls see
-  /// https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71
-  std::vector<SemanticToken> tokens;
+  /// The actual tokens.
+  std::vector<SemanticToken> tokens; // encoded as a flat integer array.
 };
 llvm::json::Value toJSON(const SemanticTokens &);
 
+/// Body of textDocument/semanticTokens/full request.
 struct SemanticTokensParams {
   /// The text document.
   TextDocumentIdentifier textDocument;
 };
 bool fromJSON(const llvm::json::Value &, SemanticTokensParams &);
 
+/// Body of textDocument/semanticTokens/full/delta request.
 /// Requests the changes in semantic tokens since a previous response.
-struct SemanticTokensEditsParams {
+struct SemanticTokensDeltaParams {
   /// The text document.
   TextDocumentIdentifier textDocument;
   /// The previous result id.
   std::string previousResultId;
 };
-bool fromJSON(const llvm::json::Value &Params, SemanticTokensEditsParams &R);
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensDeltaParams &R);
 
 /// Describes a a replacement of a contiguous range of semanticTokens.
 struct SemanticTokensEdit {
   // LSP specifies `start` and `deleteCount` which are relative to the array
   // encoding of the previous tokens.
   // We use token counts instead, and translate when serializing this struct.
   unsigned startToken = 0;
   unsigned deleteTokens = 0;
-  std::vector<SemanticToken> tokens;
+  std::vector<SemanticToken> tokens; // encoded as a flat integer array
 };
 llvm::json::Value toJSON(const SemanticTokensEdit &);
 
-/// This models LSP SemanticTokensEdits | SemanticTokens, which is the result of
-/// textDocument/semanticTokens/edits.
-struct SemanticTokensOrEdits {
+/// This models LSP SemanticTokensDelta | SemanticTokens, which is the result of
+/// textDocument/semanticTokens/full/delta.
+struct SemanticTokensOrDelta {
   std::string resultId;
   /// Set if we computed edits relative to a previous set of tokens.
   llvm::Optional<std::vector<SemanticTokensEdit>> edits;
   /// Set if we computed a fresh set of tokens.
-  llvm::Optional<std::vector<SemanticToken>> tokens;
+  llvm::Optional<std::vector<SemanticToken>> tokens; // encoded as integer array
 };
-llvm::json::Value toJSON(const SemanticTokensOrEdits &);
+llvm::json::Value toJSON(const SemanticTokensOrDelta &);
 
 /// Represents a semantic highlighting information that has to be applied on a
 /// specific line of the text document.
 struct TheiaSemanticHighlightingInformation {
   /// The line these highlightings belong to.
   int Line = 0;
   /// The base64 encoded string of highlighting tokens.
   std::string Tokens;

clang-tools-extra/clangd/Protocol.cpp

 
 llvm::json::Value toJSON(const SemanticTokensEdit &Edit) {
   return llvm::json::Object{
       {"start", SemanticTokenEncodingSize * Edit.startToken},
       {"deleteCount", SemanticTokenEncodingSize * Edit.deleteTokens},
       {"data", encodeTokens(Edit.tokens)}};
 }
 
-llvm::json::Value toJSON(const SemanticTokensOrEdits &TE) {
+llvm::json::Value toJSON(const SemanticTokensOrDelta &TE) {
   llvm::json::Object Result{{"resultId", TE.resultId}};
   if (TE.edits)
     Result["edits"] = *TE.edits;
   if (TE.tokens)
     Result["data"] = encodeTokens(*TE.tokens);
   return std::move(Result);
 }
 
 bool fromJSON(const llvm::json::Value &Params, SemanticTokensParams &R) {
   llvm::json::ObjectMapper O(Params);
   return O && O.map("textDocument", R.textDocument);
 }
 
-bool fromJSON(const llvm::json::Value &Params, SemanticTokensEditsParams &R) {
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensDeltaParams &R) {
   llvm::json::ObjectMapper O(Params);
   return O && O.map("textDocument", R.textDocument) &&
          O.map("previousResultId", R.previousResultId);
 }
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &O,
                               const DocumentHighlight &V) {
   O << V.range;

clang-tools-extra/clangd/test/initialize-params.test

 # CHECK-NEXT:          "clangd.applyTweak"
 # CHECK-NEXT:        ]
 # CHECK-NEXT:      },
 # CHECK-NEXT:      "hoverProvider": true,
 # CHECK-NEXT:      "referencesProvider": true,
 # CHECK-NEXT:      "renameProvider": true,
 # CHECK-NEXT:      "selectionRangeProvider": true,
 # CHECK-NEXT:      "semanticTokensProvider": {
-# CHECK-NEXT:        "documentProvider": {
-# CHECK-NEXT:          "edits": true
+# CHECK-NEXT:        "full": {
+# CHECK-NEXT:          "delta": true
 # CHECK-NEXT:        },
 # CHECK-NEXT:        "legend": {
 # CHECK-NEXT:          "tokenModifiers": [],
 # CHECK-NEXT:          "tokenTypes": [
 # CHECK-NEXT:            "variable",
 # CHECK:               ]
 # CHECK-NEXT:        },
-# CHECK-NEXT:        "rangeProvider": false
+# CHECK-NEXT:        "range": false
 # CHECK-NEXT:      },
 # CHECK-NEXT:      "signatureHelpProvider": {
 # CHECK-NEXT:        "triggerCharacters": [
 # CHECK-NEXT:          "(",
 # CHECK-NEXT:          ","
 # CHECK-NEXT:        ]
 # CHECK-NEXT:      },
 # CHECK-NEXT:      "textDocumentSync": {

clang-tools-extra/clangd/test/semantic-tokens.test

 # RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s -implicit-check-not=semanticHighlight
 # Send capabilities for both Theia semanticHighlight & standard semanticTokens.
 # clangd should not use/acknowledge the Theia protocol in this case.
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"capabilities":{"textDocument":{
   "semanticHighlightingCapabilities":{"semanticHighlighting":true},
   "semanticTokens":{"dynamicRegistration":true}
 }}}}
 ---
 {"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{
   "uri": "test:///foo.cpp",
   "languageId": "cpp",
   "text": "int x = 2;"
 }}}
 ---
 # Non-incremental token request.
-{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens","params":{"textDocument":{"uri":"test:///foo.cpp"}}}
+{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens/full","params":{"textDocument":{"uri":"test:///foo.cpp"}}}
 # CHECK:       "id": 1,
 # CHECK-NEXT:  "jsonrpc": "2.0",
 # CHECK-NEXT:  "result": {
 # CHECK-NEXT:    "data": [
 #                  First line, char 5, variable, no modifiers.
 # CHECK-NEXT:      0,
 # CHECK-NEXT:      4,
 # CHECK-NEXT:      1,
 # CHECK-NEXT:      0,
 # CHECK-NEXT:      0
 # CHECK-NEXT:    ],
 # CHECK-NEXT:    "resultId": "1"
 # CHECK-NEXT:  }
 ---
 {"jsonrpc":"2.0","method":"textDocument/didChange","params":{
   "textDocument": {"uri":"test:///foo.cpp","version":2},
   "contentChanges":[{"text":"int x = 2;\nint y = 3;"}]
 }}
 ---
 # Incremental token request, based on previous response.
-{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/edits","params":{
+{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/full/delta","params":{
   "textDocument": {"uri":"test:///foo.cpp"},
   "previousResultId": "1"
 }}
 # CHECK:       "id": 2,
 # CHECK-NEXT:  "jsonrpc": "2.0",
 # CHECK-NEXT:  "result": {
 # CHECK-NEXT:    "edits": [
 # CHECK-NEXT:      {

 # CHECK-NEXT:        "deleteCount": 0,
 # CHECK-NEXT:        "start": 5
 # CHECK-NEXT:      }
 # CHECK-NEXT:    ],
 # CHECK-NEXT:    "resultId": "2"
 # CHECK-NEXT:  }
 ---
 # Incremental token request with incorrect baseline => full tokens list.
-{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/edits","params":{
+{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/full/delta","params":{
   "textDocument": {"uri":"test:///foo.cpp"},
   "previousResultId": "bogus"
 }}
 # CHECK:       "id": 2,
 # CHECK-NEXT:  "jsonrpc": "2.0",
 # CHECK-NEXT:  "result": {
 # CHECK-NEXT:    "data": [
 # CHECK-NEXT:      0,