[clangd] Added the vscode SemanticHighlighting feature code but did not enable it in the client.

[jvikstrom]

Summary

Added the code for the StaticFeature that must be registered to the client. Also decoding the notification data into objects. Did not register it to the client yet.

[jvikstrom]

Added comment to decodeTokens function.

[hokein]

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

37 ↗

(On Diff #214334)

nit: name it SemanticHighlightingFeature.

38 ↗

(On Diff #214334)

I believe the type of lookup table returned from clangd is a string[][]. I'd name it scopeLookupTable

62 ↗

(On Diff #214334)

I think the params here is defined as SemanticHighlightingParams from the LSP proposal, could we use the names as defined in the LSP for these structures (e.g. SemanticHighlightingParams, SemanticHighlightingInformation)?

[jvikstrom]

Address comments.

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

62 ↗

(On Diff #214334)

Renaming HighlightingToken to SemanticHighlightingToken to be consistent with those names as well.

[hokein]

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

8 ↗

(On Diff #214364)

sorry for not being clear in the previous comment, I think we should mirror definitions from the LSP proposal, the same to the SemanticHighlightingInformation.

You may need to add a vscode-languageserver-types dependency.

32 ↗

(On Diff #214364)

I think here we should use SemanticHighlightingParams in the generic type parameter list

38 ↗

(On Diff #214364)

could you add a comment on this?

63 ↗

(On Diff #214364)

this seems doesn't do anything, can we defer it to a follow-up patch?

[jvikstrom]

Mirror the LSP proposal SemanticHighlightingParams/Information types.

[hokein]

mostly good with a few nits.

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

9 ↗

(On Diff #214613)

could you add a comment describing this is a mirror of LSP definition?

16 ↗

(On Diff #214613)

nit: I'd remove this blank line to make SemanticHighlightingParams and SemanticHighlightingInformation group closer as they are mirrors of LSP definitions.

27 ↗

(On Diff #214613)

the comment seems not precise to me, this is the data decoded from the base64-encoded string sent by clangd.

33 ↗

(On Diff #214613)

nit: scope lookup table.

34 ↗

(On Diff #214613)

nit: maybe name it scopeIndex.

clang-tools-extra/trunk/clangd/clients/clangd-vscode/package.json

"scripts": {
         "compile": "tsc -watch -p ./",
         "postinstall": "node ./node_modules/vscode/bin/install",
         "format": "clang-format --style=LLVM -i --glob=\"{src,test}/*.ts\"",
         "test": "node ./node_modules/vscode/bin/test"
     },
     "dependencies": {
         "jsonc-parser": "^2.1.0",
         "vscode-languageclient": "^5.3.0-next.6",
-        "vscode-languageserver": "^5.3.0-next.6"
+        "vscode-languageserver": "^5.3.0-next.6",
+        "vscode-languageserver-types": "^3.14.0"
     },
     "devDependencies": {
         "@types/mocha": "^2.2.32",
         "@types/node": "^6.0.40",
         "clang-format": "1.2.4",
         "mocha": "^5.2.0",
         "typescript": "^2.0.3",
         "vscode": "^1.1.0"

clang-tools-extra/trunk/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

 import * as fs from 'fs';
 import * as jsonc from "jsonc-parser";
 import * as path from 'path';
 import * as vscode from 'vscode';
+import * as vscodelc from 'vscode-languageclient';
+import * as vscodelct from 'vscode-languageserver-types';
+
+// Parameters for the semantic highlighting (server-side) push notification.
+// Mirrors the structure in the semantic highlighting proposal for LSP.
+interface SemanticHighlightingParams {
+  // The text document that has to be decorated with the semantic highlighting
+  // information.
+  textDocument: vscodelct.VersionedTextDocumentIdentifier;
+  // An array of semantic highlighting information.
+  lines: SemanticHighlightingInformation[];
+}
+// Contains the highlighting information for a specified line. Mirrors the
+// structure in the semantic highlighting proposal for LSP.
+interface SemanticHighlightingInformation {
+  // The zero-based line position in the text document.
+  line: number;
+  // A base64 encoded string representing every single highlighted characters
+  // with its start position, length and the "lookup table" index of of the
+  // semantic highlighting Text Mate scopes.
+  tokens?: string;
+}
+
+// A SemanticHighlightingToken decoded from the base64 data sent by clangd.
+interface SemanticHighlightingToken {
+  // Start column for this token.
+  character: number;
+  // Length of the token.
+  length: number;
+  // The TextMate scope index to the clangd scope lookup table.
+  scopeIndex: number;
+}
+
+// Language server push notification providing the semantic highlighting
+// information for a text document.
+export const NotificationType =
+    new vscodelc.NotificationType<SemanticHighlightingParams, void>(
+        'textDocument/semanticHighlighting');
+
+// The feature that should be registered in the vscode lsp for enabling
+// experimental semantic highlighting.
+export class SemanticHighlightingFeature implements vscodelc.StaticFeature {
+  // The TextMate scope lookup table. A token with scope index i has the scopes
+  // on index i in the lookup table.
+  scopeLookupTable: string[][];
+  fillClientCapabilities(capabilities: vscodelc.ClientCapabilities) {
+    // Extend the ClientCapabilities type and add semantic highlighting
+    // capability to the object.
+    const textDocumentCapabilities: vscodelc.TextDocumentClientCapabilities&
+        {semanticHighlightingCapabilities?: {semanticHighlighting : boolean}} =
+        capabilities.textDocument;
+    textDocumentCapabilities.semanticHighlightingCapabilities = {
+      semanticHighlighting : true,
+    };
+  }
+
+  initialize(capabilities: vscodelc.ServerCapabilities,
+             documentSelector: vscodelc.DocumentSelector|undefined) {
+    // The semantic highlighting capability information is in the capabilities
+    // object but to access the data we must first extend the ServerCapabilities
+    // type.
+    const serverCapabilities: vscodelc.ServerCapabilities&
+        {semanticHighlighting?: {scopes : string[][]}} = capabilities;
+    if (!serverCapabilities.semanticHighlighting)
+      return;
+    this.scopeLookupTable = serverCapabilities.semanticHighlighting.scopes;
+  }
+
+  handleNotification(params: SemanticHighlightingParams) {}
+}
+
+// Converts a string of base64 encoded tokens into the corresponding array of
+// HighlightingTokens.
+export function decodeTokens(tokens: string): SemanticHighlightingToken[] {
+  const scopeMask = 0xFFFF;
+  const lenShift = 0x10;
+  const uint32Size = 4;
+  const buf = Buffer.from(tokens, 'base64');
+  const retTokens = [];
+  for (let i = 0, end = buf.length / uint32Size; i < end; i += 2) {
+    const start = buf.readUInt32BE(i * uint32Size);
+    const lenKind = buf.readUInt32BE((i + 1) * uint32Size);
+    const scopeIndex = lenKind & scopeMask;
+    const len = lenKind >>> lenShift;
+    retTokens.push({character : start, scopeIndex : scopeIndex, length : len});
+  }
+
+  return retTokens;
+}
 
 // A rule for how to color TextMate scopes.
 interface TokenColorRule {
   // A TextMate scope that specifies the context of the token, e.g.
   // "entity.name.function.cpp".
   scope: string;
   // foreground is the color tokens of this scope should have.
   foreground: string;

clang-tools-extra/trunk/clangd/clients/clangd-vscode/test/semantic-highlighting.test.ts

 import * as assert from 'assert';
 import * as path from 'path';
 
 import * as TM from '../src/semantic-highlighting';
 
-suite('TextMate Tests', () => {
+suite('SemanticHighlighting Tests', () => {
   test('Parses arrays of textmate themes.', async () => {
     const themePath =
         path.join(__dirname, '../../test/assets/includeTheme.jsonc');
     const scopeColorRules = await TM.parseThemeFile(themePath);
     const getScopeRule = (scope: string) =>
         scopeColorRules.find((v) => v.scope === scope);
     assert.equal(scopeColorRules.length, 3);
-    assert.deepEqual(getScopeRule('a'), {scope : 'a', textColor : '#fff'});
-    assert.deepEqual(getScopeRule('b'), {scope : 'b', textColor : '#000'});
-    assert.deepEqual(getScopeRule('c'), {scope : 'c', textColor : '#bcd'});
+    assert.deepEqual(getScopeRule('a'), {scope : 'a', foreground : '#fff'});
+    assert.deepEqual(getScopeRule('b'), {scope : 'b', foreground : '#000'});
+    assert.deepEqual(getScopeRule('c'), {scope : 'c', foreground : '#bcd'});
+  });
+  test('Decodes tokens correctly', () => {
+    const testCases: string[] = [
+      'AAAAAAABAAA=', 'AAAAAAADAAkAAAAEAAEAAA==',
+      'AAAAAAADAAkAAAAEAAEAAAAAAAoAAQAA'
+    ];
+    const expected = [
+      [ {character : 0, scopeIndex : 0, length : 1} ],
+      [
+        {character : 0, scopeIndex : 9, length : 3},
+        {character : 4, scopeIndex : 0, length : 1}
+      ],
+      [
+        {character : 0, scopeIndex : 9, length : 3},
+        {character : 4, scopeIndex : 0, length : 1},
+        {character : 10, scopeIndex : 0, length : 1}
+      ]
+    ];
+    testCases.forEach((testCase, i) => assert.deepEqual(
+                          TM.decodeTokens(testCase), expected[i]));
   });
 });