[clangd] Added the vscode SemanticHighlighting feature code but did not enable it in the client.

[jvikstrom]

Summary

Added the code for the StaticFeature that must be registered to the client. Also decoding the notification data into objects. Did not register it to the client yet.

[jvikstrom]

Added comment to decodeTokens function.

[hokein]

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

37

nit: name it SemanticHighlightingFeature.

38

I believe the type of lookup table returned from clangd is a string[][]. I'd name it scopeLookupTable

62

I think the params here is defined as SemanticHighlightingParams from the LSP proposal, could we use the names as defined in the LSP for these structures (e.g. SemanticHighlightingParams, SemanticHighlightingInformation)?

[jvikstrom]

Address comments.

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

62

Renaming HighlightingToken to SemanticHighlightingToken to be consistent with those names as well.

[hokein]

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

8

sorry for not being clear in the previous comment, I think we should mirror definitions from the LSP proposal, the same to the SemanticHighlightingInformation.

You may need to add a vscode-languageserver-types dependency.

32

I think here we should use SemanticHighlightingParams in the generic type parameter list

38

could you add a comment on this?

63

this seems doesn't do anything, can we defer it to a follow-up patch?

[jvikstrom]

Mirror the LSP proposal SemanticHighlightingParams/Information types.

[hokein]

mostly good with a few nits.

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

9

could you add a comment describing this is a mirror of LSP definition?

16

nit: I'd remove this blank line to make SemanticHighlightingParams and SemanticHighlightingInformation group closer as they are mirrors of LSP definitions.

27

the comment seems not precise to me, this is the data decoded from the base64-encoded string sent by clangd.

33

nit: scope lookup table.

34

nit: maybe name it scopeIndex.

clang-tools-extra/clangd/clients/clangd-vscode/src/semantic-highlighting.ts

 import * as fs from 'fs';
 import * as jsonc from "jsonc-parser";
 import * as path from 'path';
 import * as vscode from 'vscode';
+import * as vscodelc from 'vscode-languageclient';
+
+// The information clangd sends when highlightings should be updated.
+interface HighlightingInformation {
+  // The information about the text document where these highlightings should be
+  // applied.
+  textDocument: {uri: string;};
+  // All the lines of highlightings that should be applied.
+  lines: [ {
+    line: number;
+    tokens: string;
+  } ];
+}
+// A single HighlightingToken that clangd sent.
+interface HighlightingToken {
+  // Start column for this token.
+  character: number;
+  // The TextMate scope index to the clangd scopes.
+  scope: number;
+  length: number;
+}
+// All highlightings for line.
+interface HighlightingLine {
+  line: number;
+  tokens: HighlightingToken[];
+}
+
+export const NotificationType = new vscodelc.NotificationType<{}, void>(
+    'textDocument/semanticHighlighting');
+
+// The feature that should be registered in the vscode lsp for enabling
+// experimental semantic highlighting.
+export class Feature implements vscodelc.StaticFeature {
+  scopes: string[];
+  fillClientCapabilities(capabilities: vscodelc.ClientCapabilities) {
+    // Extend the ClientCapabilities type and add semantic highlighting
+    // capability to the object.
+    const textDocumentCapabilities: vscodelc.TextDocumentClientCapabilities&
+        {semanticHighlightingCapabilities?: {semanticHighlighting : boolean}} =
+        capabilities.textDocument;
+    textDocumentCapabilities.semanticHighlightingCapabilities = {
+      semanticHighlighting : true,
+    };
+  }
+
+  initialize(capabilities: vscodelc.ServerCapabilities,
+             documentSelector: vscodelc.DocumentSelector|undefined) {
+    // The semantic highlighting capability information is in the capabilities
+    // object but to access the data we must first extend the ServerCapabilities
+    // type.
+    const serverCapabilities: vscodelc.ServerCapabilities&
+        {semanticHighlighting?: {scopes : string[]}} = capabilities;
+    if (!serverCapabilities.semanticHighlighting)
+      return;
+    this.scopes = serverCapabilities.semanticHighlighting.scopes;
+  }
+
+  handleNotification(params: HighlightingInformation) {
+    const tokenLines = params.lines.map((line): HighlightingLine => {
+      return { line: line.line, tokens: decodeTokens(line.tokens), }
+    });
+  }
+}
+
+// Converts a string of base64 encoded tokens into the corresponding array of
+// HighlightingTokens.
+export function decodeTokens(tokens: string): HighlightingToken[] {
+  const scopeMask = 0xFFFF;
+  const lenShift = 0x10;
+  const uint32Size = 4;
+  const buf = Buffer.from(tokens, 'base64');
+  const retTokens = [];
+  for (let i = 0, end = buf.length / uint32Size; i < end; i += 2) {
+    const start = buf.readUInt32BE(i * uint32Size);
+    const lenKind = buf.readUInt32BE((i + 1) * uint32Size);
+    const scope = lenKind & scopeMask;
+    const len = lenKind >>> lenShift;
+    retTokens.push({character : start, scope : scope, length : len});
+  }
+
+  return retTokens;
+}
 
 // A rule for how to color TextMate scopes.
 interface TokenColorRule {
   // A TextMate scope that specifies the context of the token, e.g.
   // "entity.name.function.cpp".
   scope: string;
   // foreground is the color tokens of this scope should have.
   foreground: string;

clang-tools-extra/clangd/clients/clangd-vscode/test/semantic-highlighting.test.ts

 import * as assert from 'assert';
 import * as path from 'path';
 
 import * as TM from '../src/semantic-highlighting';
 
-suite('TextMate Tests', () => {
+suite('SemanticHighlighting Tests', () => {
   test('Parses arrays of textmate themes.', async () => {
     const themePath =
         path.join(__dirname, '../../test/assets/includeTheme.jsonc');
     const scopeColorRules = await TM.parseThemeFile(themePath);
     const getScopeRule = (scope: string) =>
         scopeColorRules.find((v) => v.scope === scope);
     assert.equal(scopeColorRules.length, 3);
     assert.deepEqual(getScopeRule('a'), {scope : 'a', textColor : '#fff'});
     assert.deepEqual(getScopeRule('b'), {scope : 'b', textColor : '#000'});
     assert.deepEqual(getScopeRule('c'), {scope : 'c', textColor : '#bcd'});
   });
+  test('Decodes tokens correctly', () => {
+    const testCases: string[] = [
+      'AAAAAAABAAA=', 'AAAAAAADAAkAAAAEAAEAAA==',
+      'AAAAAAADAAkAAAAEAAEAAAAAAAoAAQAA'
+    ];
+    const expected = [
+      [ {character : 0, scope : 0, length : 1} ],
+      [
+        {character : 0, scope : 9, length : 3},
+        {character : 4, scope : 0, length : 1}
+      ],
+      [
+        {character : 0, scope : 9, length : 3},
+        {character : 4, scope : 0, length : 1},
+        {character : 10, scope : 0, length : 1}
+      ]
+    ];
+    testCases.forEach((testCase, i) => assert.deepEqual(
+                          TM.decodeTokens(testCase), expected[i]));
+  });
 });