diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/.gitignore b/clang-tools-extra/clangd/clients/clangd-vscode/.gitignore --- a/clang-tools-extra/clangd/clients/clangd-vscode/.gitignore +++ b/clang-tools-extra/clangd/clients/clangd-vscode/.gitignore @@ -1,2 +1,3 @@ out node_modules +.vscode-test diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts b/clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/clients/clangd-vscode/src/SemanticHighlighting.ts @@ -0,0 +1,179 @@ +import * as vscode from 'vscode'; +import * as vscodelc from 'vscode-languageclient'; +import * as TM from './TextMate'; +import { timingSafeEqual } from 'crypto'; + +export namespace SemanticHighlighting { + interface HighlightingInformation { + textDocument: { + uri: string, + }, + lines: [{ + line: number, + tokens: string, + }], + } + interface HighlightingToken { + character: number, + scope: number, + length: number, + } + interface HighlightingLine { + line: number, + tokens: HighlightingToken[], + } + + export const NotificationType = new vscodelc.NotificationType<{}, void>('textDocument/semanticHighlighting'); + + // The feature that should be registered in the vscode lsp for enabling experimental semantic highlighting. + export class Feature implements vscodelc.StaticFeature { + scopes: string[]; + tokenCache: HighlightingCache = new HighlightingCache([]); + ready: boolean = false + fillClientCapabilities(capabilities: vscodelc.ClientCapabilities) { + // Must use an experimental capability as the other's are strongly typed. + const textDocumentCapabilities: vscodelc.TextDocumentClientCapabilities & { semanticHighlightingCapabilities?: { semanticHighlighting: boolean } } = capabilities.textDocument; + textDocumentCapabilities.semanticHighlightingCapabilities = { semanticHighlighting: true }; + } + + initialize(capabilities: vscodelc.ServerCapabilities, documentSelector: vscodelc.DocumentSelector | undefined) { + this.tokenCache.scopes = this.scopes; + // Get the scopes from an experimental capability for the same reason that it is registered as experimental. + const serverCapabilities: vscodelc.ServerCapabilities & { semanticHighlighting?: { scopes: string[] } } = capabilities; + if(!serverCapabilities.semanticHighlighting) + return; + this.scopes = serverCapabilities.semanticHighlighting.scopes; + TM.SemanticHighlighting.setupTMScopes(() => { + this.tokenCache.rerenderAll(); // Colors changed. + }).then(() => this.ready = true); + } + + handleNotification(params: HighlightingInformation) { + this.tokenCache.scopes = this.scopes; + console.log('Params: ', params); + const tokenLines = params.lines.map((line): HighlightingLine => { + return { + line: line.line, + tokens: decodeTokens(line.tokens), + } + }); + + if(this.ready) + this.tokenCache.setFileLines(vscode.Uri.parse(params.textDocument.uri), tokenLines); + else + console.log('Not ready to go yet...'); // TOOD: This is going to get fixed in a better way... + } + } + + function decodeTokens(tokens: string): HighlightingToken[] { + const scopeMask = 0xFFFF; + const lenShift = 0x10; + const buf = Buffer.from(tokens, 'base64'); + const retTokens = []; + for (let i = 0, end = buf.length / 4; i < end; i += 2) { + const start = buf.readUInt32BE(i * 4); + const lenKind = buf.readUInt32BE((i + 1) * 4); + const scope = lenKind & scopeMask; + const len = lenKind >>> lenShift; + retTokens.push({ character: start, scope: scope, length: len }); + } + + return retTokens; + } + + export class HighlightingCache { + files: Map> = new Map(); + renderers: Map = new Map(); + dispose: vscode.Disposable + scopes: string[] + constructor(scopes: string[]) { + this.scopes = scopes; + vscode.workspace.onDidCloseTextDocument((e) => this.onCloseFile(e.uri)); + vscode.window.onDidChangeVisibleTextEditors((e) => { + // Let's just rerender everything + this.rerenderAll(); + // And remove any texteditors that shouldn't be here + const actualVisible = e.map((e) => e.document.uri.toString()); + const keys = Array.from(this.renderers.keys()); + keys.forEach((key) => { + if(actualVisible.indexOf(key) == -1) + this.renderers.delete(key); + }); + }); + + } + + setFileLines(uri: vscode.Uri, tokens: HighlightingLine[]) { + const uriString = uri.toString(); + if(!this.files.has(uriString)) { + const m = new Map(); + tokens.forEach((line) => m.set(line.line, line.tokens)); + this.files.set(uriString, m); + } + if(!this.renderers.has(uriString)) { + const rend = new HighlightingRenderer(uri, this.scopes); + this.renderers.set(uriString, rend); + rend.render(tokens); + return; + } + const m = this.files.get(uriString); + tokens.forEach((line) => m.set(line.line, line.tokens)); + this.renderers.get(uriString).render(tokens); + } + + rerenderAll() { + vscode.window.visibleTextEditors.forEach((e) => { + const uriString = e.document.uri.toString(); + const cached: HighlightingLine[] = []; + if(!this.files.has(uriString)) { + console.error('File does not exit?'); + return; + } + this.files.get(uriString).forEach((toks, line) => cached.push({line: line, tokens: toks})); + if(!this.renderers.has(uriString)) { + this.renderers.set(uriString, new HighlightingRenderer(e.document.uri, this.scopes)); + } + this.renderers.get(uriString).render(cached); + }); + } + onCloseFile(uri: vscode.Uri) { + this.renderers.delete(uri.toString()); + this.files.delete(uri.toString()); + } + } + + // Renders highlightings. + class HighlightingRenderer { + decorations: Map = new Map(); + uri: vscode.Uri + tmColors: TM.SemanticHighlighting.TextMateColorProvider = TM.SemanticHighlighting.TextMateColorProvider.get() + scopes: string[] + constructor(uri: vscode.Uri, scopes: string[]) { + this.uri = uri; + this.scopes = scopes; + } + render(lines: HighlightingLine[]) { + const editor = vscode.window.visibleTextEditors.find(e => e.document.uri.toString() == this.uri.toString()); + if(!editor) + return; // The TextEditor this renderer belongs to is not currently visible. + lines.forEach(line => { + const old = this.decorations.get(line.line); + const newLineDecorations = line.tokens.map(tok => { + const options: vscode.DecorationRenderOptions = { + // FIXME: This is a race condition. + color: this.tmColors.getColors().getColor(this.scopes[tok.scope][0]) ,//'#ff0000', + rangeBehavior: vscode.DecorationRangeBehavior.ClosedClosed, + }; + const decoration = vscode.window.createTextEditorDecorationType(options); + editor.setDecorations(decoration, [new vscode.Range(new vscode.Position(line.line, tok.character), new vscode.Position(line.line, tok.character + tok.length))]); + return decoration; + }); + // Must clear any previous highlightings on this line or vscode will keep stale highlightings. Does this after the new highlightings have been added so there aren't weird gaps of time where there aren't any highlightings. + if(old) + old.forEach(o => o.dispose()); + this.decorations.set(line.line, newLineDecorations); + }); + } + } + +}; diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts b/clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/clients/clangd-vscode/src/TextMate.ts @@ -0,0 +1,151 @@ +import * as vscode from 'vscode'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as jsonc from "jsonc-parser"; +import { SSL_OP_NETSCAPE_CA_DN_BUG } from 'constants'; + +export namespace SemanticHighlighting { + class TMColors { + private colors: Map = new Map(); + private divided: Map = new Map(); + setColor(scope: string | Array, color: string) { + if (scope instanceof Array) { + scope.forEach((s: string) => this.setColor(s, color)); + return; + } + + this.colors.set(scope, color); + const split = scope.split('.'); + while(split.length > 0) { + split.splice(-1) + const joined = split.join('.'); + this.divided.set(joined, color); + } + } + + getColor(scope: string): string { + if(this.colors.has(scope)) + return this.colors.get(scope); + // Here we should find the TM scope (should also probably preprocess indexes -> colors) + const split = scope.split('.'); + while(split.length > 0) { + split.splice(-1); + const joined = split.join('.'); + if(this.divided.has(joined)) { + return this.divided.get(joined); + } + + } + return '#ff0000'; + } + } + + // Singleton for reading/writing TM scopes/colors. Emits an event with new TMColors whenever the colors changes. + export class TextMateColorProvider extends vscode.EventEmitter { + // Provides a mapping from TM scopes -> colors + private static instance: TextMateColorProvider = new TextMateColorProvider(); + private constructor() { + super(); + } + private colors: TMColors = undefined; + static get() { + return TextMateColorProvider.instance; + } + + setColors(colors: TMColors) { + this.colors = colors; + this.fire(colors); + } + getColors(): TMColors { + return this.colors; + } + } + + // Returns a dispose function to a listener that listens for changes to the TM scopes. + export async function setupTMScopes(cb: Function): Promise { + async function setTMColors() { + const name = vscode.workspace.getConfiguration('workbench').get('colorTheme'); + if(typeof name != 'string') { + console.warn('The current theme name is not a string, is: ' + (typeof name) + ', value: ', name); + return; + } + + const tm = await getTextMateColors(name as string); + TextMateColorProvider.get().setColors(tm); + } + + // Initialize the TM scopes and colors. + await setTMColors(); + + // Listen for changes to the theme and reload the TM scopes and colors if it changes. + return vscode.workspace.onDidChangeConfiguration((conf: vscode.ConfigurationChangeEvent) => { + if (conf.affectsConfiguration('workbench')) + // Configuration affected the workbench meaning the current theme might have changed. + setTMColors().then(() => cb()); + }); + } + + // Gets a TM theme with themeName and returns class with a mapping from TM scopes to colors. + async function getTextMateColors(themeName: string): Promise { + const fileContents = await getFullNamedTheme(themeName); + const tmColors = new TMColors(); + fileContents.forEach((content) => { + if (!content.tokenColors) + return; + content.tokenColors.forEach((rule: any) => { + if(!rule.scope || !rule.settings || !rule.settings.foreground) + return; + + tmColors.setColor(rule.scope, rule.settings.foreground); + }); + }); + + return tmColors; + } + + // Gets a TextMate theme and all its included themes. + async function getFullNamedTheme(themeName: string): Promise { + const extension = vscode.extensions.all.find((extension: vscode.Extension) => { + const contribs = extension.packageJSON.contributes; + if (!contribs || !contribs.themes) + return false; + return contribs.themes.some((theme: any) => theme.id === themeName || theme.label === themeName); + }); + + if (!extension) { + // Could not find an extension with the specified themeName. + return Promise.reject('Could not find a theme with name: ' + themeName); + } + + const extensionInfo = extension.packageJSON.contributes.themes.find((theme: any) => theme.id === themeName || theme.label === themeName); + return recursiveGetTextMateGrammarPath(path.join(extension.extensionPath, extensionInfo.path)); + } + + // TM grammars can include other TM grammars, this function recursively gets all of them. + async function recursiveGetTextMateGrammarPath(fullPath: string): Promise { + // If there is an error opening a file the TM files that were correctly found further up the chain should be returned. + try { + const contents = await readFileText(fullPath); + const parsed = jsonc.parse(contents); + if (parsed.include) + // Get all includes and merge into a flat list of parsed json. + return [...(await recursiveGetTextMateGrammarPath(path.join(path.dirname(fullPath), parsed.include))), parsed]; + return [parsed]; + } catch (err) { + console.warn('Could not open file: ' + fullPath + ', error: ', err); + } + + return []; + } + + function readFileText(path: string): Promise { + return new Promise((res, rej) => { + fs.readFile(path, 'utf8', (err, data) => { + if (err) { + rej(err); + } + res(data); + }); + }); + } +} diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts b/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts --- a/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts +++ b/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts @@ -1,5 +1,6 @@ import * as vscode from 'vscode'; import * as vscodelc from 'vscode-languageclient'; +import { SemanticHighlighting } from './SemanticHighlighting'; /** * Method to get workspace configuration option @@ -12,9 +13,9 @@ } namespace SwitchSourceHeaderRequest { -export const type = - new vscodelc.RequestType('textDocument/switchSourceHeader'); + export const type = + new vscodelc.RequestType('textDocument/switchSourceHeader'); } class FileStatus { @@ -32,8 +33,8 @@ const path = vscode.window.activeTextEditor.document.fileName; const status = this.statuses.get(path); if (!status) { - this.statusBarItem.hide(); - return; + this.statusBarItem.hide(); + return; } this.statusBarItem.text = `clangd: ` + status.state; this.statusBarItem.show(); @@ -73,25 +74,30 @@ // However, VSCode does not have CUDA as a supported language yet, so we // cannot add a corresponding activationEvent for CUDA files and clangd will // *not* load itself automatically on '.cu' files. - const cudaFilePattern: string = '**/*.{' +['cu'].join()+ '}'; + const cudaFilePattern: string = '**/*.{' + ['cu'].join() + '}'; const clientOptions: vscodelc.LanguageClientOptions = { // Register the server for c-family and cuda files. documentSelector: [ { scheme: 'file', language: 'c' }, { scheme: 'file', language: 'cpp' }, - { scheme: 'file', language: 'objective-c'}, - { scheme: 'file', language: 'objective-cpp'}, + { scheme: 'file', language: 'objective-c' }, + { scheme: 'file', language: 'objective-cpp' }, { scheme: 'file', pattern: cudaFilePattern }, ], synchronize: !syncFileEvents ? undefined : { - // FIXME: send sync file events when clangd provides implemenatations. + // FIXME: send sync file events when clangd provides implemenatations. }, initializationOptions: { clangdFileStatus: true }, // Do not switch to output window when clangd returns output revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never }; - const clangdClient = new vscodelc.LanguageClient('Clang Language Server',serverOptions, clientOptions); + const clangdClient = new vscodelc.LanguageClient('Clang Language Server', serverOptions, clientOptions); + const semanticHighlightingFeature = new SemanticHighlighting.Feature(); + clangdClient.registerFeature(semanticHighlightingFeature); + // The notification handler must be registered after the client is ready or the client will crash. + clangdClient.onReady().then(() => clangdClient.onNotification(SemanticHighlighting.NotificationType, semanticHighlightingFeature.handleNotification.bind(semanticHighlightingFeature))); + console.log('Clang Language Server is now active!'); context.subscriptions.push(clangdClient.start()); context.subscriptions.push(vscode.commands.registerCommand( @@ -131,5 +137,5 @@ // An empty place holder for the activate command, otherwise we'll get an // "command is not registered" error. context.subscriptions.push(vscode.commands.registerCommand( - 'clangd-vscode.activate', async () => {})); + 'clangd-vscode.activate', async () => { })); } diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts b/clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts --- a/clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts +++ b/clang-tools-extra/clangd/clients/clangd-vscode/test/extension.test.ts @@ -2,14 +2,30 @@ import * as assert from 'assert'; import * as vscode from 'vscode'; -import * as myExtension from '../src/extension'; +import {SemanticHighlighting} from '../src/SemanticHighlighting'; -// TODO: add tests suite("Extension Tests", () => { - - // Defines a Mocha unit test - test("Something 1", () => { - assert.equal(-1, [1, 2, 3].indexOf(5)); - assert.equal(-1, [1, 2, 3].indexOf(0)); + test('HighlightingCache caches incrementaly', () => { + const feature = new SemanticHighlighting.Feature(); + const uri = 'file://text'; + feature.handleNotification({ + textDocument: {uri: uri}, + lines: [{ + line: 1, + tokens: 'AAAABAABAAA=', + }], + }); + feature.handleNotification({ + textDocument: {uri: uri}, + lines: [{ + line: 2, + tokens: 'AAAABAABAAA=', + }], + }); +// const tokens = feature.tokenCache.files.get(vscode.Uri.parse(uri).toString()).tokens; +// assert.deepEqual(tokens.get(1), [{character: 4, length: 1, scope: 0}]); +// assert.deepEqual(tokens.get(2), [{character: 4, length: 1, scope: 0}]); + + }); }); \ No newline at end of file