Merge pull request #3667 from JoeRobich/semantic-highlight

Add experimental Semantic Highlighter

.travis.yml

@@ -7,7 +7,7 @@ mono:
   - latest
 
 env:
-- CODE_VERSION=1.36.0
+- CODE_VERSION=1.45.0
 
 before_install:
   - if [ $TRAVIS_OS_NAME == "linux" ]; then

src/features/commands.ts

@@ -23,6 +23,7 @@ import reportIssue from './reportIssue';
 import { IMonoResolver } from '../constants/IMonoResolver';
 import { getDotnetInfo } from '../utils/getDotnetInfo';
 import { getDecompilationAuthorization } from '../omnisharp/decompilationPrompt';
+import { getSemanticTokensProvider } from '../omnisharp/extension';
 
 export default function registerCommands(context: vscode.ExtensionContext, server: OmniSharpServer, platformInfo: PlatformInformation, eventStream: EventStream, optionProvider: OptionProvider, monoResolver: IMonoResolver, packageJSON: any, extensionPath: string): CompositeDisposable {
     let disposable = new CompositeDisposable();
@@ -57,6 +58,12 @@ export default function registerCommands(context: vscode.ExtensionContext, serve
 
     disposable.add(vscode.commands.registerCommand('csharp.showDecompilationTerms', async () => showDecompilationTerms(context, server, optionProvider)));
 
+    if (process.env.OSVC_SUITE !== undefined) {
+        // Register commands used for integration tests.
+        disposable.add(vscode.commands.registerCommand('csharp.private.getSemanticTokensLegend', async () => getSemanticTokensLegend()));
+        disposable.add(vscode.commands.registerCommand('csharp.private.getSemanticTokens', async (fileUri) => await getSemanticTokens(fileUri)));
+    }
+
     return new CompositeDisposable(disposable);
 }
 
@@ -177,6 +184,15 @@ async function getProjectDescriptors(server: OmniSharpServer): Promise<protocol.
     return descriptors;
 }
 
+function getSemanticTokensLegend() {
+    return getSemanticTokensProvider().getLegend();
+}
+
+async function getSemanticTokens(fileUri: vscode.Uri) {
+    const document = await vscode.workspace.openTextDocument(fileUri);
+    return await getSemanticTokensProvider().provideDocumentSemanticTokens(document, null);
+}
+
 export async function dotnetRestore(cwd: string, eventStream: EventStream, filePath?: string): Promise<void> {
     return new Promise<void>((resolve, reject) => {
         let cmd = 'dotnet';

src/omnisharp/extension.ts

@@ -39,12 +39,18 @@ import { StructureProvider } from '../features/structureProvider';
 import { OmniSharpMonoResolver } from './OmniSharpMonoResolver';
 import { getMonoVersion } from '../utils/getMonoVersion';
 import { LanguageMiddlewareFeature } from './LanguageMiddlewareFeature';
+import SemanticTokensProvider from '../features/semanticTokensProvider';
 
 export interface ActivationResult {
     readonly server: OmniSharpServer;
     readonly advisor: Advisor;
 }
 
+let _semanticTokensProvider: SemanticTokensProvider;
+export function getSemanticTokensProvider() {
+    return _semanticTokensProvider;
+}
+
 export async function activate(context: vscode.ExtensionContext, packageJSON: any, platformInfo: PlatformInformation, provider: NetworkSettingsProvider, eventStream: EventStream, optionProvider: OptionProvider, extensionPath: string) {
     const documentSelector: vscode.DocumentSelector = {
         language: 'csharp',
@@ -93,6 +99,15 @@ export async function activate(context: vscode.ExtensionContext, packageJSON: an
         localDisposables.add(forwardChanges(server));
         localDisposables.add(trackVirtualDocuments(server, eventStream));
         localDisposables.add(vscode.languages.registerFoldingRangeProvider(documentSelector, new StructureProvider(server, languageMiddlewareFeature)));
+
+        const semanticTokensProvider = new SemanticTokensProvider(server, optionProvider, languageMiddlewareFeature);
+        // Make the semantic token provider available for testing
+        if (process.env.OSVC_SUITE !== undefined) {
+            _semanticTokensProvider = semanticTokensProvider;
+        }
+
+        localDisposables.add(vscode.languages.registerDocumentSemanticTokensProvider(documentSelector, semanticTokensProvider, semanticTokensProvider.getLegend()));
+        localDisposables.add(vscode.languages.registerDocumentRangeSemanticTokensProvider(documentSelector, semanticTokensProvider, semanticTokensProvider.getLegend()));
     }));
 
     disposables.add(server.onServerStop(() => {

src/omnisharp/options.ts

@@ -28,6 +28,7 @@ export class Options {
         public enableRoslynAnalyzers: boolean,
         public enableEditorConfigSupport: boolean,
         public enableDecompilationSupport: boolean,
+        public useSemanticHighlighting: boolean,
         public razorPluginPath?: string,
         public defaultLaunchSolution?: string,
         public monoPath?: string,
@@ -75,6 +76,8 @@ export class Options {
         const showReferencesCodeLens = csharpConfig.get<boolean>('referencesCodeLens.enabled', true);
         const showTestsCodeLens = csharpConfig.get<boolean>('testsCodeLens.enabled', true);
 
+        const useSemanticHighlighting = csharpConfig.get<boolean>('semanticHighlighting.enabled', false);
+
         const disableCodeActions = csharpConfig.get<boolean>('disableCodeActions', false);
 
         const disableMSBuildDiagnosticWarning = omnisharpConfig.get<boolean>('disableMSBuildDiagnosticWarning', false);
@@ -114,6 +117,7 @@ export class Options {
             enableRoslynAnalyzers,
             enableEditorConfigSupport,
             enableDecompilationSupport,
+            useSemanticHighlighting,
             razorPluginPath,
             defaultLaunchSolution,
             monoPath,

src/omnisharp/protocol.ts

@@ -483,6 +483,24 @@ export namespace V2 {
         export const DebugTestStop = '/v2/debugtest/stop';
         export const BlockStructure = '/v2/blockstructure';
         export const CodeStructure = '/v2/codestructure';
+        export const Highlight = '/v2/highlight';
+    }
+
+    export interface SemanticHighlightSpan {
+        StartLine: number;
+        StartColumn: number;
+        EndLine: number;
+        EndColumn: number;
+        Type: number;
+        Modifiers: number[];
+    }
+
+    export interface SemanticHighlightRequest extends Request {
+        Range?: Range;
+    }
+
+    export interface SemanticHighlightResponse {
+        Spans: SemanticHighlightSpan[];
     }
 
     export interface Point {

src/omnisharp/utils.ts

@@ -151,6 +151,10 @@ export async function debugTestStop(server: OmniSharpServer, request: protocol.V
     return server.makeRequest<protocol.V2.DebugTestStopResponse>(protocol.V2.Requests.DebugTestStop, request);
 }
 
+export async function getSemanticHighlights(server: OmniSharpServer, request: protocol.V2.SemanticHighlightRequest) {
+    return server.makeRequest<protocol.V2.SemanticHighlightResponse>(protocol.V2.Requests.Highlight, request);
+}
+
 export async function isNetCoreProject(project: protocol.MSBuildProject) {
     return project.TargetFrameworks.find(tf => tf.ShortName.startsWith('netcoreapp') || tf.ShortName.startsWith('netstandard')) !== undefined;
 }

src/statusBarItemAdapter.ts

@@ -40,11 +40,11 @@ export class StatusBarItemAdapter implements vscodeAdapter.StatusBarItem {
         this.statusBarItem.color = value;
     }
 
-    get command(): string {
+    get command(): string | vscode.Command {
         return this.statusBarItem.command;
     }
 
-    set command(value: string) {
+    set command(value: string | vscode.Command) {
         this.statusBarItem.command = value;
     }
 

src/vscodeAdapter.ts

@@ -194,6 +194,34 @@ export enum StatusBarAlignment {
     Right = 2
 }
 
+/**
+ * Represents a reference to a command. Provides a title which
+ * will be used to represent a command in the UI and, optionally,
+ * an array of arguments which will be passed to the command handler
+ * function when invoked.
+ */
+export interface Command {
+    /**
+     * Title of the command, like `save`.
+     */
+    title: string;
+
+    /**
+     * The identifier of the actual command handler.
+     */
+    command: string;
+
+    /**
+     * A tooltip for the command, when represented in the UI.
+     */
+    tooltip?: string;
+
+    /**
+     * Arguments that the command handler should be
+     * invoked with.
+     */
+    arguments?: any[];
+}
 
 export interface StatusBarItem {
 
@@ -232,7 +260,7 @@ export interface StatusBarItem {
      * The identifier of a command to run on click. The command must be
      * [known](#commands.getCommands).
      */
-    command: string | undefined;
+    command: string | Command | undefined;
 
     /**
      * Shows the entry in the status bar.

test/integrationTests/launchConfiguration.integration.test.ts

@@ -38,7 +38,8 @@ suite(`Tasks generation: ${testAssetWorkspace.description}`, function () {
 
     test("Starting .NET Core Launch (console) from the workspace root should create an Active Debug Session", async () => {
 
-        vscode.debug.onDidChangeActiveDebugSession((e) => {
+        const onChangeSubscription = vscode.debug.onDidChangeActiveDebugSession((e) => {
+            onChangeSubscription.dispose();
             expect(vscode.debug.activeDebugSession).not.to.be.undefined;
             expect(vscode.debug.activeDebugSession.type).to.equal("coreclr");
         });
@@ -47,7 +48,10 @@ suite(`Tasks generation: ${testAssetWorkspace.description}`, function () {
         expect(result, "Debugger could not be started.");
 
         let debugSessionTerminated = new Promise(resolve => {
-            vscode.debug.onDidTerminateDebugSession((e) => resolve());
+            const onTerminateSubscription = vscode.debug.onDidTerminateDebugSession((e) => {
+                onTerminateSubscription.dispose();
+                resolve();
+            });
         });
 
         await debugSessionTerminated;

test/integrationTests/poll.ts

@@ -44,7 +44,7 @@ export async function assertWithPoll<T>(
 }
 
 function defaultPollExpression<T>(value: T): boolean {
-    return value !== undefined && ((Array.isArray(value) && value.length > 0) || !Array.isArray(value));
+    return value !== undefined && ((Array.isArray(value) && value.length > 0) || (!Array.isArray(value) && !!value));
 }
 
 export async function pollDoesNotHappen<T>(
@@ -68,7 +68,7 @@ export async function pollDoesNotHappen<T>(
 }
 
 export async function poll<T>(
-    getValue: () => T,
+    getValue: () => Promise<T> | T,
     duration: number,
     step: number,
     expression: (input: T) => boolean = defaultPollExpression): Promise<T> {

test/integrationTests/semanticTokensProvider.test.ts

@@ -0,0 +1,135 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as path from 'path';
+
+import { should, assert } from 'chai';
+import { activateCSharpExtension, isRazorWorkspace } from './integrationHelpers';
+import testAssetWorkspace from './testAssets/testAssetWorkspace';
+import { EventType } from '../../src/omnisharp/EventType';
+import { poll } from './poll';
+
+const chai = require('chai');
+chai.use(require('chai-arrays'));
+chai.use(require('chai-fs'));
+
+interface ExpectedToken {
+    startLine: number;
+    character: number;
+    length: number;
+    tokenClassifiction: string;
+}
+
+async function assertTokens(fileUri: vscode.Uri, expected: ExpectedToken[] | null, message?: string): Promise<void> {
+
+    const legend = <vscode.SemanticTokensLegend>await vscode.commands.executeCommand("csharp.private.getSemanticTokensLegend");
+    const actual = <vscode.SemanticTokens>await vscode.commands.executeCommand("csharp.private.getSemanticTokens", fileUri);
+
+    if (actual === null) {
+        assert.isNull(expected, message);
+        return;
+    }
+
+    let actualRanges = [];
+    let lastLine = 0;
+    let lastCharacter = 0;
+    for (let i = 0; i < actual.data.length; i += 5) {
+        const lineDelta = actual.data[i], charDelta = actual.data[i + 1], len = actual.data[i + 2], typeIdx = actual.data[i + 3], modSet = actual.data[i + 4];
+        const line = lastLine + lineDelta;
+        const character = lineDelta === 0 ? lastCharacter + charDelta : charDelta;
+        const tokenClassifiction = [legend.tokenTypes[typeIdx], ...legend.tokenModifiers.filter((_, i) => modSet & 1 << i)].join('.');
+        actualRanges.push(t(line, character, len, tokenClassifiction));
+        lastLine = line;
+        lastCharacter = character;
+    }
+    assert.deepEqual(actualRanges, expected, message);
+}
+
+suite(`SemanticTokensProvider: ${testAssetWorkspace.description}`, function () {
+    let fileUri: vscode.Uri;
+
+    suiteSetup(async function () {
+        should();
+
+        // These tests don't run on the BasicRazorApp2_1 solution
+        if (isRazorWorkspace(vscode.workspace)) {
+            this.skip();
+        }
+
+        const activation = await activateCSharpExtension();
+        await testAssetWorkspace.restore();
+
+        // Wait for workspace information to be returned
+        let isWorkspaceLoaded = false;
+
+        const subscription = activation.eventStream.subscribe(event => {
+            if (event.type === EventType.WorkspaceInformationUpdated) {
+                isWorkspaceLoaded = true;
+                subscription.unsubscribe();
+            }
+        });
+
+        await poll(() => isWorkspaceLoaded, 25000, 500);
+
+        const fileName = 'semantictokens.cs';
+        const projectDirectory = testAssetWorkspace.projects[0].projectDirectoryPath;
+
+        fileUri = vscode.Uri.file(path.join(projectDirectory, fileName));
+    });
+
+    test('Semantic Highlighting returns null when disabled', async () => {
+        let csharpConfig = vscode.workspace.getConfiguration('csharp');
+        await csharpConfig.update('semanticHighlighting.enabled', false, vscode.ConfigurationTarget.Global);
+
+        await assertTokens(fileUri, /*expected*/ null);
+    });
+
+    test('Semantic Highlighting returns classified tokens when enabled', async () => {
+        let csharpConfig = vscode.workspace.getConfiguration('csharp');
+        await csharpConfig.update('semanticHighlighting.enabled', true, vscode.ConfigurationTarget.Global);
+
+        await assertTokens(fileUri, [
+            // 0:namespace Test
+            _keyword("namespace", 0, 0), _namespace("Test", 0, 10),
+            // 1:{
+            _punctuation("{", 1, 0),
+            // 2:    public class TestProgram
+            _keyword("public", 2, 4), _keyword("class", 2, 11), _class("TestProgram", 2, 17),
+            // 3:    {
+            _punctuation("{", 3, 4),
+            // 4:        public static int TestMain(string[] args)
+            _keyword("public", 4, 8), _keyword("static", 4, 15), _keyword("int", 4, 22), _staticMethod("TestMain", 4, 26), _punctuation("(", 4, 34), _keyword("string", 4, 35), _punctuation("[", 4, 41), _punctuation("]", 4, 42), _parameter("args", 4, 44), _punctuation(")", 4, 48),
+            // 5:        {
+            _punctuation("{", 5, 8),
+            // 6:            System.Console.WriteLine(string.Join(',', args));
+            _namespace("System", 6, 12), _operator(".", 6, 18), _staticClass("Console", 6, 19), _operator(".", 6, 26), _staticMethod("WriteLine", 6, 27), _punctuation("(", 6, 36), _keyword("string", 6, 37), _operator(".", 6, 43), _staticMethod("Join", 6, 44), _punctuation("(", 6, 48), _string("','", 6, 49), _punctuation(")", 6, 52), _parameter("args", 6, 54), _punctuation(")", 6, 58), _punctuation(")", 6, 59), _punctuation(";", 6, 60),
+            // 7:            return 0;
+            _controlKeyword("return", 7, 12), _number("0", 7, 19), _punctuation(";", 7, 20),
+            // 8:        }
+            _punctuation("}", 8, 8),
+            // 9:    }
+            _punctuation("}", 9, 4),
+            //10: }
+            _punctuation("}", 10, 0),
+        ]);
+    });
+});
+
+function t(startLine: number, character: number, length: number, tokenClassifiction: string): ExpectedToken {
+    return { startLine, character, length, tokenClassifiction };
+}
+
+const _keyword = (text: string, line: number, col: number) => t(line, col, text.length, "plainKeyword");
+const _controlKeyword = (text: string, line: number, col: number) => t(line, col, text.length, "controlKeyword");
+const _punctuation = (text: string, line: number, col: number) => t(line, col, text.length, "punctuation");
+const _operator = (text: string, line: number, col: number) => t(line, col, text.length, "operator");
+const _number = (text: string, line: number, col: number) => t(line, col, text.length, "number");
+const _string = (text: string, line: number, col: number) => t(line, col, text.length, "string");
+const _namespace = (text: string, line: number, col: number) => t(line, col, text.length, "namespace");
+const _class = (text: string, line: number, col: number) => t(line, col, text.length, "class");
+const _staticClass = (text: string, line: number, col: number) => t(line, col, text.length, "class.static");
+const _staticMethod = (text: string, line: number, col: number) => t(line, col, text.length, "member.static");
+const _parameter = (text: string, line: number, col: number) => t(line, col, text.length, "parameter");

test/integrationTests/testAssets/singleCsproj/semantictokens.cs

@@ -0,0 +1,11 @@
+namespace Test
+{
+    public class TestProgram
+    {
+        public static int TestMain(string[] args)
+        {
+            System.Console.WriteLine(string.Join(',', args));
+            return 0;
+        }
+    }
+}

test/integrationTests/testAssets/slnWithCsproj/src/app/semantictokens.cs

@@ -0,0 +1,11 @@
+namespace Test
+{
+    public class TestProgram
+    {
+        public static int TestMain(string[] args)
+        {
+            System.Console.WriteLine(string.Join(',', args));
+            return 0;
+        }
+    }
+}