From b257b0071ca3d9dd301ded82f0c9a04ca6a6bfa1 Mon Sep 17 00:00:00 2001 From: Lyu Jason Date: Thu, 7 Jan 2021 18:12:35 +0800 Subject: [PATCH 01/12] first crack at typescript semantic tokens --- .../lib/semanticToken/semanticTokenLabel.ts | 54 ++++++++++ .../language-server/src/plugins/PluginHost.ts | 28 +++++- .../plugins/collector/collectSemanticToken.ts | 18 ++++ .../language-server/src/plugins/interfaces.ts | 12 ++- .../plugins/typescript/TypeScriptPlugin.ts | 14 ++- .../features/SemanticTokensProvider.ts | 98 +++++++++++++++++++ .../src/plugins/typescript/utils.ts | 10 ++ packages/language-server/src/server.ts | 15 ++- packages/language-server/src/utils.ts | 25 +++++ 9 files changed, 267 insertions(+), 7 deletions(-) create mode 100644 packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts create mode 100644 packages/language-server/src/plugins/collector/collectSemanticToken.ts create mode 100644 packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts diff --git a/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts b/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts new file mode 100644 index 000000000..ff566ee55 --- /dev/null +++ b/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts @@ -0,0 +1,54 @@ +import { SemanticTokensLegend } from 'vscode-languageserver'; + +/** + * extended from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L9 + * so that we don't have to map it into our own legend + */ +export enum TokenType { + class, + enum, + interface, + namespace, + typeParameter, + type, + parameter, + variable, + enumMember, + property, + function, + + // member is renamed to method in vscode to match LSP default + method, + + // svelte + event +} + +/** + * adopted from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L13 + * so that we don't have to map it into our own legend + */ +export enum TokenModifier { + declaration, + static, + async, + readonly, + defaultLibrary, + local +} + +function isEnumMember(value: string | T): value is T { + return typeof value === 'number'; +} + +function extractEnumValues(values: Array) { + return values.filter(isEnumMember).sort((a, b) => a - b); +} + +// enum is transpiled into an object with enum name and value mapping each other +export const semanticTokenLegends: SemanticTokensLegend = { + tokenModifiers: extractEnumValues(Object.values(TokenModifier)).map( + (modifier) => TokenModifier[modifier] + ), + tokenTypes: extractEnumValues(Object.values(TokenType)).map((tokenType) => TokenType[tokenType]) +}; diff --git a/packages/language-server/src/plugins/PluginHost.ts b/packages/language-server/src/plugins/PluginHost.ts index cbd2f1884..3bc336a98 100644 --- a/packages/language-server/src/plugins/PluginHost.ts +++ b/packages/language-server/src/plugins/PluginHost.ts @@ -17,6 +17,7 @@ import { Range, ReferenceContext, SelectionRange, + SemanticTokens, SignatureHelp, SignatureHelpContext, SymbolInformation, @@ -27,6 +28,7 @@ import { import { DocumentManager } from '../lib/documents'; import { Logger } from '../logger'; import { regexLastIndexOf } from '../utils'; +import { collectSemanticTokens } from './collector/collectSemanticToken'; import { AppCompletionItem, FileRename, @@ -400,6 +402,20 @@ export class PluginHost implements LSProvider, OnWatchFileChanges { } } + async getSemanticTokens(textDocument: TextDocumentIdentifier, range?: Range) { + const document = this.getDocument(textDocument.uri); + if (!document) { + throw new Error('Cannot call methods on an unopened document'); + } + + return this.execute( + 'getSemanticTokens', + [document, range], + ExecuteMode.Collect, + collectSemanticTokens + ); + } + onWatchFileChanges(onWatchFileChangesParas: OnWatchFileChangesPara[]): void { for (const support of this.plugins) { support.onWatchFileChanges?.(onWatchFileChangesParas); @@ -420,11 +436,18 @@ export class PluginHost implements LSProvider, OnWatchFileChanges { args: any[], mode: ExecuteMode.Collect ): Promise; + private execute( + name: keyof LSProvider, + args: any[], + mode: ExecuteMode.Collect, + collector: (plugins: Plugin[], ...args: any[]) => Promise + ): Promise; private execute(name: keyof LSProvider, args: any[], mode: ExecuteMode.None): Promise; private async execute( name: keyof LSProvider, args: any[], - mode: ExecuteMode + mode: ExecuteMode, + collector?: (plugins: Plugin[], ...args: any[]) => T ): Promise<(T | null) | T[] | void> { const plugins = this.plugins.filter((plugin) => typeof plugin[name] === 'function'); @@ -438,6 +461,9 @@ export class PluginHost implements LSProvider, OnWatchFileChanges { } return null; case ExecuteMode.Collect: + if (typeof collector === 'function') { + return collector(this.plugins, ...args); + } return Promise.all( plugins.map((plugin) => this.tryExecutePlugin(plugin, name, args, [])) ); diff --git a/packages/language-server/src/plugins/collector/collectSemanticToken.ts b/packages/language-server/src/plugins/collector/collectSemanticToken.ts new file mode 100644 index 000000000..15c5bcaa7 --- /dev/null +++ b/packages/language-server/src/plugins/collector/collectSemanticToken.ts @@ -0,0 +1,18 @@ +import { Range, SemanticTokens } from 'vscode-languageserver'; +import { Document } from '../../lib/documents'; +import { flatten, isNotNullOrUndefined } from '../../utils'; +import { Plugin } from '../interfaces'; + +export async function collectSemanticTokens( + plugins: Plugin[], + textDocument: Document, + range?: Range +): Promise { + const partials = (await Promise.all( + plugins.map(plugin => plugin.getSemanticTokens?.(textDocument, range)) + )).filter(isNotNullOrUndefined); + + return { + data: flatten(partials.map(p => p.data)) + }; +} diff --git a/packages/language-server/src/plugins/interfaces.ts b/packages/language-server/src/plugins/interfaces.ts index 6b171d8fa..f56d376f4 100644 --- a/packages/language-server/src/plugins/interfaces.ts +++ b/packages/language-server/src/plugins/interfaces.ts @@ -1,4 +1,4 @@ -import { CompletionContext, FileChangeType, SignatureHelpContext } from 'vscode-languageserver'; +import { CompletionContext, FileChangeType, SemanticTokens, SignatureHelpContext } from 'vscode-languageserver'; import { CodeAction, CodeActionContext, @@ -141,6 +141,13 @@ export interface SelectionRangeProvider { getSelectionRange(document: Document, position: Position): Resolvable; } +export interface SemanticTokensProvider { + getSemanticTokens( + textDocument: Document, + range?: Range + ): Resolvable +} + export interface OnWatchFileChangesPara { fileName: string; changeType: FileChangeType; @@ -162,7 +169,8 @@ type ProviderBase = DiagnosticsProvider & CodeActionsProvider & FindReferencesProvider & RenameProvider & - SignatureHelpProvider; + SignatureHelpProvider & + SemanticTokensProvider; export type LSProvider = ProviderBase & BackwardsCompatibleDefinitionsProvider; diff --git a/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts b/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts index f39f86226..be07a7686 100644 --- a/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts +++ b/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts @@ -17,7 +17,8 @@ import { CompletionList, SelectionRange, SignatureHelp, - SignatureHelpContext + SignatureHelpContext, + SemanticTokens } from 'vscode-languageserver'; import { Document, @@ -43,7 +44,8 @@ import { SelectionRangeProvider, SignatureHelpProvider, UpdateImportsProvider, - OnWatchFileChangesPara + OnWatchFileChangesPara, + SemanticTokensProvider } from '../interfaces'; import { SnapshotFragment } from './DocumentSnapshot'; import { CodeActionsProviderImpl } from './features/CodeActionsProvider'; @@ -62,6 +64,7 @@ import { FindReferencesProviderImpl } from './features/FindReferencesProvider'; import { SelectionRangeProviderImpl } from './features/SelectionRangeProvider'; import { SignatureHelpProviderImpl } from './features/SignatureHelpProvider'; import { SnapshotManager } from './SnapshotManager'; +import { SemanticTokensProviderImpl } from './features/SemanticTokensProvider'; export class TypeScriptPlugin implements @@ -75,6 +78,7 @@ export class TypeScriptPlugin FindReferencesProvider, SelectionRangeProvider, SignatureHelpProvider, + SemanticTokensProvider, OnWatchFileChanges, CompletionsProvider { private readonly configManager: LSConfigManager; @@ -88,6 +92,7 @@ export class TypeScriptPlugin private readonly findReferencesProvider: FindReferencesProviderImpl; private readonly selectionRangeProvider: SelectionRangeProviderImpl; private readonly signatureHelpProvider: SignatureHelpProviderImpl; + private readonly semanticTokensProvider: SemanticTokensProviderImpl; constructor( docManager: DocumentManager, @@ -112,6 +117,7 @@ export class TypeScriptPlugin this.findReferencesProvider = new FindReferencesProviderImpl(this.lsAndTsDocResolver); this.selectionRangeProvider = new SelectionRangeProviderImpl(this.lsAndTsDocResolver); this.signatureHelpProvider = new SignatureHelpProviderImpl(this.lsAndTsDocResolver); + this.semanticTokensProvider = new SemanticTokensProviderImpl(this.lsAndTsDocResolver); } async getDiagnostics(document: Document): Promise { @@ -401,6 +407,10 @@ export class TypeScriptPlugin return this.signatureHelpProvider.getSignatureHelp(document, position, context); } + async getSemanticTokens(textDocument: Document, range?: Range): Promise { + return this.semanticTokensProvider.getSemanticTokens(textDocument, range); + } + private getLSAndTSDoc(document: Document) { return this.lsAndTsDocResolver.getLSAndTSDoc(document); } diff --git a/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts b/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts new file mode 100644 index 000000000..8ddd9803c --- /dev/null +++ b/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts @@ -0,0 +1,98 @@ +import ts from 'typescript'; +import { + Range, + SemanticTokens, + SemanticTokensBuilder +} from 'vscode-languageserver'; +import { Document } from '../../../lib/documents'; +import { SemanticTokensProvider } from '../../interfaces'; +import { SnapshotFragment } from '../DocumentSnapshot'; +import { LSAndTSDocResolver } from '../LSAndTSDocResolver'; +import { convertToTextSpan } from '../utils'; + +export class SemanticTokensProviderImpl implements SemanticTokensProvider { + constructor(private readonly lsAndTsDocResolver: LSAndTSDocResolver) {} + + async getSemanticTokens(textDocument: Document, range?: Range): Promise { + const { lang, tsDoc } = this.lsAndTsDocResolver.getLSAndTSDoc(textDocument); + const fragment = await tsDoc.getFragment(); + const textSpan = range + ? convertToTextSpan(range, fragment) + : { + start: 0, + length: fragment.text.length + }; + + const { spans } = lang.getEncodedSemanticClassifications( + tsDoc.filePath, + textSpan, + ts.SemanticClassificationFormat.TwentyTwenty + ); + + const builder = new SemanticTokensBuilder(); + let index = 0; + + while (index < spans.length) { + // [start, length, encodedClassification, start2, length2, encodedClassification2] + const generatedOffset = spans[index++]; + const generatedLength = spans[index++]; + const encodedClassification = spans[index++]; + const classificationType = this.getTokenTypeFromClassification(encodedClassification); + if (classificationType < 0) { + continue; + } + + const originalPosition = this.mapToOrigin( + textDocument, + fragment, + generatedOffset, + generatedLength + ); + if (!originalPosition) { + continue; + } + const [line, character, length] = originalPosition; + const modifier = this.getTokenModifierFromClassification(encodedClassification); + + builder.push(line, character, length, classificationType , modifier); + } + + return builder.build(); + } + + private mapToOrigin( + document: Document, + fragment: SnapshotFragment, + generatedOffset: number, + generatedLength: number + ): [line: number, character: number, length: number] | undefined { + const startPosition = fragment.getOriginalPosition(fragment.positionAt(generatedOffset)); + + if (startPosition.line < 0) { + return; + } + + const endPosition = fragment.getOriginalPosition( + fragment.positionAt(generatedOffset + generatedLength) + ); + const startOffset = document.offsetAt(startPosition); + const endOffset = document.offsetAt(endPosition); + + return [startPosition.line, startPosition.character, endOffset - startOffset]; + } + + /** TSClassification = + * (TokenType + 1) << TokenEncodingConsts.typeOffset + TokenModifier */ + private getTokenTypeFromClassification(tsClassification: number): number { + return (tsClassification >> TokenEncodingConsts.typeOffset) - 1; + } + + private getTokenModifierFromClassification(tsClassification: number) { + return tsClassification & TokenEncodingConsts.modifierMask; + } +} + +const enum TokenEncodingConsts { + typeOffset = 8, + modifierMask = (1 << typeOffset) - 1 +} diff --git a/packages/language-server/src/plugins/typescript/utils.ts b/packages/language-server/src/plugins/typescript/utils.ts index aef8ae25f..32408d220 100644 --- a/packages/language-server/src/plugins/typescript/utils.ts +++ b/packages/language-server/src/plugins/typescript/utils.ts @@ -286,3 +286,13 @@ export function getTsCheckComment(str = ''): string | undefined { } } } + +export function convertToTextSpan(range: Range, fragment: SnapshotFragment): ts.TextSpan { + const start = fragment.offsetAt(fragment.getGeneratedPosition(range.start)); + const end = fragment.offsetAt(fragment.getGeneratedPosition(range.end)); + + return { + start, + length: end - start + }; +} diff --git a/packages/language-server/src/server.ts b/packages/language-server/src/server.ts index 0b3ac3e24..73cb96ce8 100644 --- a/packages/language-server/src/server.ts +++ b/packages/language-server/src/server.ts @@ -4,7 +4,7 @@ import { ApplyWorkspaceEditRequest, CodeActionKind, DocumentUri, - _Connection, + Connection, MessageType, RenameFile, RequestType, @@ -17,6 +17,7 @@ import { import { IPCMessageReader, IPCMessageWriter, createConnection } from 'vscode-languageserver/node'; import { DiagnosticsManager } from './lib/DiagnosticsManager'; import { Document, DocumentManager } from './lib/documents'; +import { semanticTokenLegends } from './lib/semanticToken/semanticTokenLabel'; import { Logger } from './logger'; import { LSConfigManager } from './ls-config'; import { @@ -43,7 +44,7 @@ export interface LSOptions { * If you have a connection already that the ls should use, pass it in. * Else the connection will be created from `process`. */ - connection?: _Connection; + connection?: Connection; /** * If you want only errors getting logged. * Defaults to false. @@ -186,6 +187,11 @@ export function startServer(options?: LSOptions) { signatureHelpProvider: { triggerCharacters: ['(', ',', '<'], retriggerCharacters: [')'] + }, + semanticTokensProvider: { + legend: semanticTokenLegends, + range: true, + full: true } } }; @@ -290,6 +296,11 @@ export function startServer(options?: LSOptions) { }); connection.onDidSaveTextDocument(() => diagnosticsManager.updateAll()); + connection.languages.semanticTokens.on((evt) => pluginHost.getSemanticTokens(evt.textDocument)); + connection.languages.semanticTokens.onRange((evt) => + pluginHost.getSemanticTokens(evt.textDocument, evt.range) + ); + docManager.on( 'documentChange', _.debounce(async (document: Document) => diagnosticsManager.update(document), 500) diff --git a/packages/language-server/src/utils.ts b/packages/language-server/src/utils.ts index a4a597f04..240265cf2 100644 --- a/packages/language-server/src/utils.ts +++ b/packages/language-server/src/utils.ts @@ -107,3 +107,28 @@ export function getRegExpMatches(regex: RegExp, str: string) { } return matches; } + +/** + * + * @param map + * @param modifierSet bitwise combined modifier + */ +export function mapSemanticTokenModifiers( + map: Map, modifierSet: number +) { + let index = 0; + let result = 0; + + while (modifierSet > 0) { + if ((modifierSet & 0) !== 0) { + const mapped = map.get(index); + if (mapped != null) { + result += 1 << mapped; + } + } + index++; + modifierSet = modifierSet >> 1; + } + + return result; +} From cc58c1b361d97afd4b6d5defc5c3408159ba4032 Mon Sep 17 00:00:00 2001 From: Lyu Jason Date: Fri, 8 Jan 2021 09:26:34 +0800 Subject: [PATCH 02/12] use index mapping for legend instead instead of rely on the runtime behaviour of typescript enum --- .../lib/semanticToken/semanticTokenLabel.ts | 62 +++++++++++++------ packages/language-server/src/server.ts | 4 +- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts b/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts index ff566ee55..87bbc3f0c 100644 --- a/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts +++ b/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts @@ -1,10 +1,14 @@ -import { SemanticTokensLegend } from 'vscode-languageserver'; +import { + SemanticTokensLegend, + SemanticTokenModifiers, + SemanticTokenTypes +} from 'vscode-languageserver'; /** * extended from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L9 * so that we don't have to map it into our own legend */ -export enum TokenType { +export const enum TokenType { class, enum, interface, @@ -16,9 +20,7 @@ export enum TokenType { enumMember, property, function, - - // member is renamed to method in vscode to match LSP default - method, + member, // svelte event @@ -28,7 +30,7 @@ export enum TokenType { * adopted from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L13 * so that we don't have to map it into our own legend */ -export enum TokenModifier { +export const enum TokenModifier { declaration, static, async, @@ -37,18 +39,40 @@ export enum TokenModifier { local } -function isEnumMember(value: string | T): value is T { - return typeof value === 'number'; -} +export function getSemanticTokenLegends(): SemanticTokensLegend { + const tokenModifiers: string[] = []; -function extractEnumValues(values: Array) { - return values.filter(isEnumMember).sort((a, b) => a - b); -} + ([ + [TokenModifier.declaration, SemanticTokenModifiers.declaration], + [TokenModifier.static, SemanticTokenModifiers.static], + [TokenModifier.async, SemanticTokenModifiers.async], + [TokenModifier.readonly, SemanticTokenModifiers.readonly], + [TokenModifier.defaultLibrary, SemanticTokenModifiers.defaultLibrary], + [TokenModifier.local, 'local'] + ] as const).forEach(([tsModifier, legend]) => (tokenModifiers[tsModifier] = legend)); + + const tokenTypes: string[] = []; -// enum is transpiled into an object with enum name and value mapping each other -export const semanticTokenLegends: SemanticTokensLegend = { - tokenModifiers: extractEnumValues(Object.values(TokenModifier)).map( - (modifier) => TokenModifier[modifier] - ), - tokenTypes: extractEnumValues(Object.values(TokenType)).map((tokenType) => TokenType[tokenType]) -}; + ([ + [TokenType.class, SemanticTokenTypes.class], + [TokenType.enum, SemanticTokenTypes.enum], + [TokenType.interface, SemanticTokenTypes.interface], + [TokenType.namespace, SemanticTokenTypes.namespace], + [TokenType.typeParameter, SemanticTokenTypes.typeParameter], + [TokenType.type, SemanticTokenTypes.type], + [TokenType.parameter, SemanticTokenTypes.parameter], + [TokenType.variable, SemanticTokenTypes.variable], + [TokenType.enumMember, SemanticTokenTypes.enumMember], + [TokenType.property, SemanticTokenTypes.property], + [TokenType.function, SemanticTokenTypes.function], + + // member is renamed to method in vscode codebase to match LSP default + [TokenType.member, SemanticTokenTypes.method], + [TokenType.event, SemanticTokenTypes.event] + ] as const).forEach(([tokenType, legend]) => (tokenTypes[tokenType] = legend)); + + return { + tokenModifiers, + tokenTypes + }; +} diff --git a/packages/language-server/src/server.ts b/packages/language-server/src/server.ts index 73cb96ce8..91f82b5ad 100644 --- a/packages/language-server/src/server.ts +++ b/packages/language-server/src/server.ts @@ -17,7 +17,7 @@ import { import { IPCMessageReader, IPCMessageWriter, createConnection } from 'vscode-languageserver/node'; import { DiagnosticsManager } from './lib/DiagnosticsManager'; import { Document, DocumentManager } from './lib/documents'; -import { semanticTokenLegends } from './lib/semanticToken/semanticTokenLabel'; +import { getSemanticTokenLegends } from './lib/semanticToken/semanticTokenLabel'; import { Logger } from './logger'; import { LSConfigManager } from './ls-config'; import { @@ -189,7 +189,7 @@ export function startServer(options?: LSOptions) { retriggerCharacters: [')'] }, semanticTokensProvider: { - legend: semanticTokenLegends, + legend: getSemanticTokenLegends(), range: true, full: true } From d5e6b7f5e01d044af40f7fc57d3ba0d31ea71d3b Mon Sep 17 00:00:00 2001 From: Lyu Jason Date: Fri, 8 Jan 2021 16:16:24 +0800 Subject: [PATCH 03/12] reanme label => legend --- .../{semanticTokenLabel.ts => semanticTokenLegend.ts} | 0 packages/language-server/src/server.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename packages/language-server/src/lib/semanticToken/{semanticTokenLabel.ts => semanticTokenLegend.ts} (100%) diff --git a/packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts b/packages/language-server/src/lib/semanticToken/semanticTokenLegend.ts similarity index 100% rename from packages/language-server/src/lib/semanticToken/semanticTokenLabel.ts rename to packages/language-server/src/lib/semanticToken/semanticTokenLegend.ts diff --git a/packages/language-server/src/server.ts b/packages/language-server/src/server.ts index 91f82b5ad..a79062fd4 100644 --- a/packages/language-server/src/server.ts +++ b/packages/language-server/src/server.ts @@ -17,7 +17,7 @@ import { import { IPCMessageReader, IPCMessageWriter, createConnection } from 'vscode-languageserver/node'; import { DiagnosticsManager } from './lib/DiagnosticsManager'; import { Document, DocumentManager } from './lib/documents'; -import { getSemanticTokenLegends } from './lib/semanticToken/semanticTokenLabel'; +import { getSemanticTokenLegends } from './lib/semanticToken/semanticTokenLegend'; import { Logger } from './logger'; import { LSConfigManager } from './ls-config'; import { From 82cf63336200596df637f06ea44f92d02a97bef8 Mon Sep 17 00:00:00 2001 From: Lyu Jason Date: Fri, 8 Jan 2021 17:51:04 +0800 Subject: [PATCH 04/12] test --- .../features/SemanticTokensProvider.ts | 8 + .../features/SemanticTokensProvider.test.ts | 142 ++++++++++++++++++ .../testfiles/semantic-tokens/tokens.svelte | 13 ++ 3 files changed, 163 insertions(+) create mode 100644 packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts create mode 100644 packages/language-server/test/plugins/typescript/testfiles/semantic-tokens/tokens.svelte diff --git a/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts b/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts index 8ddd9803c..f4b9c395a 100644 --- a/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts +++ b/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts @@ -51,7 +51,15 @@ export class SemanticTokensProviderImpl implements SemanticTokensProvider { if (!originalPosition) { continue; } + const [line, character, length] = originalPosition; + + // remove identifers whose start and end mapped to the some location + // like the svelte2tsx inserted render function + if (!length) { + continue; + } + const modifier = this.getTokenModifierFromClassification(encodedClassification); builder.push(line, character, length, classificationType , modifier); diff --git a/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts b/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts new file mode 100644 index 000000000..94c30a820 --- /dev/null +++ b/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts @@ -0,0 +1,142 @@ +import path from 'path'; +import ts from 'typescript'; +import assert from 'assert'; +import { SemanticTokensBuilder } from 'vscode-languageserver'; +import { Document, DocumentManager } from '../../../../src/lib/documents'; +import { TokenModifier, TokenType } from '../../../../src/lib/semanticToken/semanticTokenLegend'; +import { LSConfigManager } from '../../../../src/ls-config'; +import { SemanticTokensProviderImpl } from '../../../../src/plugins/typescript/features/SemanticTokensProvider'; +import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver'; +import { pathToUrl } from '../../../../src/utils'; + +const testDir = path.join(__dirname, '..'); + +describe('SemanticTokensProvider', () => { + function setup() { + const docManager = new DocumentManager( + (textDocument) => new Document(textDocument.uri, textDocument.text) + ); + const filePath = path.join(testDir, 'testfiles', 'semantic-tokens', 'tokens.svelte'); + const lsAndTsDocResolver = new LSAndTSDocResolver( + docManager, + [pathToUrl(testDir)], + new LSConfigManager() + ); + const provider = new SemanticTokensProviderImpl(lsAndTsDocResolver); + const document = docManager.openDocument({ + uri: pathToUrl(filePath), + text: ts.sys.readFile(filePath) + }); + return { provider, document }; + } + + it('provides semantic token', async () => { + const { provider, document } = setup(); + + const { data } = await provider.getSemanticTokens(document); + + assert.deepStrictEqual(data, getExpected()); + }); + + function getExpected() { + const tokenData: Array<{ + line: number; + character: number; + length: number; + type: number; + modifiers: number[]; + }> = [ + { + line: 1, + character: 14, + length: 'TextContent'.length, + type: TokenType.interface, + modifiers: [TokenModifier.declaration] + }, + { + line: 2, + character: 8, + length: 'text'.length, + type: TokenType.property, + modifiers: [TokenModifier.declaration] + }, + { + line: 5, + character: 15, + length: 'textPromise'.length, + type: TokenType.variable, + modifiers: [TokenModifier.declaration, TokenModifier.local] + }, + { + line: 5, + character: 28, + length: 'Promise'.length, + type: TokenType.interface, + modifiers: [TokenModifier.defaultLibrary] + }, + { + line: 5, + character: 36, + length: 'TextContent'.length, + type: TokenType.interface, + modifiers: [] + }, + { + line: 7, + character: 19, + length: 'blurHandler'.length, + type: TokenType.function, + modifiers: [TokenModifier.async, TokenModifier.declaration, TokenModifier.local] + }, + { + line: 10, + character: 8, + length: 'textPromise'.length, + type: TokenType.variable, + modifiers: [TokenModifier.local] + }, + { + line: 10, + character: 25, + length: 'text'.length, + type: TokenType.parameter, + modifiers: [TokenModifier.declaration] + }, + { + line: 11, + character: 23, + length: 'blurHandler'.length, + type: TokenType.function, + modifiers: [TokenModifier.async, TokenModifier.local] + }, + { + line: 11, + character: 43, + length: 'text'.length, + type: TokenType.parameter, + modifiers: [] + }, + { + line: 11, + character: 48, + length: 'text'.length, + type: TokenType.property, + modifiers: [] + } + ]; + + const builder = new SemanticTokensBuilder(); + for (const token of tokenData) { + builder.push( + token.line, + token.character, + token.length, + token.type, + token.modifiers.reduce((pre, next) => (pre | 1 << next), 0) + ); + } + + const data = builder.build().data; + return data; + } +}); diff --git a/packages/language-server/test/plugins/typescript/testfiles/semantic-tokens/tokens.svelte b/packages/language-server/test/plugins/typescript/testfiles/semantic-tokens/tokens.svelte new file mode 100644 index 000000000..4019975bd --- /dev/null +++ b/packages/language-server/test/plugins/typescript/testfiles/semantic-tokens/tokens.svelte @@ -0,0 +1,13 @@ + + +{#await textPromise then text} +