diff --git a/packages/language-server/src/lib/semanticToken/semanticTokenLegend.ts b/packages/language-server/src/lib/semanticToken/semanticTokenLegend.ts new file mode 100644 index 000000000..87bbc3f0c --- /dev/null +++ b/packages/language-server/src/lib/semanticToken/semanticTokenLegend.ts @@ -0,0 +1,78 @@ +import { + SemanticTokensLegend, + SemanticTokenModifiers, + SemanticTokenTypes +} from 'vscode-languageserver'; + +/** + * extended from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L9 + * so that we don't have to map it into our own legend + */ +export const enum TokenType { + class, + enum, + interface, + namespace, + typeParameter, + type, + parameter, + variable, + enumMember, + property, + function, + member, + + // svelte + event +} + +/** + * adopted from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L13 + * so that we don't have to map it into our own legend + */ +export const enum TokenModifier { + declaration, + static, + async, + readonly, + defaultLibrary, + local +} + +export function getSemanticTokenLegends(): SemanticTokensLegend { + const tokenModifiers: string[] = []; + + ([ + [TokenModifier.declaration, SemanticTokenModifiers.declaration], + [TokenModifier.static, SemanticTokenModifiers.static], + [TokenModifier.async, SemanticTokenModifiers.async], + [TokenModifier.readonly, SemanticTokenModifiers.readonly], + [TokenModifier.defaultLibrary, SemanticTokenModifiers.defaultLibrary], + [TokenModifier.local, 'local'] + ] as const).forEach(([tsModifier, legend]) => (tokenModifiers[tsModifier] = legend)); + + const tokenTypes: string[] = []; + + ([ + [TokenType.class, SemanticTokenTypes.class], + [TokenType.enum, SemanticTokenTypes.enum], + [TokenType.interface, SemanticTokenTypes.interface], + [TokenType.namespace, SemanticTokenTypes.namespace], + [TokenType.typeParameter, SemanticTokenTypes.typeParameter], + [TokenType.type, SemanticTokenTypes.type], + [TokenType.parameter, SemanticTokenTypes.parameter], + [TokenType.variable, SemanticTokenTypes.variable], + [TokenType.enumMember, SemanticTokenTypes.enumMember], + [TokenType.property, SemanticTokenTypes.property], + [TokenType.function, SemanticTokenTypes.function], + + // member is renamed to method in vscode codebase to match LSP default + [TokenType.member, SemanticTokenTypes.method], + [TokenType.event, SemanticTokenTypes.event] + ] as const).forEach(([tokenType, legend]) => (tokenTypes[tokenType] = legend)); + + return { + tokenModifiers, + tokenTypes + }; +} diff --git a/packages/language-server/src/ls-config.ts b/packages/language-server/src/ls-config.ts index d5d01cf68..406ab6f4c 100644 --- a/packages/language-server/src/ls-config.ts +++ b/packages/language-server/src/ls-config.ts @@ -17,7 +17,8 @@ const defaultLSConfig: LSConfig = { codeActions: { enable: true }, rename: { enable: true }, selectionRange: { enable: true }, - signatureHelp: { enable: true } + signatureHelp: { enable: true }, + semanticTokens: { enable: true } }, css: { enable: true, @@ -93,6 +94,9 @@ export interface LSTypescriptConfig { signatureHelp: { enable: boolean; }; + semanticTokens: { + enable: boolean; + } } export interface LSCSSConfig { diff --git a/packages/language-server/src/plugins/PluginHost.ts b/packages/language-server/src/plugins/PluginHost.ts index cbd2f1884..695f3379a 100644 --- a/packages/language-server/src/plugins/PluginHost.ts +++ b/packages/language-server/src/plugins/PluginHost.ts @@ -17,6 +17,7 @@ import { Range, ReferenceContext, SelectionRange, + SemanticTokens, SignatureHelp, SignatureHelpContext, SymbolInformation, @@ -400,6 +401,23 @@ export class PluginHost implements LSProvider, OnWatchFileChanges { } } + async getSemanticTokens(textDocument: TextDocumentIdentifier, range?: Range) { + const document = this.getDocument(textDocument.uri); + if (!document) { + throw new Error('Cannot call methods on an unopened document'); + } + + return ( + (await this.execute( + 'getSemanticTokens', + [document, range], + ExecuteMode.FirstNonNull + )) ?? { + data: [] + } + ); + } + onWatchFileChanges(onWatchFileChangesParas: OnWatchFileChangesPara[]): void { for (const support of this.plugins) { support.onWatchFileChanges?.(onWatchFileChangesParas); diff --git a/packages/language-server/src/plugins/interfaces.ts b/packages/language-server/src/plugins/interfaces.ts index 6b171d8fa..f56d376f4 100644 --- a/packages/language-server/src/plugins/interfaces.ts +++ b/packages/language-server/src/plugins/interfaces.ts @@ -1,4 +1,4 @@ -import { CompletionContext, FileChangeType, SignatureHelpContext } from 'vscode-languageserver'; +import { CompletionContext, FileChangeType, SemanticTokens, SignatureHelpContext } from 'vscode-languageserver'; import { CodeAction, CodeActionContext, @@ -141,6 +141,13 @@ export interface SelectionRangeProvider { getSelectionRange(document: Document, position: Position): Resolvable; } +export interface SemanticTokensProvider { + getSemanticTokens( + textDocument: Document, + range?: Range + ): Resolvable +} + export interface OnWatchFileChangesPara { fileName: string; changeType: FileChangeType; @@ -162,7 +169,8 @@ type ProviderBase = DiagnosticsProvider & CodeActionsProvider & FindReferencesProvider & RenameProvider & - SignatureHelpProvider; + SignatureHelpProvider & + SemanticTokensProvider; export type LSProvider = ProviderBase & BackwardsCompatibleDefinitionsProvider; diff --git a/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts b/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts index f39f86226..ca2d19f99 100644 --- a/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts +++ b/packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts @@ -17,7 +17,8 @@ import { CompletionList, SelectionRange, SignatureHelp, - SignatureHelpContext + SignatureHelpContext, + SemanticTokens } from 'vscode-languageserver'; import { Document, @@ -43,7 +44,8 @@ import { SelectionRangeProvider, SignatureHelpProvider, UpdateImportsProvider, - OnWatchFileChangesPara + OnWatchFileChangesPara, + SemanticTokensProvider } from '../interfaces'; import { SnapshotFragment } from './DocumentSnapshot'; import { CodeActionsProviderImpl } from './features/CodeActionsProvider'; @@ -62,6 +64,7 @@ import { FindReferencesProviderImpl } from './features/FindReferencesProvider'; import { SelectionRangeProviderImpl } from './features/SelectionRangeProvider'; import { SignatureHelpProviderImpl } from './features/SignatureHelpProvider'; import { SnapshotManager } from './SnapshotManager'; +import { SemanticTokensProviderImpl } from './features/SemanticTokensProvider'; export class TypeScriptPlugin implements @@ -75,6 +78,7 @@ export class TypeScriptPlugin FindReferencesProvider, SelectionRangeProvider, SignatureHelpProvider, + SemanticTokensProvider, OnWatchFileChanges, CompletionsProvider { private readonly configManager: LSConfigManager; @@ -88,6 +92,7 @@ export class TypeScriptPlugin private readonly findReferencesProvider: FindReferencesProviderImpl; private readonly selectionRangeProvider: SelectionRangeProviderImpl; private readonly signatureHelpProvider: SignatureHelpProviderImpl; + private readonly semanticTokensProvider: SemanticTokensProviderImpl; constructor( docManager: DocumentManager, @@ -112,6 +117,7 @@ export class TypeScriptPlugin this.findReferencesProvider = new FindReferencesProviderImpl(this.lsAndTsDocResolver); this.selectionRangeProvider = new SelectionRangeProviderImpl(this.lsAndTsDocResolver); this.signatureHelpProvider = new SignatureHelpProviderImpl(this.lsAndTsDocResolver); + this.semanticTokensProvider = new SemanticTokensProviderImpl(this.lsAndTsDocResolver); } async getDiagnostics(document: Document): Promise { @@ -401,6 +407,16 @@ export class TypeScriptPlugin return this.signatureHelpProvider.getSignatureHelp(document, position, context); } + async getSemanticTokens(textDocument: Document, range?: Range): Promise { + if (!this.featureEnabled('semanticTokens')) { + return { + data: [] + }; + } + + return this.semanticTokensProvider.getSemanticTokens(textDocument, range); + } + private getLSAndTSDoc(document: Document) { return this.lsAndTsDocResolver.getLSAndTSDoc(document); } diff --git a/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts b/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts new file mode 100644 index 000000000..57be29d0b --- /dev/null +++ b/packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts @@ -0,0 +1,110 @@ +import ts from 'typescript'; +import { + Range, + SemanticTokens, + SemanticTokensBuilder +} from 'vscode-languageserver'; +import { Document } from '../../../lib/documents'; +import { SemanticTokensProvider } from '../../interfaces'; +import { SnapshotFragment } from '../DocumentSnapshot'; +import { LSAndTSDocResolver } from '../LSAndTSDocResolver'; +import { convertToTextSpan } from '../utils'; + +export class SemanticTokensProviderImpl implements SemanticTokensProvider { + constructor(private readonly lsAndTsDocResolver: LSAndTSDocResolver) {} + + async getSemanticTokens(textDocument: Document, range?: Range): Promise { + const { lang, tsDoc } = this.lsAndTsDocResolver.getLSAndTSDoc(textDocument); + const fragment = await tsDoc.getFragment(); + const textSpan = range + ? convertToTextSpan(range, fragment) + : { + start: 0, + length: tsDoc.parserError + ? fragment.text.length + : // This is appended by svelte2tsx, there's nothing mappable afterwards + fragment.text.lastIndexOf('return { props:') || fragment.text.length + }; + + const { spans } = lang.getEncodedSemanticClassifications( + tsDoc.filePath, + textSpan, + ts.SemanticClassificationFormat.TwentyTwenty + ); + + const builder = new SemanticTokensBuilder(); + let index = 0; + + while (index < spans.length) { + // [start, length, encodedClassification, start2, length2, encodedClassification2] + const generatedOffset = spans[index++]; + const generatedLength = spans[index++]; + const encodedClassification = spans[index++]; + const classificationType = this.getTokenTypeFromClassification(encodedClassification); + if (classificationType < 0) { + continue; + } + + const originalPosition = this.mapToOrigin( + textDocument, + fragment, + generatedOffset, + generatedLength + ); + if (!originalPosition) { + continue; + } + + const [line, character, length] = originalPosition; + + // remove identifers whose start and end mapped to the same location + // like the svelte2tsx inserted render function + if (!length) { + continue; + } + + const modifier = this.getTokenModifierFromClassification(encodedClassification); + + builder.push(line, character, length, classificationType , modifier); + } + + return builder.build(); + } + + private mapToOrigin( + document: Document, + fragment: SnapshotFragment, + generatedOffset: number, + generatedLength: number + ): [line: number, character: number, length: number] | undefined { + const startPosition = fragment.getOriginalPosition(fragment.positionAt(generatedOffset)); + + if (startPosition.line < 0) { + return; + } + + const endPosition = fragment.getOriginalPosition( + fragment.positionAt(generatedOffset + generatedLength) + ); + const startOffset = document.offsetAt(startPosition); + const endOffset = document.offsetAt(endPosition); + + return [startPosition.line, startPosition.character, endOffset - startOffset]; + } + + /** + * TSClassification = (TokenType + 1) << TokenEncodingConsts.typeOffset + TokenModifier + */ + private getTokenTypeFromClassification(tsClassification: number): number { + return (tsClassification >> TokenEncodingConsts.typeOffset) - 1; + } + + private getTokenModifierFromClassification(tsClassification: number) { + return tsClassification & TokenEncodingConsts.modifierMask; + } +} + +const enum TokenEncodingConsts { + typeOffset = 8, + modifierMask = (1 << typeOffset) - 1 +} diff --git a/packages/language-server/src/plugins/typescript/utils.ts b/packages/language-server/src/plugins/typescript/utils.ts index aef8ae25f..32408d220 100644 --- a/packages/language-server/src/plugins/typescript/utils.ts +++ b/packages/language-server/src/plugins/typescript/utils.ts @@ -286,3 +286,13 @@ export function getTsCheckComment(str = ''): string | undefined { } } } + +export function convertToTextSpan(range: Range, fragment: SnapshotFragment): ts.TextSpan { + const start = fragment.offsetAt(fragment.getGeneratedPosition(range.start)); + const end = fragment.offsetAt(fragment.getGeneratedPosition(range.end)); + + return { + start, + length: end - start + }; +} diff --git a/packages/language-server/src/server.ts b/packages/language-server/src/server.ts index 0b3ac3e24..a79062fd4 100644 --- a/packages/language-server/src/server.ts +++ b/packages/language-server/src/server.ts @@ -4,7 +4,7 @@ import { ApplyWorkspaceEditRequest, CodeActionKind, DocumentUri, - _Connection, + Connection, MessageType, RenameFile, RequestType, @@ -17,6 +17,7 @@ import { import { IPCMessageReader, IPCMessageWriter, createConnection } from 'vscode-languageserver/node'; import { DiagnosticsManager } from './lib/DiagnosticsManager'; import { Document, DocumentManager } from './lib/documents'; +import { getSemanticTokenLegends } from './lib/semanticToken/semanticTokenLegend'; import { Logger } from './logger'; import { LSConfigManager } from './ls-config'; import { @@ -43,7 +44,7 @@ export interface LSOptions { * If you have a connection already that the ls should use, pass it in. * Else the connection will be created from `process`. */ - connection?: _Connection; + connection?: Connection; /** * If you want only errors getting logged. * Defaults to false. @@ -186,6 +187,11 @@ export function startServer(options?: LSOptions) { signatureHelpProvider: { triggerCharacters: ['(', ',', '<'], retriggerCharacters: [')'] + }, + semanticTokensProvider: { + legend: getSemanticTokenLegends(), + range: true, + full: true } } }; @@ -290,6 +296,11 @@ export function startServer(options?: LSOptions) { }); connection.onDidSaveTextDocument(() => diagnosticsManager.updateAll()); + connection.languages.semanticTokens.on((evt) => pluginHost.getSemanticTokens(evt.textDocument)); + connection.languages.semanticTokens.onRange((evt) => + pluginHost.getSemanticTokens(evt.textDocument, evt.range) + ); + docManager.on( 'documentChange', _.debounce(async (document: Document) => diagnosticsManager.update(document), 500) diff --git a/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts b/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts new file mode 100644 index 000000000..205236fac --- /dev/null +++ b/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts @@ -0,0 +1,177 @@ +import path from 'path'; +import ts from 'typescript'; +import assert from 'assert'; +import { Position, Range, SemanticTokensBuilder } from 'vscode-languageserver'; +import { Document, DocumentManager } from '../../../../src/lib/documents'; +import { TokenModifier, TokenType } from '../../../../src/lib/semanticToken/semanticTokenLegend'; +import { LSConfigManager } from '../../../../src/ls-config'; +import { SemanticTokensProviderImpl } from '../../../../src/plugins/typescript/features/SemanticTokensProvider'; +import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver'; +import { pathToUrl } from '../../../../src/utils'; + +const testDir = path.join(__dirname, '..'); + +describe('SemanticTokensProvider', () => { + function setup() { + const docManager = new DocumentManager( + (textDocument) => new Document(textDocument.uri, textDocument.text) + ); + const filePath = path.join(testDir, 'testfiles', 'semantic-tokens', 'tokens.svelte'); + const lsAndTsDocResolver = new LSAndTSDocResolver( + docManager, + [pathToUrl(testDir)], + new LSConfigManager() + ); + const provider = new SemanticTokensProviderImpl(lsAndTsDocResolver); + const document = docManager.openDocument({ + uri: pathToUrl(filePath), + text: ts.sys.readFile(filePath) + }); + return { provider, document }; + } + + it('provides semantic token', async () => { + const { provider, document } = setup(); + + const { data } = await provider.getSemanticTokens(document); + + assertResult(data, getExpected(/* isFull */ true)); + }); + + it('provides partial semantic token', async () => { + const { provider, document } = setup(); + + const { data } = await provider.getSemanticTokens( + document, + Range.create(Position.create(0, 0), Position.create(9, 0)) + ); + + assertResult(data, getExpected(/* isFull */ false)); + }); + + function getExpected(full: boolean) { + const tokenDataScript: Array<{ + line: number; + character: number; + length: number; + type: number; + modifiers: number[]; + }> = [ + { + line: 1, + character: 14, + length: 'TextContent'.length, + type: TokenType.interface, + modifiers: [TokenModifier.declaration] + }, + { + line: 2, + character: 8, + length: 'text'.length, + type: TokenType.property, + modifiers: [TokenModifier.declaration] + }, + { + line: 5, + character: 15, + length: 'textPromise'.length, + type: TokenType.variable, + modifiers: [TokenModifier.declaration, TokenModifier.local] + }, + { + line: 5, + character: 28, + length: 'Promise'.length, + type: TokenType.interface, + modifiers: [TokenModifier.defaultLibrary] + }, + { + line: 5, + character: 36, + length: 'TextContent'.length, + type: TokenType.interface, + modifiers: [] + }, + { + line: 7, + character: 19, + length: 'blurHandler'.length, + type: TokenType.function, + modifiers: [TokenModifier.async, TokenModifier.declaration, TokenModifier.local] + } + ]; + const tokenDataAll = [ + ...tokenDataScript, + { + line: 10, + character: 8, + length: 'textPromise'.length, + type: TokenType.variable, + modifiers: [TokenModifier.local] + }, + { + line: 10, + character: 25, + length: 'text'.length, + type: TokenType.parameter, + modifiers: [TokenModifier.declaration] + }, + { + line: 11, + character: 23, + length: 'blurHandler'.length, + type: TokenType.function, + modifiers: [TokenModifier.async, TokenModifier.local] + }, + { + line: 11, + character: 43, + length: 'text'.length, + type: TokenType.parameter, + modifiers: [] + }, + { + line: 11, + character: 48, + length: 'text'.length, + type: TokenType.property, + modifiers: [] + } + ]; + + const builder = new SemanticTokensBuilder(); + for (const token of full ? tokenDataAll : tokenDataScript) { + builder.push( + token.line, + token.character, + token.length, + token.type, + token.modifiers.reduce((pre, next) => pre | (1 << next), 0) + ); + } + + const data = builder.build().data; + return data; + } + + /** + * group result by tokens to better distinguish + */ + function assertResult(actual: number[], expected: number[]) { + const actualGrouped = group(actual); + const expectedGrouped = group(expected); + + assert.deepStrictEqual(actualGrouped, expectedGrouped); + } + + function group(tokens: number[]) { + const result: number[][] = []; + + let index = 0; + while (index < tokens.length) { + result.push(tokens.splice(index, index += 5)); + } + + return result; + } +}); diff --git a/packages/language-server/test/plugins/typescript/testfiles/semantic-tokens/tokens.svelte b/packages/language-server/test/plugins/typescript/testfiles/semantic-tokens/tokens.svelte new file mode 100644 index 000000000..4019975bd --- /dev/null +++ b/packages/language-server/test/plugins/typescript/testfiles/semantic-tokens/tokens.svelte @@ -0,0 +1,13 @@ + + +{#await textPromise then text} +