Skip to content

Commit

Permalink
Import the semantic highlighter from typescript-vscode-sh-plugin (#39119
Browse files Browse the repository at this point in the history 
)

* Initial import of the vscode semantic highlight code

* Adds the ability to test modern semantic classification via strings instead of numbers

* Adds existing tests

* Port over the semantic classification tests

* Update baselines

* Update src/harness/fourslashImpl.ts

Co-authored-by: Nathan Shively-Sanders <[email protected]>

* Handle feedback from #39119

* Consistent formatting in the 2020 classifier

* Update baselines

* Apply suggestions from code review

Co-authored-by: Daniel Rosenwasser <[email protected]>

* Update src/harness/fourslashImpl.ts

Co-authored-by: Daniel Rosenwasser <[email protected]>

* Reafactor after comments

* Use 2020 everywhere

* Handle feedback

* WIP - don't provide a breaking change

* Fix all build errors

* Update baselines

* Update src/services/classifier2020.ts

Co-authored-by: Sheetal Nandi <[email protected]>

* Addresses Ron's feedback

Co-authored-by: Nathan Shively-Sanders <[email protected]>
Co-authored-by: Daniel Rosenwasser <[email protected]>
Co-authored-by: Sheetal Nandi <[email protected]>
  • Loading branch information
@sandersn @DanielRosenwasser @sheetalkamat
4 people authored Sep 11, 2020
1 parent a36f17c commit db5368d
Show file tree
Hide file tree
Showing 81 changed files with 1,169 additions and 194 deletions.
2 changes: 1 addition & 1 deletion src/harness/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ namespace ts.server {
return notImplemented();
}

getEncodedSemanticClassifications(_fileName: string, _span: TextSpan): Classifications {
getEncodedSemanticClassifications(_fileName: string, _span: TextSpan, _format?: SemanticClassificationFormat): Classifications {
return notImplemented();
}

Expand Down
76 changes: 70 additions & 6 deletions src/harness/fourslashImpl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2492,7 +2492,49 @@ namespace FourSlash {
Harness.IO.log(this.spanInfoToString(this.getNameOrDottedNameSpan(pos)!, "**"));
}

private verifyClassifications(expected: { classificationType: string; text: string; textSpan?: TextSpan }[], actual: ts.ClassifiedSpan[], sourceFileText: string) {
private classificationToIdentifier(classification: number){

const tokenTypes: string[] = [];
tokenTypes[ts.classifier.v2020.TokenType.class] = "class";
tokenTypes[ts.classifier.v2020.TokenType.enum] = "enum";
tokenTypes[ts.classifier.v2020.TokenType.interface] = "interface";
tokenTypes[ts.classifier.v2020.TokenType.namespace] = "namespace";
tokenTypes[ts.classifier.v2020.TokenType.typeParameter] = "typeParameter";
tokenTypes[ts.classifier.v2020.TokenType.type] = "type";
tokenTypes[ts.classifier.v2020.TokenType.parameter] = "parameter";
tokenTypes[ts.classifier.v2020.TokenType.variable] = "variable";
tokenTypes[ts.classifier.v2020.TokenType.enumMember] = "enumMember";
tokenTypes[ts.classifier.v2020.TokenType.property] = "property";
tokenTypes[ts.classifier.v2020.TokenType.function] = "function";
tokenTypes[ts.classifier.v2020.TokenType.member] = "member";

const tokenModifiers: string[] = [];
tokenModifiers[ts.classifier.v2020.TokenModifier.async] = "async";
tokenModifiers[ts.classifier.v2020.TokenModifier.declaration] = "declaration";
tokenModifiers[ts.classifier.v2020.TokenModifier.readonly] = "readonly";
tokenModifiers[ts.classifier.v2020.TokenModifier.static] = "static";
tokenModifiers[ts.classifier.v2020.TokenModifier.local] = "local";
tokenModifiers[ts.classifier.v2020.TokenModifier.defaultLibrary] = "defaultLibrary";


function getTokenTypeFromClassification(tsClassification: number): number | undefined {
if (tsClassification > ts.classifier.v2020.TokenEncodingConsts.modifierMask) {
return (tsClassification >> ts.classifier.v2020.TokenEncodingConsts.typeOffset) - 1;
}
return undefined;
}

function getTokenModifierFromClassification(tsClassification: number) {
return tsClassification & ts.classifier.v2020.TokenEncodingConsts.modifierMask;
}

const typeIdx = getTokenTypeFromClassification(classification) || 0;
const modSet = getTokenModifierFromClassification(classification);

return [tokenTypes[typeIdx], ...tokenModifiers.filter((_, i) => modSet & 1 << i)].join(".");
}

private verifyClassifications(expected: { classificationType: string | number, text?: string; textSpan?: TextSpan }[], actual: (ts.ClassifiedSpan | ts.ClassifiedSpan2020)[] , sourceFileText: string) {
if (actual.length !== expected.length) {
this.raiseError("verifyClassifications failed - expected total classifications to be " + expected.length +
", but was " + actual.length +
Expand All @@ -2501,10 +2543,12 @@ namespace FourSlash {

ts.zipWith(expected, actual, (expectedClassification, actualClassification) => {
const expectedType = expectedClassification.classificationType;
if (expectedType !== actualClassification.classificationType) {
const actualType = typeof actualClassification.classificationType === "number" ? this.classificationToIdentifier(actualClassification.classificationType) : actualClassification.classificationType;

if (expectedType !== actualType) {
this.raiseError("verifyClassifications failed - expected classifications type to be " +
expectedType + ", but was " +
actualClassification.classificationType +
actualType +
jsonMismatchString());
}

Expand Down Expand Up @@ -2555,10 +2599,30 @@ namespace FourSlash {
}
}

public verifySemanticClassifications(expected: { classificationType: string; text: string }[]) {
public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
ts.createTextSpan(0, this.activeFile.content.length));
ts.createTextSpan(0, this.activeFile.content.length), format);
const replacement = [`const c2 = classification("2020");`,`verify.semanticClassificationsAre("2020",`];
for (const a of actual) {
const identifier = this.classificationToIdentifier(a.classificationType as number);
const text = this.activeFile.content.slice(a.textSpan.start, a.textSpan.start + a.textSpan.length);
replacement.push(` c2.semanticToken("${identifier}", "${text}"), `);
};
replacement.push(");");

throw new Error("You need to change the source code of fourslash test to use replaceWithSemanticClassifications");

// const fs = require("fs");
// const testfilePath = this.originalInputFileName.slice(1);
// const testfile = fs.readFileSync(testfilePath, "utf8");
// const newfile = testfile.replace("verify.replaceWithSemanticClassifications(\"2020\")", replacement.join("\n"));
// fs.writeFileSync(testfilePath, newfile);
}


public verifySemanticClassifications(format: ts.SemanticClassificationFormat, expected: { classificationType: string | number; text?: string }[]) {
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
ts.createTextSpan(0, this.activeFile.content.length), format);
this.verifyClassifications(expected, actual, this.activeFile.content);
}

Expand Down Expand Up @@ -3859,7 +3923,7 @@ namespace FourSlash {
const cancellation = new FourSlashInterface.Cancellation(state);
// eslint-disable-next-line no-eval
const f = eval(wrappedCode);
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.Classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
}
catch (err) {
// ensure 'source-map-support' is triggered while we still have the handler attached by accessing `error.stack`.
Expand Down
112 changes: 85 additions & 27 deletions src/harness/fourslashInterfaceImpl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -524,8 +524,12 @@ namespace FourSlashInterface {
/**
* This method *requires* an ordered stream of classifications for a file, and spans are highly recommended.
*/
public semanticClassificationsAre(...classifications: Classification[]) {
this.state.verifySemanticClassifications(classifications);
public semanticClassificationsAre(format: ts.SemanticClassificationFormat, ...classifications: Classification[]) {
this.state.verifySemanticClassifications(format, classifications);
}

public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
this.state.replaceWithSemanticClassifications(format);
}

public renameInfoSucceeded(displayName?: string, fullDisplayName?: string, kind?: string, kindModifiers?: string, fileToRename?: string, expectedRange?: FourSlash.Range, options?: ts.RenameInfoOptions) {
Expand Down Expand Up @@ -768,109 +772,163 @@ namespace FourSlashInterface {
}
}

interface Classification {
interface OlderClassification {
classificationType: ts.ClassificationTypeNames;
text: string;
textSpan?: FourSlash.TextSpan;
}
export namespace Classification {
export function comment(text: string, position?: number): Classification {

// The VS Code LSP
interface ModernClassification {
classificationType: string;
text?: string;
textSpan?: FourSlash.TextSpan;
}

type Classification = OlderClassification | ModernClassification;

export function classification(format: ts.SemanticClassificationFormat) {

function semanticToken(identifier: string, text: string, _position: number): Classification {
return {
classificationType: identifier,
text
};
}

if (format === ts.SemanticClassificationFormat.TwentyTwenty) {
return {
semanticToken
};
}

// Defaults to the previous semantic classifier factory functions

function comment(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.comment, text, position);
}

export function identifier(text: string, position?: number): Classification {
function identifier(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.identifier, text, position);
}

export function keyword(text: string, position?: number): Classification {
function keyword(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.keyword, text, position);
}

export function numericLiteral(text: string, position?: number): Classification {
function numericLiteral(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.numericLiteral, text, position);
}

export function operator(text: string, position?: number): Classification {
function operator(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.operator, text, position);
}

export function stringLiteral(text: string, position?: number): Classification {
function stringLiteral(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.stringLiteral, text, position);
}

export function whiteSpace(text: string, position?: number): Classification {
function whiteSpace(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.whiteSpace, text, position);
}

export function text(text: string, position?: number): Classification {
function text(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.text, text, position);
}

export function punctuation(text: string, position?: number): Classification {
function punctuation(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.punctuation, text, position);
}

export function docCommentTagName(text: string, position?: number): Classification {
function docCommentTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.docCommentTagName, text, position);
}

export function className(text: string, position?: number): Classification {
function className(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.className, text, position);
}

export function enumName(text: string, position?: number): Classification {
function enumName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.enumName, text, position);
}

export function interfaceName(text: string, position?: number): Classification {
function interfaceName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.interfaceName, text, position);
}

export function moduleName(text: string, position?: number): Classification {
function moduleName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.moduleName, text, position);
}

export function typeParameterName(text: string, position?: number): Classification {
function typeParameterName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.typeParameterName, text, position);
}

export function parameterName(text: string, position?: number): Classification {
function parameterName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.parameterName, text, position);
}

export function typeAliasName(text: string, position?: number): Classification {
function typeAliasName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.typeAliasName, text, position);
}

export function jsxOpenTagName(text: string, position?: number): Classification {
function jsxOpenTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxOpenTagName, text, position);
}

export function jsxCloseTagName(text: string, position?: number): Classification {
function jsxCloseTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxCloseTagName, text, position);
}

export function jsxSelfClosingTagName(text: string, position?: number): Classification {
function jsxSelfClosingTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxSelfClosingTagName, text, position);
}

export function jsxAttribute(text: string, position?: number): Classification {
function jsxAttribute(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxAttribute, text, position);
}

export function jsxText(text: string, position?: number): Classification {
function jsxText(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxText, text, position);
}

export function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxAttributeStringLiteralValue, text, position);
}

function getClassification(classificationType: ts.ClassificationTypeNames, text: string, position?: number): Classification {
const textSpan = position === undefined ? undefined : { start: position, end: position + text.length };
return { classificationType, text, textSpan };
}

return {
comment,
identifier,
keyword,
numericLiteral,
operator,
stringLiteral,
whiteSpace,
text,
punctuation,
docCommentTagName,
className,
enumName,
interfaceName,
moduleName,
typeParameterName,
parameterName,
typeAliasName,
jsxOpenTagName,
jsxCloseTagName,
jsxSelfClosingTagName,
jsxAttribute,
jsxText,
jsxAttributeStringLiteralValue,
getClassification
};
}

export namespace Completion {
export import SortText = ts.Completions.SortText;
export import CompletionSource = ts.Completions.CompletionSource;
Expand Down
9 changes: 5 additions & 4 deletions src/harness/harnessLanguageService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -461,14 +461,15 @@ namespace Harness.LanguageService {
getSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSyntacticClassifications(fileName, span.start, span.length));
}
getSemanticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length));
getSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length, format));
}
getEncodedSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
return unwrapJSONCallResult(this.shim.getEncodedSyntacticClassifications(fileName, span.start, span.length));
}
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length));
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.Classifications {
const responseFormat = format || ts.SemanticClassificationFormat.Original;
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length, responseFormat));
}
getCompletionsAtPosition(fileName: string, position: number, preferences: ts.UserPreferences | undefined): ts.CompletionInfo {
return unwrapJSONCallResult(this.shim.getCompletionsAtPosition(fileName, position, preferences));
Expand Down
Loading

0 comments on commit db5368d

Please sign in to comment.