From f4776e2398a345007f1a4d801b98861302cb7359 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Mon, 19 May 2025 14:00:39 +0200 Subject: [PATCH 01/10] Fix `test:components` --- tests/components-test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/components-test.ts b/tests/components-test.ts index f8314d7688..38374ea6b2 100644 --- a/tests/components-test.ts +++ b/tests/components-test.ts @@ -1,8 +1,9 @@ -import { assert } from 'chai'; import { readFileSync } from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; -import { forEach, noop, toArray } from '../src/shared/util'; +import { assert } from 'chai'; +import { noop } from '../src/shared/util'; +import { forEach, toArray } from '../src/util/iterables'; import { getComponent, getComponentIds, getLanguageIds } from './helper/prism-loader'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -100,7 +101,7 @@ describe('components.json', () => { describe('- should have valid alias titles', () => { for (const lang of getLanguageIds()) { it(`- ${lang} should have all alias titles registered as alias`, async () => { - const aliases = new Set(toArray((await getComponent(lang)).alias)); + const aliases = new Set(toArray((await getComponent(lang))?.alias)); const aliasTitles = (components.languages[lang] as ComponentEntry | undefined)?.aliasTitles ?? {}; From d36d05b219f3fa348121c3ddc3e160c66afcaaf4 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Mon, 19 May 2025 16:37:30 +0200 Subject: [PATCH 02/10] Partially adopt the new API --- tests/helper/prism-loader.ts | 17 ++++++++--------- tests/helper/test-case.ts | 6 +++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/helper/prism-loader.ts b/tests/helper/prism-loader.ts index 71b40503cc..c61f798fa8 100644 --- a/tests/helper/prism-loader.ts +++ b/tests/helper/prism-loader.ts @@ -60,10 +60,8 @@ getComponentIds().forEach(getComponent); * Creates a new Prism instance with the given language loaded */ export async function createInstance (languages?: string | string[]) { - const instance = new Prism(); - - const protos = await Promise.all(toArray(languages).map(getComponent)); - instance.components.add(...protos); + const instance = new Prism({ languages: toArray(languages) as string[], languagePath: path.join(SRC_DIR, 'languages') }); + await instance.languagesReady; return instance; } @@ -139,11 +137,12 @@ export function createPrismDOM (): PrismDOM<{}> { /** * Loads the given languages or plugins. */ - const load = async (languagesOrPlugins: string | string[]) => { - const protos = await Promise.all(toArray(languagesOrPlugins).map(getComponent)); + const load = async (ids: string | string[], type: 'languages' | 'plugins' = 'languages') => { + let registry = instance[type === 'languages' ? 'languageRegistry' : 'pluginRegistry']; withGlobals(() => { - instance.components.add(...protos); + registry.loadAll(toArray(ids) as string[]); }); + await registry.ready; }; return { @@ -151,8 +150,8 @@ export function createPrismDOM (): PrismDOM<{}> { window: window as PrismWindow<{}>, document: window.document, Prism: window.Prism as never, - loadLanguages: load, - loadPlugins: load, + loadLanguages: async (ids: string | string[]) => await load(ids), + loadPlugins: async (ids: string | string[]) => await load(ids, 'plugins'), withGlobals, }; } diff --git a/tests/helper/test-case.ts b/tests/helper/test-case.ts index 173a56ec0c..fdd59a9922 100644 --- a/tests/helper/test-case.ts +++ b/tests/helper/test-case.ts @@ -3,8 +3,8 @@ import fs from 'fs'; import { createInstance } from './prism-loader'; import * as TokenStreamTransformer from './token-stream-transformer'; import { formatHtml, getLeadingSpaces } from './util'; -import type { Prism } from '../../src/core'; -import type { TokenStream } from '../../src/core/token'; +import type { Prism } from '../../src/types'; +import type { TokenStream } from '../../src/types'; const defaultCreateInstance = createInstance; @@ -134,7 +134,7 @@ interface Runner { } const jsonRunner: Runner = { run (Prism, code, language) { - const grammar = Prism.components.getLanguage(language); + const grammar = Prism.languageRegistry.getLanguage(language)?.resolvedGrammar; return Prism.tokenize(code, grammar ?? {}); }, print (actual) { From 3435645f36ea4934d4e8b80a4dc4751567b1a66f Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Mon, 19 May 2025 17:26:27 +0200 Subject: [PATCH 03/10] Fix `test:identifiers` --- tests/identifier-test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/identifier-test.ts b/tests/identifier-test.ts index 969cf62c87..4672cfd20f 100644 --- a/tests/identifier-test.ts +++ b/tests/identifier-test.ts @@ -3,7 +3,7 @@ import { toArray } from '../src/util/iterables'; import { createInstance, getComponent, getLanguageIds } from './helper/prism-loader'; import { prettyprint } from './helper/token-stream-transformer'; import type { Prism, Token } from '../src/core'; -import type { TokenStream } from '../src/core/token'; +import type { TokenStream } from '../src/types'; // This is where you can exclude a language from the identifier test. // @@ -89,7 +89,7 @@ for (const lang of getLanguageIds()) { describe(`Patterns of '${lang}' with optional dependencies`, () => { const getPrism = async () => { const component = await getComponent(lang); - const optional = toArray(component.optional); + const optional = toArray(component?.optional); const Prism = await createInstance([lang, ...optional]); return Prism; }; From 2a1cf4eae6d175f159b0b43808d8435f62baabd3 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Mon, 19 May 2025 18:09:32 +0200 Subject: [PATCH 04/10] Use `languageRegistry` --- tests/pattern-tests.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pattern-tests.ts b/tests/pattern-tests.ts index 38c77fadf9..1a306a56a3 100644 --- a/tests/pattern-tests.ts +++ b/tests/pattern-tests.ts @@ -159,8 +159,8 @@ function testPatterns (getPrism: () => Promise, mainLanguage: } // static analysis - for (const id of Prism.components['entries'].keys()) { - const grammar = Prism.components.getLanguage(id); + for (const id of Object.keys(Prism.languageRegistry.cache)) { + const grammar = Prism.languageRegistry.getLanguage(id)?.resolvedGrammar; if (grammar) { traverse(grammar, id); } @@ -169,7 +169,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage: // dynamic analysis for (const lang of getRelevantLanguages()) { const snippets = testSnippets.get(lang); - const grammar = Prism.components.getLanguage(lang); + const grammar = Prism.languageRegistry.getLanguage(lang)?.resolvedGrammar; // eslint-disable-next-line @typescript-eslint/unbound-method const oldTokenize = Prism.tokenize; From bf27e7249dd4b78c5c69a7bb3606eb28fac33df8 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Mon, 19 May 2025 22:13:57 +0200 Subject: [PATCH 05/10] Use cache --- tests/helper/prism-loader.ts | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/helper/prism-loader.ts b/tests/helper/prism-loader.ts index c61f798fa8..9abd069e07 100644 --- a/tests/helper/prism-loader.ts +++ b/tests/helper/prism-loader.ts @@ -60,8 +60,12 @@ getComponentIds().forEach(getComponent); * Creates a new Prism instance with the given language loaded */ export async function createInstance (languages?: string | string[]) { - const instance = new Prism({ languages: toArray(languages) as string[], languagePath: path.join(SRC_DIR, 'languages') }); - await instance.languagesReady; + const instance = new Prism(); + + const protos = await Promise.all(toArray(languages).map(getComponent)); + protos.filter(Boolean).forEach(proto => { + instance.languageRegistry.add(proto as LanguageProto); + }); return instance; } @@ -137,12 +141,18 @@ export function createPrismDOM (): PrismDOM<{}> { /** * Loads the given languages or plugins. */ - const load = async (ids: string | string[], type: 'languages' | 'plugins' = 'languages') => { - let registry = instance[type === 'languages' ? 'languageRegistry' : 'pluginRegistry']; + const load = async (languagesOrPlugins: string | string[]) => { + const protos = await Promise.all(toArray(languagesOrPlugins).map(getComponent)); withGlobals(() => { - registry.loadAll(toArray(ids) as string[]); + protos.filter(Boolean).forEach(proto => { + if (proto.grammar) { + instance.languageRegistry.add(proto); + } + else { + instance.pluginRegistry.add(proto); + } + }); }); - await registry.ready; }; return { @@ -150,8 +160,8 @@ export function createPrismDOM (): PrismDOM<{}> { window: window as PrismWindow<{}>, document: window.document, Prism: window.Prism as never, - loadLanguages: async (ids: string | string[]) => await load(ids), - loadPlugins: async (ids: string | string[]) => await load(ids, 'plugins'), + loadLanguages: load, + loadPlugins: load, withGlobals, }; } From b08255d87d5bc00178a79dcec4e3b756d7a45f23 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Mon, 19 May 2025 22:37:27 +0200 Subject: [PATCH 06/10] Use registry --- tests/coverage.ts | 4 ++-- tests/identifier-test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/coverage.ts b/tests/coverage.ts index 2738a52c2d..f0934fc8b7 100644 --- a/tests/coverage.ts +++ b/tests/coverage.ts @@ -17,9 +17,9 @@ describe('Pattern test coverage', () => { const Prism = await PrismLoader.createInstance(languages); const root = Object.fromEntries( - [...Prism.components['entries'].keys()].map(id => [ + Object.keys(Prism.languageRegistry.cache).map(id => [ id, - Prism.components.getLanguage(id), + Prism.languageRegistry.getLanguage(id)?.resolvedGrammar, ]) ); diff --git a/tests/identifier-test.ts b/tests/identifier-test.ts index 4672cfd20f..7b65af2729 100644 --- a/tests/identifier-test.ts +++ b/tests/identifier-test.ts @@ -122,8 +122,8 @@ function testLiterals (getPrism: Promise, lang: string) { identifierType: keyof IdentifierTestOptions ) { const Prism = await getPrism; - for (const id of Prism.components['entries'].keys()) { - const grammar = Prism.components.getLanguage(id); + for (const id of Object.keys(Prism.languageRegistry.cache)) { + const grammar = Prism.languageRegistry.getLanguage(id)?.resolvedGrammar; if (!grammar) { continue; } From e4aeafacb790e2c4523a07c6fa7da487809ceb74 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Wed, 21 May 2025 15:06:34 +0200 Subject: [PATCH 07/10] Fix core tests --- tests/core/registry.ts | 56 +++++++++++------------------------------- 1 file changed, 15 insertions(+), 41 deletions(-) diff --git a/tests/core/registry.ts b/tests/core/registry.ts index d2afb7a9cd..6dae213aa1 100644 --- a/tests/core/registry.ts +++ b/tests/core/registry.ts @@ -1,60 +1,34 @@ import { assert } from 'chai'; import { Prism } from '../../src/core/prism'; +import type { Grammar } from '../../src/types'; describe('Registry', () => { it('should resolve aliases', () => { - const { components } = new Prism(); + const { languageRegistry } = new Prism(); - const grammar = {}; - components.add({ id: 'a', alias: 'b', grammar }); + const grammar = {} as Grammar; + languageRegistry.add({ id: 'a', alias: 'b', grammar }); - assert.isTrue(components.has('a')); - assert.isTrue(components.has('b')); + assert.equal(languageRegistry.resolveRef('a').id, 'a'); + assert.equal(languageRegistry.resolveRef('b').id, 'a'); - assert.strictEqual(components.resolveAlias('a'), 'a'); - assert.strictEqual(components.resolveAlias('b'), 'a'); + assert.strictEqual(languageRegistry.aliases['b'], 'a'); - assert.strictEqual(components.getLanguage('a'), grammar); - assert.strictEqual(components.getLanguage('b'), grammar); + assert.deepStrictEqual(languageRegistry.getLanguage('a')?.resolvedGrammar, grammar); + assert.deepStrictEqual(languageRegistry.getLanguage('b')?.resolvedGrammar, grammar); }); it('should resolve aliases in optional dependencies', () => { - const { components } = new Prism(); + const { languageRegistry } = new Prism(); - const grammar = {}; - components.add({ id: 'a', alias: 'b', grammar }); - components.add({ + const grammar = {} as Grammar; + languageRegistry.add({ id: 'a', alias: 'b', grammar }); + languageRegistry.add({ id: 'c', optional: 'b', - grammar ({ getOptionalLanguage }) { - return getOptionalLanguage('b') ?? {}; - }, + grammar: ({ getLanguage }) => getLanguage('b') ?? {}, }); - assert.strictEqual(components.getLanguage('c'), grammar); - }); - - it('should throw on circular dependencies', () => { - assert.throws(() => { - const { components } = new Prism(); - - components.add({ id: 'a', optional: 'b', grammar: {} }); - components.add({ id: 'b', optional: 'a', grammar: {} }); - }, /Circular dependency a -> b -> a not allowed/); - - assert.throws(() => { - const { components } = new Prism(); - - components.add( - { id: 'a', optional: 'b', grammar: {} }, - { id: 'b', optional: 'a', grammar: {} } - ); - }, /Circular dependency a -> b -> a not allowed/); - - assert.throws(() => { - const { components } = new Prism(); - - components.add({ id: 'a', optional: 'a', grammar: {} }); - }, /Circular dependency a -> a not allowed/); + assert.deepStrictEqual(languageRegistry.getLanguage('c')?.resolvedGrammar, grammar); }); }); From 935642a686692fc22a0bec63b49d936980607221 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Sat, 24 May 2025 23:46:59 +0200 Subject: [PATCH 08/10] [components-tests] Add types --- tests/components-test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/components-test.ts b/tests/components-test.ts index 38374ea6b2..7c4cd50ced 100644 --- a/tests/components-test.ts +++ b/tests/components-test.ts @@ -5,6 +5,7 @@ import { assert } from 'chai'; import { noop } from '../src/shared/util'; import { forEach, toArray } from '../src/util/iterables'; import { getComponent, getComponentIds, getLanguageIds } from './helper/prism-loader'; +import type { LanguageProto } from '../src/types'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -42,7 +43,7 @@ describe('Components', () => { for (const id of getComponentIds()) { const proto = await getComponent(id).catch(noop); add(id, 'a component id'); - forEach(proto?.alias, a => add(a, `an alias of ${id}`)); + forEach((proto as LanguageProto)?.alias, a => add(a, `an alias of ${id}`)); } }); }); @@ -101,7 +102,7 @@ describe('components.json', () => { describe('- should have valid alias titles', () => { for (const lang of getLanguageIds()) { it(`- ${lang} should have all alias titles registered as alias`, async () => { - const aliases = new Set(toArray((await getComponent(lang))?.alias)); + const aliases = new Set(toArray(((await getComponent(lang)) as LanguageProto)?.alias)); const aliasTitles = (components.languages[lang] as ComponentEntry | undefined)?.aliasTitles ?? {}; @@ -124,7 +125,6 @@ describe('components.json', () => { .map((key): { id: string; title: string } => { return { id: key, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion title: (components.languages[key] as ComponentEntry).title!, }; }); From 35d3b2ae98b7f208d704f599bef6764b2def5c48 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Sat, 24 May 2025 23:51:35 +0200 Subject: [PATCH 09/10] [coverage] Fix ESLint and TS errors + typo --- tests/coverage.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/coverage.ts b/tests/coverage.ts index f0934fc8b7..872da0d341 100644 --- a/tests/coverage.ts +++ b/tests/coverage.ts @@ -24,7 +24,6 @@ describe('Pattern test coverage', () => { ); BFS(root, (path, object) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const { key, value } = path[path.length - 1]; const tokenPath = BFSPathToPrismTokenPath(path); @@ -78,8 +77,8 @@ describe('Pattern test coverage', () => { try { await runTestCase(languageIdentifier, filePath, 'none', createInstance); } - catch (error) { - // we don't case about whether the test succeeds, + catch (error) { // eslint-disable-line @typescript-eslint/no-unused-vars + // we don't care about whether the test succeeds, // we just want to gather usage data } } @@ -112,7 +111,7 @@ describe('Pattern test coverage', () => { }); it(`- should exhaustively cover all keywords in keyword lists`, () => { - const problems = []; + const problems: string[] = []; for (const data of getAllOf(language)) { if (data.matches.length === 0) { From 88567910d881d30f8890173b4ad9d56dcba86ed3 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Sat, 24 May 2025 23:57:29 +0200 Subject: [PATCH 10/10] [pattern-tests] Fix ESLint and TS errors --- tests/pattern-tests.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/pattern-tests.ts b/tests/pattern-tests.ts index 1a306a56a3..65cd6457be 100644 --- a/tests/pattern-tests.ts +++ b/tests/pattern-tests.ts @@ -108,7 +108,6 @@ function testPatterns (getPrism: () => Promise, mainLanguage: visited.add(grammar); BFS(grammar, path => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const { key, value } = path[path.length - 1]; const tokenPath = BFSPathToPrismTokenPath(path, rootStr); visited.add(value); @@ -125,9 +124,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage: ); } - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const parent = path.length > 1 ? path[path.length - 2].value : undefined; - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const lookbehind = key === 'pattern' && !!parent && !!(parent as GrammarToken).lookbehind; const lookbehindGroup = lookbehind @@ -138,7 +135,6 @@ function testPatterns (getPrism: () => Promise, mainLanguage: ast, tokenPath, name: key, - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment parent, path, lookbehind, @@ -282,7 +278,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage: forEachCapturingGroup(ast.pattern, ({ group, number }) => { const isLookbehindGroup = group === lookbehindGroup; if (group.references.length === 0 && !isLookbehindGroup) { - const fixes = []; + const fixes: string[] = []; fixes.push( `Make this group a non-capturing group ('(?:...)' instead of '(...)'). (It's usually this option.)` ); @@ -406,7 +402,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage: * Returns the first capturing group in the given pattern. */ function getFirstCapturingGroup (pattern: Pattern): CapturingGroup | undefined { - let cap = undefined; + let cap: CapturingGroup | undefined = undefined; try { visitRegExpAST(pattern, { @@ -416,7 +412,7 @@ function getFirstCapturingGroup (pattern: Pattern): CapturingGroup | undefined { }, }); } - catch (error) { + catch (error) { // eslint-disable-line @typescript-eslint/no-unused-vars // ignore errors } @@ -784,9 +780,9 @@ interface Highlight { function highlight (highlights: Highlight[], offset = 0) { highlights.sort((a, b) => a.start - b.start); - const lines = []; + const lines: string[] = []; while (highlights.length > 0) { - const newHighlights = []; + const newHighlights: Highlight[] = []; let l = ''; for (const highlight of highlights) { const start = highlight.start + offset;