fix(themes): an imported theme keeps its colours under the semantic layer

scripts/markdownTokenColours.test.ts

@@ -63,3 +63,29 @@ test('an unusable entry is dropped rather than passed to defineTheme', () => {
 	assert.deepEqual(tokens([{ settings: { foreground: '#ff0000' } }]), []);
 	assert.deepEqual(tokens(undefined), []);
 });
+
+test('an imported theme reaches the semantic names too, not only the grammar ones', () => {
+	// The grammar calls italics `emphasis` and the renderer's parse calls it
+	// `emph`; a scope that stopped at the first spelling would leave the second
+	// unstyled. `~~strikethrough~~` has no grammar name at all, so `strike` is
+	// the only way a theme can colour it — which is the half of #676 that could
+	// not be answered until the parse drove the colours.
+	const byToken = new Map(
+		rulesFor([
+			{ scope: 'markup.italic', settings: { foreground: '#00ff00' } },
+			{ scope: 'markup.strikethrough', settings: { foreground: '#888888' } },
+			{ scope: 'markup.heading', settings: { foreground: '#61afef' } },
+			{ scope: 'markup.inline.raw', settings: { foreground: '#98c379' } },
+			{ scope: 'markup.underline.link', settings: { foreground: '#56b6c2' } },
+			{ scope: 'markup.quote', settings: { foreground: '#5c6370' } },
+		]).map((rule) => [rule.token, rule.foreground]),
+	);
+	assert.equal(byToken.get('emph'), '00ff00');
+	assert.equal(byToken.get('emphasis'), '00ff00');
+	assert.equal(byToken.get('strike'), '888888');
+	assert.equal(byToken.get('heading'), '61afef');
+	assert.equal(byToken.get('code'), '98c379');
+	assert.equal(byToken.get('link'), '56b6c2');
+	assert.equal(byToken.get('quote'), '5c6370');
+});
+

scripts/monacoInternals.d.ts

@@ -257,3 +257,33 @@ declare module 'monaco-editor/esm/vs/editor/common/core/wordCharacterClassifier.
 		intlSegmenterLocales: readonly string[],
 	): WordCharacterClassifier;
 }
+
+// The theme trie, for `semanticTokens.test.ts`. It is where a semantic token's
+// colour is actually decided — `_match` walking the token name and falling back
+// to the parent rule — so the test asks Monaco rather than re-implementing the
+// lookup and agreeing with itself.
+declare module 'monaco-editor/esm/vs/editor/common/languages/supports/tokenization.js' {
+	/** Opaque here: the test only passes these from `parseTokenTheme` to `TokenTheme`. */
+	export type ParsedThemeRule = unknown;
+
+	/** `metadata` packs the foreground colour id and the font-style bits. */
+	export interface ResolvedRule {
+		readonly metadata: number;
+	}
+
+	export function parseTokenTheme(
+		source: ReadonlyArray<{ token: string; foreground?: string; background?: string; fontStyle?: string }>,
+	): ParsedThemeRule[];
+
+	export const TokenTheme: {
+		createFromParsedTokenTheme(
+			source: ParsedThemeRule[],
+			customTokenColors: string[],
+		): {
+			/** The rule a token name resolves to, or its nearest ancestor's. */
+			_match(token: string): ResolvedRule;
+			/** Indexed by the colour id `metadata` carries; entries stringify to hex. */
+			getColorMap(): readonly unknown[];
+		};
+	};
+}

scripts/semanticTokens.test.ts

@@ -1,7 +1,10 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
 
+import { TokenTheme, parseTokenTheme } from 'monaco-editor/esm/vs/editor/common/languages/supports/tokenization.js';
+
 import { semanticTokenRules } from '../src/lib/utils/editorTheme.js';
+import { MARKDOWN_SCOPE_ALIASES, importedThemeRules } from '../src/lib/utils/theme.js';
 import { TOKEN_MODIFIERS, TOKEN_TYPES, encodeSemanticTokens } from '../src/lib/utils/semanticTokens.js';
 import { readRustBackend, readSource } from './sourceTree.js';
 
@@ -98,3 +101,81 @@ test('the editor turns the layer on, because the default is off', () => {
 	assert.match(editor, /'semanticHighlighting\.enabled': true/);
 	assert.match(editor, /registerDocumentSemanticTokensProvider\(/);
 });
+
+test('no kind falls through to an imported theme that has nothing to say', () => {
+	// The failure this guards is silent and total. A semantic token whose type
+	// the theme does not name resolves to the theme's *root* rule, whose
+	// foreground is a real colour id rather than "none"
+	// (`semanticTokensProviderStyling.js`, the `if (tokenStyle.foreground)`
+	// branch), and `sparseTokensStore` then masks the grammar's colour out and
+	// paints that plain default over it. An imported theme lost its heading,
+	// link, quote and inline-code colours to this path the moment the semantic
+	// layer shipped, and nothing in either file said so.
+	const rust = readRustBackend();
+	const emitted = [
+		...new Set([...rust.matchAll(/"([a-z]+(?:\.marker)?)"\s*,?\s*out\s*\)/g)].map((match) => match[1])),
+	];
+	assert.ok(emitted.length > 0, 'the extractor must still name its kinds as literals');
+
+	const rules = importedThemeRules(
+		[{ scope: 'markup.bold', settings: { foreground: '#e06c75', fontStyle: 'bold' } }],
+		true,
+	);
+	// `base: 'vs-dark'` with `inherit: true` contributes the default foreground
+	// that every unmatched token resolves to.
+	const theme = TokenTheme.createFromParsedTokenTheme(
+		parseTokenTheme([{ token: '', foreground: '#d4d4d4', background: '#1e1e1e' }, ...rules]),
+		[],
+	);
+	const fallthrough = theme._match('a-token-no-rule-names').metadata;
+
+	const unstyled = emitted.filter((kind) => theme._match(kind).metadata === fallthrough);
+	assert.deepEqual(unstyled, [], 'these repaint an imported theme with its own default foreground');
+});
+
+test('a theme that colours a construct colours all of it, markers included', () => {
+	// Rules are sorted by name, and a child node is cloned from its parent at the
+	// moment it is created. The app's base carries `strike.marker`, which sorts
+	// after the theme's `strike`, so an alias that stopped at the content left
+	// the theme colouring a word and the app colouring the `~~` on either side of
+	// it — one construct in two greys, and the same for `##` against its title.
+	//
+	// Driven off the alias table rather than a list written out here: the failure
+	// arrives when the app names a *longer* token than the alias does, so a new
+	// entry is exactly the case that would otherwise go unchecked.
+	// A colour per *kind*, not per scope: two scopes can name the same construct
+	// (`markup.inline.raw` and `markup.raw.inline` are both inline code), and
+	// giving those two different colours would only test which one sorts last.
+	// Distinct across kinds is what catches a rule landing on the wrong one.
+	const kinds = [...new Set(Object.values(MARKDOWN_SCOPE_ALIASES).map((alias) => alias.kind))];
+	const colourOf = (kind: string) =>
+		`#${(kinds.indexOf(kind) + 1).toString(16).padStart(2, '0').repeat(3)}`;
+	const scopes = Object.entries(MARKDOWN_SCOPE_ALIASES).map(([scope, alias], index) => ({
+		// Every other one language-qualified, which is the commoner spelling in
+		// real themes and reaches the alias lookup by its prefix rule rather
+		// than by an exact hit.
+		scope: index % 2 ? `${scope}.markdown` : scope,
+		kind: alias.kind,
+		foreground: colourOf(alias.kind),
+	}));
+
+	const theme = TokenTheme.createFromParsedTokenTheme(
+		parseTokenTheme([
+			{ token: '', foreground: '#e6e6e6', background: '#12141a' },
+			...importedThemeRules(
+				scopes.map(({ scope, foreground }) => ({ scope, settings: { foreground } })),
+				true,
+			),
+		]),
+		[],
+	);
+	const colours = theme.getColorMap();
+	// The map holds Monaco `Color` objects; their string form is the hex.
+	const foregroundOf = (token: string) =>
+		String(colours[(theme._match(token).metadata >>> 15) & 511]).toLowerCase();
+
+	for (const { scope, kind, foreground } of scopes) {
+		assert.equal(foregroundOf(kind), foreground, `${scope} must reach ${kind}`);
+		assert.equal(foregroundOf(`${kind}.marker`), foreground, `${scope} must reach ${kind}.marker`);
+	}
+});

src/lib/utils/theme.ts

@@ -1,3 +1,5 @@
+import { semanticTokenRules } from './editorTheme.js';
+
 // Values taken from an imported VS Code theme end up concatenated into a global
 // `<style>` block. Anything that is not a colour literal could close the rule and
 // open a new one (`#fff; } * { display:none; background-image:url(https://evil) } :root {`),
@@ -70,6 +72,57 @@ export function sanitizeThemeColors(colors: unknown): Record<string, string> {
 
 export type MonacoTokenRule = { token: string; foreground?: string; fontStyle?: string };
 
+/**
+ * Every rule an imported theme's editor gets, semantic layer included.
+ *
+ * The layer needs a rule for every kind it emits or it erases the theme. A
+ * semantic token whose type the theme does not name resolves to the theme's
+ * *root* rule, whose foreground is a real colour id rather than "none"
+ * (`semanticTokensProviderStyling.js`, the `if (tokenStyle.foreground)`
+ * branch), and `sparseTokensStore` then masks the grammar's colour out and
+ * paints that plain default over it. So the app's own rules go in as a base.
+ *
+ * The base is *rewritten* rather than layered under, and that is the point.
+ * Monaco has no notion of one rule set overriding another: rules are sorted by
+ * name and merged into a trie, where a child node is cloned from its parent at
+ * the moment it is created and never revisited. Between two rules of the same
+ * name, array order decides and a later theme rule wins. Between `strike` and
+ * `strike.marker` it is the trie shape that decides — the longer name is
+ * inserted last, creates the child, and overwrites it — and which set the rule
+ * came from does not enter into it. Layering by array order therefore holds for
+ * exactly as long as the two sets happen to name the same tokens, and inverts
+ * silently the moment the base names a longer one.
+ *
+ * Resolving the theme's colour into the base here means the base's shape stops
+ * mattering: whatever names it declares for a construct, marker or content or
+ * something not invented yet, all of them carry the theme's colour before
+ * Monaco ever sees them.
+ *
+ * Font styles stay the base's. `strike` has to be struck through and `insert`
+ * underlined whatever colour they take, and a theme that does spell out a style
+ * still applies it through its own rule below.
+ */
+export function importedThemeRules(tokenColors: unknown, isDark: boolean): MonacoTokenRule[] {
+	const themeRules = monacoTokenRules(tokenColors);
+
+	/** The colour the theme gave each construct, by the kind the extractor emits. */
+	const byKind = new Map<string, string>();
+	for (const rule of themeRules) {
+		// Read off the scope the theme wrote, which `monacoTokenRules` keeps
+		// beside every alias it derives, so a language-qualified spelling like
+		// `markup.bold.markdown` is matched by the same prefix rule.
+		const alias = markdownAliasFor(rule.token);
+		if (alias && rule.foreground) byKind.set(alias.kind, rule.foreground);
+	}
+
+	const base = semanticTokenRules(isDark ? 'dark' : 'light').map((rule) => {
+		const foreground = byKind.get(rule.token.split('.')[0]);
+		return foreground ? { ...rule, foreground } : rule;
+	});
+
+	return [...base, ...themeRules];
+}
+
 /**
  * The Monaco token names a TextMate scope has to be renamed to before it can
  * colour anything in the editor.
@@ -83,18 +136,33 @@ export type MonacoTokenRule = { token: string; foreground?: string; fontStyle?:
  * which is why the reporter saw *some* of their theme arrive and concluded the
  * rest was unstyleable.
  *
- * Only the four that Monaco actually emits are here. `~~strikethrough~~`,
- * `==highlight==` and `++insert++` are absent from the tokenizer entirely
- * (`basic-languages/markdown/markdown.js` never leaves `linecontent` for
- * them), so no rename reaches them; colouring those means owning a fork of the
- * grammar, and the preview renders all three today.
+ * There are now two sets of names to reach, not one. The grammar's are still
+ * here, and beside them the construct names the semantic layer answers under
+ * (`utils/semanticTokens.ts`) — which is how a scope reaches `~~strikethrough~~`
+ * at last: Monaco's tokenizer never leaves `linecontent` for it, but the
+ * renderer's parse sees it, and a theme rule on `strike` styles what the parse
+ * reports. The two spellings rarely coincide: the grammar calls italics
+ * `emphasis` and the parse calls it `emph`, one letter apart and no match
+ * between them.
+ *
+ * A construct with no `markup.*` scope of its own — a task checkbox, a table,
+ * a wikilink, maths — is not listed and does not need to be: `importedThemeRules`
+ * gives every legend entry a rule before these are applied, so an unlisted one
+ * takes the app's own colour rather than the theme's default foreground.
  */
-const MARKDOWN_SCOPE_ALIASES: Record<string, string> = {
-	'markup.bold': 'strong',
-	'markup.italic': 'emphasis',
-	'markup.inline.raw': 'variable',
-	'markup.raw.inline': 'variable',
-	'markup.underline.link': 'string.link',
+export const MARKDOWN_SCOPE_ALIASES: Record<string, { readonly grammar: readonly string[]; readonly kind: string }> = {
+	'markup.bold': { grammar: [], kind: 'strong' },
+	'markup.italic': { grammar: ['emphasis'], kind: 'emph' },
+	'markup.inline.raw': { grammar: ['variable'], kind: 'code' },
+	'markup.raw.inline': { grammar: ['variable'], kind: 'code' },
+	'markup.underline.link': { grammar: ['string.link'], kind: 'link' },
+	'markup.heading': { grammar: [], kind: 'heading' },
+	'entity.name.section': { grammar: [], kind: 'heading' },
+	'markup.strikethrough': { grammar: [], kind: 'strike' },
+	'markup.quote': { grammar: [], kind: 'quote' },
+	'markup.list': { grammar: [], kind: 'list' },
+	'markup.fenced_code': { grammar: [], kind: 'fence' },
+	'meta.separator': { grammar: [], kind: 'rule' },
 };
 
 /**
@@ -103,13 +171,21 @@ const MARKDOWN_SCOPE_ALIASES: Record<string, string> = {
  * miss most of the themes it exists for. The trailing dot keeps
  * `markup.underline` from being read as `markup.underline.link`.
  */
-function markdownTokenFor(scope: string): string | undefined {
-	for (const [tmScope, token] of Object.entries(MARKDOWN_SCOPE_ALIASES)) {
-		if (scope === tmScope || scope.startsWith(`${tmScope}.`)) return token;
+function markdownAliasFor(scope: string): { readonly grammar: readonly string[]; readonly kind: string } | undefined {
+	for (const [tmScope, alias] of Object.entries(MARKDOWN_SCOPE_ALIASES)) {
+		if (scope === tmScope || scope.startsWith(`${tmScope}.`)) return alias;
 	}
 	return undefined;
 }
 
+function markdownTokensFor(scope: string): readonly string[] {
+	const alias = markdownAliasFor(scope);
+	// The construct name only. Its markers are reached by `importedThemeRules`
+	// rewriting the base, which is the one place that has to know how the base
+	// is spelled — naming them here as well would put that knowledge in two.
+	return alias ? [...alias.grammar, alias.kind] : [];
+}
+
 /**
  * A VS Code theme's `tokenColors` as Monaco theme rules.
  *
@@ -140,12 +216,13 @@ export function monacoTokenRules(tokenColors: unknown): MonacoTokenRule[] {
 					fontStyle: item.settings.fontStyle,
 				};
 				rules.push(rule);
-				const alias = markdownTokenFor(trimmed);
 				// `fontStyle` is carried over, and its absence is not "regular":
 				// Monaco reads a missing one as NotSet and leaves the base
 				// theme's `strong: bold` / `emphasis: italic` standing, so a
 				// colour-only rule adds colour without flattening the text.
-				if (alias) rules.push({ ...rule, token: alias });
+				for (const alias of markdownTokensFor(trimmed)) {
+					rules.push({ ...rule, token: alias });
+				}
 			}
 		}
 	}
@@ -236,7 +313,7 @@ export async function parseAndApplyVscodeTheme(themeJsonStr: string, name: strin
 		// editor, so the dynamic import resolves from cache).
 		const monaco = await import('monaco-editor');
 		if (monaco) {
-			const rules = monacoTokenRules(theme.tokenColors);
+			const rules = importedThemeRules(theme.tokenColors, isDark);
 
 			// Monaco only understands hex colours here; anything else makes
 			// `defineTheme` throw and drops the whole editor theme.