fix(editor): stop Monaco outlining fullwidth CJK punctuation

scripts/editorOptionWiring.test.ts

@@ -1,6 +1,17 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
 
+import {
+	EditorOptions,
+	inUntrustedWorkspace,
+	type UnicodeHighlightOptions,
+} from 'monaco-editor/esm/vs/editor/common/config/editorOptions.js';
+import {
+	UnicodeTextModelHighlighter,
+	type UnicodeHighlightResult,
+} from 'monaco-editor/esm/vs/editor/common/services/unicodeTextModelHighlighter.js';
+import ts from 'typescript';
+
 import { readSource, sliceBetween } from './sourceTree.js';
 
 // Editor.svelte translates the settings store into Monaco options and
@@ -162,3 +173,165 @@ test('Show Whitespace renders every whitespace run, not just trailing', () => {
 		'creation and updateOptions agree on "all"',
 	);
 });
+
+// ----------------------------------------------------------------- executed
+//
+// Everything above matches Editor.svelte as text, because a keybinding and a
+// settings-driven enum cannot be exercised without a DOM. `unicodeHighlight`
+// can do better, and the weak form would be particularly bad here: asserting
+// that the string "ambiguousCharacters" occurs in the file says nothing about
+// which characters Monaco ends up outlining, which is the entire contract.
+//
+// So the option literal is lifted out of the real `monaco.editor.create` call,
+// evaluated, and pushed through the same two pieces of Monaco the running
+// editor uses — `EditorOptions.unicodeHighlight.applyUpdate` to merge it over
+// the shipped defaults, and `UnicodeTextModelHighlighter` to decide what gets a
+// box. A regression has to survive Monaco's own code to reach the assertions.
+
+/** The options object Editor.svelte hands to `monaco.editor.create`, evaluated. */
+function createdEditorOptions(): Record<string, unknown> {
+	const script = sliceBetween(editor, '<script lang="ts">', '</script>');
+	const source = ts.createSourceFile('Editor.ts', script, ts.ScriptTarget.Latest, true);
+
+	const literals: ts.Expression[] = [];
+	const visit = (node: ts.Node): void => {
+		if (ts.isCallExpression(node) && node.expression.getText(source) === 'monaco.editor.create') {
+			assert.equal(node.arguments.length, 2, 'monaco.editor.create(container, options)');
+			literals.push(node.arguments[1]);
+		}
+		ts.forEachChild(node, visit);
+	};
+	visit(source);
+
+	// Also the check that this file is looking at the only editor in the app: a
+	// second `create` call would need the same option and would not be covered
+	// by anything below.
+	assert.equal(literals.length, 1, 'exactly one Monaco editor is constructed in Editor.svelte');
+
+	const js = ts.transpileModule(`(${literals[0].getText(source)})`, {
+		compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext },
+	}).outputText;
+
+	// The literal closes over four locals. None of them can reach
+	// `unicodeHighlight`, which is a constant, so they are stubbed rather than
+	// reconstructed — the settings proxy answers undefined for every key, which
+	// is enough to let the object build.
+	return new Function('settings', 'value', 'language', 'getTheme', `return ${js};`)(
+		new Proxy({}, { get: () => undefined }),
+		'',
+		'markdown',
+		() => 'app-theme-dark',
+	) as Record<string, unknown>;
+}
+
+/**
+ * Monaco's shipped defaults with the component's option merged over them.
+ *
+ * `?? {}` models the real absence case rather than guarding: a component that
+ * passes no `unicodeHighlight` gets Monaco's defaults, which is the state this
+ * whole section exists to move away from. Without it, deleting the option from
+ * Editor.svelte fails the tests below with a TypeError raised inside Monaco
+ * instead of the assertion naming the characters that came back.
+ */
+function effectiveUnicodeHighlight(passed: unknown): UnicodeHighlightOptions {
+	const option = EditorOptions.unicodeHighlight;
+	return option.applyUpdate(option.defaultValue, passed ?? {}).newValue;
+}
+
+/**
+ * What Monaco would outline in `lines`, given an effective option set.
+ *
+ * Mirrors `resolveOptions()` in unicodeHighlighter.ts, which is not exported.
+ * `trusted` is true because that is what standalone Monaco reports —
+ * `StandaloneWorkspaceTrustManagementService.isWorkspaceTrusted()` returns true
+ * unconditionally. That is why `nonBasicASCII`, whose default is the
+ * `inUntrustedWorkspace` sentinel, is already off and needs no setting in
+ * Editor.svelte: it resolves through `!trusted`.
+ */
+function outlined(lines: string[], effective: UnicodeHighlightOptions): UnicodeHighlightResult {
+	const trusted = true;
+	const through = (value: boolean | 'inUntrustedWorkspace') =>
+		value === inUntrustedWorkspace ? !trusted : value;
+	return UnicodeTextModelHighlighter.computeUnicodeHighlights(
+		{ getLineCount: () => lines.length, getLineContent: (n: number) => lines[n - 1] },
+		{
+			nonBasicASCII: through(effective.nonBasicASCII),
+			ambiguousCharacters: effective.ambiguousCharacters,
+			invisibleCharacters: effective.invisibleCharacters,
+			includeComments: through(effective.includeComments),
+			includeStrings: through(effective.includeStrings),
+			allowedCodePoints: Object.keys(effective.allowedCharacters).map((c) => c.codePointAt(0)),
+			// The real value is derived from the OS locale and Monaco's UI
+			// language. It is pinned here because it makes no difference: the
+			// fullwidth forms are confusable in every locale Monaco ships, zh-CN
+			// and ja-JP included, which is why the reporters saw boxes on
+			// CJK systems in the first place.
+			allowedLocales: ['en'],
+		},
+	);
+}
+
+// Prose a CJK author actually types. The boxes need a basic-ASCII character in
+// the same word as the fullwidth punctuation — `shouldHighlightNonBasicASCII`
+// suppresses the highlight when the surrounding word is entirely non-ASCII —
+// so a Latin technical term or a Markdown emphasis marker is what triggers it.
+const CJK_PROSE = [
+	'使用 Monaco，然后保存。',
+	'安装依赖（npm ci），再运行测试。',
+	'这是 Markdown，不是 HTML。',
+	'打开 Settings：Editor，字体大小。',
+	'**粗体**，*斜体*，`代码`。',
+	'标题（English Title）说明',
+	'你好，世界。（测试）！？',
+];
+
+test('the editor turns off ambiguous-character highlighting and nothing else', () => {
+	// Reports #186 and #94 are both Monaco outlining fullwidth punctuation.
+	// Narrowness is the assertion: `unicodeHighlight: false` or a third
+	// sub-option would also make the boxes go away, and would take the
+	// invisible-character warning with it.
+	const options = createdEditorOptions();
+	assert.deepEqual(
+		options.unicodeHighlight,
+		{ ambiguousCharacters: false },
+		'exactly one sub-option is overridden',
+	);
+});
+
+test('no box is drawn on fullwidth CJK punctuation', () => {
+	const effective = effectiveUnicodeHighlight(createdEditorOptions().unicodeHighlight);
+	const found = outlined(CJK_PROSE, effective);
+	assert.equal(
+		found.ranges.length,
+		0,
+		`nothing outlined in CJK prose, got ${JSON.stringify(
+			found.ranges.map((r) => CJK_PROSE[r.startLineNumber - 1].slice(r.startColumn - 1, r.endColumn - 1)),
+		)}`,
+	);
+	assert.equal(found.ambiguousCharacterCount, 0);
+});
+
+test('the fixtures still reproduce the bug once the option is taken away', () => {
+	// Without this the test above could pass because Monaco stopped treating
+	// the fullwidth forms as confusable — an upgrade would quietly leave it
+	// asserting nothing. It fails loudly instead, on the fixtures rather than
+	// on the fix.
+	const withDefaults = effectiveUnicodeHighlight({ ambiguousCharacters: true });
+	const found = outlined(CJK_PROSE, withDefaults);
+	assert.ok(
+		found.ambiguousCharacterCount >= 6,
+		`Monaco's defaults still box CJK punctuation, got ${found.ambiguousCharacterCount}`,
+	);
+});
+
+test('the invisible-character warning survives the fix', () => {
+	// A stray NBSP or zero-width space is a real hazard in Markdown — an NBSP
+	// after `-` stops a list from parsing — and unlike the punctuation it is
+	// never something the author typed on purpose. Turning the whole feature
+	// off would have taken it with it.
+	const effective = effectiveUnicodeHighlight(createdEditorOptions().unicodeHighlight);
+	assert.equal(effective.invisibleCharacters, true, 'left at the Monaco default');
+
+	const found = outlined(['a b', 'x​z'], effective);
+	assert.equal(found.invisibleCharacterCount, 2, 'NBSP and zero-width space are still flagged');
+});

scripts/monacoInternals.d.ts

@@ -0,0 +1,88 @@
+// Types for the two Monaco internals `editorOptionWiring.test.ts` drives
+// directly.
+//
+// Monaco ships declarations for its public API surface (`monaco-editor`) only.
+// The option registry and the Unicode highlighter are plain ESM modules that
+// resolve at runtime — Editor.svelte already deep-imports `monaco-editor/esm/…`
+// for its five workers — but they carry no `.d.ts`, so `npm run check` reads
+// them as implicit `any`.
+//
+// They are declared here rather than left as `any` because the test asserts on
+// what comes back out of them: `ambiguousCharacterCount` silently becoming
+// `undefined` after a Monaco upgrade would turn `assert.equal(count, 0)` into a
+// test that passes while checking nothing. Narrow declarations make that a type
+// error instead. A module that moves outright fails loudly at import time.
+//
+// Only the members the test calls are declared. This is not an attempt to type
+// Monaco's internals in general.
+
+declare module 'monaco-editor/esm/vs/editor/common/config/editorOptions.js' {
+	/** Sentinel default for the options gated on workspace trust. */
+	export const inUntrustedWorkspace: 'inUntrustedWorkspace';
+
+	export interface UnicodeHighlightOptions {
+		nonBasicASCII: boolean | 'inUntrustedWorkspace';
+		invisibleCharacters: boolean;
+		ambiguousCharacters: boolean;
+		includeComments: boolean | 'inUntrustedWorkspace';
+		includeStrings: boolean | 'inUntrustedWorkspace';
+		allowedCharacters: Record<string, boolean>;
+		allowedLocales: Record<string, boolean>;
+	}
+
+	export const EditorOptions: {
+		unicodeHighlight: {
+			readonly defaultValue: UnicodeHighlightOptions;
+			/** Merges a partial option object over `value`, as `editor.create` does. */
+			applyUpdate(
+				value: UnicodeHighlightOptions,
+				update: unknown,
+			): { newValue: UnicodeHighlightOptions };
+		};
+	};
+}
+
+declare module 'monaco-editor/esm/vs/editor/common/services/unicodeTextModelHighlighter.js' {
+	export interface UnicodeHighlightRange {
+		startLineNumber: number;
+		startColumn: number;
+		endLineNumber: number;
+		endColumn: number;
+	}
+
+	export interface UnicodeHighlightResult {
+		ranges: UnicodeHighlightRange[];
+		ambiguousCharacterCount: number;
+		invisibleCharacterCount: number;
+		nonBasicAsciiCharacterCount: number;
+		hasMore: boolean;
+	}
+
+	/** The minimum of `ITextModel` the highlighter reads. */
+	export interface HighlightableModel {
+		getLineCount(): number;
+		getLineContent(lineNumber: number): string;
+	}
+
+	/**
+	 * Resolved form of `UnicodeHighlightOptions`: the workspace-trust sentinels
+	 * are already collapsed to booleans, and the two allow-maps are flattened to
+	 * lists by the caller.
+	 */
+	export interface ResolvedUnicodeHighlightOptions {
+		nonBasicASCII: boolean;
+		ambiguousCharacters: boolean;
+		invisibleCharacters: boolean;
+		includeComments: boolean;
+		includeStrings: boolean;
+		allowedCodePoints: (number | undefined)[];
+		allowedLocales: string[];
+	}
+
+	export const UnicodeTextModelHighlighter: {
+		computeUnicodeHighlights(
+			model: HighlightableModel,
+			options: ResolvedUnicodeHighlightOptions,
+		): UnicodeHighlightResult;
+	};
+}

src/lib/components/Editor.svelte

@@ -273,6 +273,27 @@
 			fontFamily: settings.editorFont,
 			wordBasedSuggestions: "off",
 			quickSuggestions: false,
+			// Monaco's Unicode highlighter is built for source code, where a
+			// character that looks like ASCII but is not is an attack vector. In
+			// prose it fires on the punctuation CJK authors type all day: `，`
+			// `！` `？` `（` `）` `；` `：` are all confusables of an ASCII
+			// counterpart, so Monaco outlines each one (#186, #94).
+			//
+			// It only fires when the confusable shares a word with basic ASCII —
+			// shouldHighlightNonBasicASCII() suppresses the box when the
+			// surrounding word is entirely non-ASCII — which is why pure CJK
+			// looks fine and `使用 Monaco，然后保存。` or `安装依赖（npm ci）` does
+			// not. Latin technical terms inside CJK prose are the normal case,
+			// and `**粗体**，` triggers it with no Latin word at all.
+			//
+			// invisibleCharacters stays at its default: a stray zero-width space
+			// or NBSP is a real hazard in Markdown (an NBSP after `-` stops a
+			// list from parsing) and it never fires on something typed on
+			// purpose. nonBasicASCII needs no setting — it defaults to
+			// `inUntrustedWorkspace` and standalone Monaco's workspace-trust
+			// service returns true unconditionally, so it is already off; were it
+			// on, every ideograph would be boxed rather than the punctuation.
+			unicodeHighlight: { ambiguousCharacters: false },
 			renderWhitespace: settings.showWhitespace ? "all" : "none",
 			padding: { top: 20 },
 			scrollbar: {