diff --git a/scripts/editorOptionWiring.test.ts b/scripts/editorOptionWiring.test.ts index 26e5c95b..9e15aa7e 100644 --- a/scripts/editorOptionWiring.test.ts +++ b/scripts/editorOptionWiring.test.ts @@ -1,6 +1,17 @@ import assert from 'node:assert/strict'; import test from 'node:test'; +import { + EditorOptions, + inUntrustedWorkspace, + type UnicodeHighlightOptions, +} from 'monaco-editor/esm/vs/editor/common/config/editorOptions.js'; +import { + UnicodeTextModelHighlighter, + type UnicodeHighlightResult, +} from 'monaco-editor/esm/vs/editor/common/services/unicodeTextModelHighlighter.js'; +import ts from 'typescript'; + import { readSource, sliceBetween } from './sourceTree.js'; // Editor.svelte translates the settings store into Monaco options and @@ -162,3 +173,165 @@ test('Show Whitespace renders every whitespace run, not just trailing', () => { 'creation and updateOptions agree on "all"', ); }); + +// ----------------------------------------------------------------- executed +// +// Everything above matches Editor.svelte as text, because a keybinding and a +// settings-driven enum cannot be exercised without a DOM. `unicodeHighlight` +// can do better, and the weak form would be particularly bad here: asserting +// that the string "ambiguousCharacters" occurs in the file says nothing about +// which characters Monaco ends up outlining, which is the entire contract. +// +// So the option literal is lifted out of the real `monaco.editor.create` call, +// evaluated, and pushed through the same two pieces of Monaco the running +// editor uses — `EditorOptions.unicodeHighlight.applyUpdate` to merge it over +// the shipped defaults, and `UnicodeTextModelHighlighter` to decide what gets a +// box. A regression has to survive Monaco's own code to reach the assertions. + +/** The options object Editor.svelte hands to `monaco.editor.create`, evaluated. */ +function createdEditorOptions(): Record { + const script = sliceBetween(editor, ''); + const source = ts.createSourceFile('Editor.ts', script, ts.ScriptTarget.Latest, true); + + const literals: ts.Expression[] = []; + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node) && node.expression.getText(source) === 'monaco.editor.create') { + assert.equal(node.arguments.length, 2, 'monaco.editor.create(container, options)'); + literals.push(node.arguments[1]); + } + ts.forEachChild(node, visit); + }; + visit(source); + + // Also the check that this file is looking at the only editor in the app: a + // second `create` call would need the same option and would not be covered + // by anything below. + assert.equal(literals.length, 1, 'exactly one Monaco editor is constructed in Editor.svelte'); + + const js = ts.transpileModule(`(${literals[0].getText(source)})`, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText; + + // The literal closes over four locals. None of them can reach + // `unicodeHighlight`, which is a constant, so they are stubbed rather than + // reconstructed — the settings proxy answers undefined for every key, which + // is enough to let the object build. + return new Function('settings', 'value', 'language', 'getTheme', `return ${js};`)( + new Proxy({}, { get: () => undefined }), + '', + 'markdown', + () => 'app-theme-dark', + ) as Record; +} + +/** + * Monaco's shipped defaults with the component's option merged over them. + * + * `?? {}` models the real absence case rather than guarding: a component that + * passes no `unicodeHighlight` gets Monaco's defaults, which is the state this + * whole section exists to move away from. Without it, deleting the option from + * Editor.svelte fails the tests below with a TypeError raised inside Monaco + * instead of the assertion naming the characters that came back. + */ +function effectiveUnicodeHighlight(passed: unknown): UnicodeHighlightOptions { + const option = EditorOptions.unicodeHighlight; + return option.applyUpdate(option.defaultValue, passed ?? {}).newValue; +} + +/** + * What Monaco would outline in `lines`, given an effective option set. + * + * Mirrors `resolveOptions()` in unicodeHighlighter.ts, which is not exported. + * `trusted` is true because that is what standalone Monaco reports — + * `StandaloneWorkspaceTrustManagementService.isWorkspaceTrusted()` returns true + * unconditionally. That is why `nonBasicASCII`, whose default is the + * `inUntrustedWorkspace` sentinel, is already off and needs no setting in + * Editor.svelte: it resolves through `!trusted`. + */ +function outlined(lines: string[], effective: UnicodeHighlightOptions): UnicodeHighlightResult { + const trusted = true; + const through = (value: boolean | 'inUntrustedWorkspace') => + value === inUntrustedWorkspace ? !trusted : value; + return UnicodeTextModelHighlighter.computeUnicodeHighlights( + { getLineCount: () => lines.length, getLineContent: (n: number) => lines[n - 1] }, + { + nonBasicASCII: through(effective.nonBasicASCII), + ambiguousCharacters: effective.ambiguousCharacters, + invisibleCharacters: effective.invisibleCharacters, + includeComments: through(effective.includeComments), + includeStrings: through(effective.includeStrings), + allowedCodePoints: Object.keys(effective.allowedCharacters).map((c) => c.codePointAt(0)), + // The real value is derived from the OS locale and Monaco's UI + // language. It is pinned here because it makes no difference: the + // fullwidth forms are confusable in every locale Monaco ships, zh-CN + // and ja-JP included, which is why the reporters saw boxes on + // CJK systems in the first place. + allowedLocales: ['en'], + }, + ); +} + +// Prose a CJK author actually types. The boxes need a basic-ASCII character in +// the same word as the fullwidth punctuation — `shouldHighlightNonBasicASCII` +// suppresses the highlight when the surrounding word is entirely non-ASCII — +// so a Latin technical term or a Markdown emphasis marker is what triggers it. +const CJK_PROSE = [ + '使用 Monaco,然后保存。', + '安装依赖(npm ci),再运行测试。', + '这是 Markdown,不是 HTML。', + '打开 Settings:Editor,字体大小。', + '**粗体**,*斜体*,`代码`。', + '标题(English Title)说明', + '你好,世界。(测试)!?', +]; + +test('the editor turns off ambiguous-character highlighting and nothing else', () => { + // Reports #186 and #94 are both Monaco outlining fullwidth punctuation. + // Narrowness is the assertion: `unicodeHighlight: false` or a third + // sub-option would also make the boxes go away, and would take the + // invisible-character warning with it. + const options = createdEditorOptions(); + assert.deepEqual( + options.unicodeHighlight, + { ambiguousCharacters: false }, + 'exactly one sub-option is overridden', + ); +}); + +test('no box is drawn on fullwidth CJK punctuation', () => { + const effective = effectiveUnicodeHighlight(createdEditorOptions().unicodeHighlight); + const found = outlined(CJK_PROSE, effective); + assert.equal( + found.ranges.length, + 0, + `nothing outlined in CJK prose, got ${JSON.stringify( + found.ranges.map((r) => CJK_PROSE[r.startLineNumber - 1].slice(r.startColumn - 1, r.endColumn - 1)), + )}`, + ); + assert.equal(found.ambiguousCharacterCount, 0); +}); + +test('the fixtures still reproduce the bug once the option is taken away', () => { + // Without this the test above could pass because Monaco stopped treating + // the fullwidth forms as confusable — an upgrade would quietly leave it + // asserting nothing. It fails loudly instead, on the fixtures rather than + // on the fix. + const withDefaults = effectiveUnicodeHighlight({ ambiguousCharacters: true }); + const found = outlined(CJK_PROSE, withDefaults); + assert.ok( + found.ambiguousCharacterCount >= 6, + `Monaco's defaults still box CJK punctuation, got ${found.ambiguousCharacterCount}`, + ); +}); + +test('the invisible-character warning survives the fix', () => { + // A stray NBSP or zero-width space is a real hazard in Markdown — an NBSP + // after `-` stops a list from parsing — and unlike the punctuation it is + // never something the author typed on purpose. Turning the whole feature + // off would have taken it with it. + const effective = effectiveUnicodeHighlight(createdEditorOptions().unicodeHighlight); + assert.equal(effective.invisibleCharacters, true, 'left at the Monaco default'); + + const found = outlined(['a b', 'x​z'], effective); + assert.equal(found.invisibleCharacterCount, 2, 'NBSP and zero-width space are still flagged'); +}); diff --git a/scripts/monacoInternals.d.ts b/scripts/monacoInternals.d.ts new file mode 100644 index 00000000..3ca885b5 --- /dev/null +++ b/scripts/monacoInternals.d.ts @@ -0,0 +1,88 @@ +// Types for the two Monaco internals `editorOptionWiring.test.ts` drives +// directly. +// +// Monaco ships declarations for its public API surface (`monaco-editor`) only. +// The option registry and the Unicode highlighter are plain ESM modules that +// resolve at runtime — Editor.svelte already deep-imports `monaco-editor/esm/…` +// for its five workers — but they carry no `.d.ts`, so `npm run check` reads +// them as implicit `any`. +// +// They are declared here rather than left as `any` because the test asserts on +// what comes back out of them: `ambiguousCharacterCount` silently becoming +// `undefined` after a Monaco upgrade would turn `assert.equal(count, 0)` into a +// test that passes while checking nothing. Narrow declarations make that a type +// error instead. A module that moves outright fails loudly at import time. +// +// Only the members the test calls are declared. This is not an attempt to type +// Monaco's internals in general. + +declare module 'monaco-editor/esm/vs/editor/common/config/editorOptions.js' { + /** Sentinel default for the options gated on workspace trust. */ + export const inUntrustedWorkspace: 'inUntrustedWorkspace'; + + export interface UnicodeHighlightOptions { + nonBasicASCII: boolean | 'inUntrustedWorkspace'; + invisibleCharacters: boolean; + ambiguousCharacters: boolean; + includeComments: boolean | 'inUntrustedWorkspace'; + includeStrings: boolean | 'inUntrustedWorkspace'; + allowedCharacters: Record; + allowedLocales: Record; + } + + export const EditorOptions: { + unicodeHighlight: { + readonly defaultValue: UnicodeHighlightOptions; + /** Merges a partial option object over `value`, as `editor.create` does. */ + applyUpdate( + value: UnicodeHighlightOptions, + update: unknown, + ): { newValue: UnicodeHighlightOptions }; + }; + }; +} + +declare module 'monaco-editor/esm/vs/editor/common/services/unicodeTextModelHighlighter.js' { + export interface UnicodeHighlightRange { + startLineNumber: number; + startColumn: number; + endLineNumber: number; + endColumn: number; + } + + export interface UnicodeHighlightResult { + ranges: UnicodeHighlightRange[]; + ambiguousCharacterCount: number; + invisibleCharacterCount: number; + nonBasicAsciiCharacterCount: number; + hasMore: boolean; + } + + /** The minimum of `ITextModel` the highlighter reads. */ + export interface HighlightableModel { + getLineCount(): number; + getLineContent(lineNumber: number): string; + } + + /** + * Resolved form of `UnicodeHighlightOptions`: the workspace-trust sentinels + * are already collapsed to booleans, and the two allow-maps are flattened to + * lists by the caller. + */ + export interface ResolvedUnicodeHighlightOptions { + nonBasicASCII: boolean; + ambiguousCharacters: boolean; + invisibleCharacters: boolean; + includeComments: boolean; + includeStrings: boolean; + allowedCodePoints: (number | undefined)[]; + allowedLocales: string[]; + } + + export const UnicodeTextModelHighlighter: { + computeUnicodeHighlights( + model: HighlightableModel, + options: ResolvedUnicodeHighlightOptions, + ): UnicodeHighlightResult; + }; +} diff --git a/src/lib/components/Editor.svelte b/src/lib/components/Editor.svelte index 63a88d4d..1240d765 100644 --- a/src/lib/components/Editor.svelte +++ b/src/lib/components/Editor.svelte @@ -273,6 +273,27 @@ fontFamily: settings.editorFont, wordBasedSuggestions: "off", quickSuggestions: false, + // Monaco's Unicode highlighter is built for source code, where a + // character that looks like ASCII but is not is an attack vector. In + // prose it fires on the punctuation CJK authors type all day: `,` + // `!` `?` `(` `)` `;` `:` are all confusables of an ASCII + // counterpart, so Monaco outlines each one (#186, #94). + // + // It only fires when the confusable shares a word with basic ASCII — + // shouldHighlightNonBasicASCII() suppresses the box when the + // surrounding word is entirely non-ASCII — which is why pure CJK + // looks fine and `使用 Monaco,然后保存。` or `安装依赖(npm ci)` does + // not. Latin technical terms inside CJK prose are the normal case, + // and `**粗体**,` triggers it with no Latin word at all. + // + // invisibleCharacters stays at its default: a stray zero-width space + // or NBSP is a real hazard in Markdown (an NBSP after `-` stops a + // list from parsing) and it never fires on something typed on + // purpose. nonBasicASCII needs no setting — it defaults to + // `inUntrustedWorkspace` and standalone Monaco's workspace-trust + // service returns true unconditionally, so it is already off; were it + // on, every ideograph would be boxed rather than the punctuation. + unicodeHighlight: { ambiguousCharacters: false }, renderWhitespace: settings.showWhitespace ? "all" : "none", padding: { top: 20 }, scrollbar: {