diff --git a/apps/desktop/src/snapShot/DesktopSnapShot.test.ts b/apps/desktop/src/snapShot/DesktopSnapShot.test.ts index 7b25a2d8e942..ee9782248354 100644 --- a/apps/desktop/src/snapShot/DesktopSnapShot.test.ts +++ b/apps/desktop/src/snapShot/DesktopSnapShot.test.ts @@ -2479,7 +2479,7 @@ it.each(["client", "frame"] as const)( }, ); -it.each(["darwin", "win32"] as const)( +it.each(["darwin", "win32", "linux"] as const)( "extracts the same structured accessibility tree on %s", async (platform) => { const bounds = { x: 100, y: 200, width: 800, height: 600 }; @@ -2487,7 +2487,13 @@ it.each(["darwin", "win32"] as const)( role: "window", name: "Editor", bounds, - tree: async () => ({ name: "Editor", children: [{ name: "Save", children: [] }] }), + tree: async () => ({ + name: "Editor", + children: [ + { name: "Save", children: [] }, + { name: "Below scroll view", children: [] }, + ], + }), children: async () => [ { role: "button", @@ -2495,6 +2501,13 @@ it.each(["darwin", "win32"] as const)( bounds: { x: 300, y: 350, width: 100, height: 50 }, children: async () => [], }, + { + role: "static_text", + name: "Below scroll view", + bounds: { x: 300, y: 1_000, width: 100, height: 50 }, + visible: false, + children: async () => [], + }, ], }; accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] }); @@ -2516,6 +2529,12 @@ it.each(["darwin", "win32"] as const)( : undefined, "Save", ); + assert.deepInclude( + result?.accessibility?.format === "element-tree" + ? result.accessibility.root.children[1] + : undefined, + { name: "Below scroll view", bounds: null, state: { visible: false } }, + ); assert.lengthOf(accessibilityByPidMock.mock.calls, platform === "win32" ? 0 : 1); assert.lengthOf(accessibilityForegroundMock.mock.calls, platform === "win32" ? 1 : 0); }, @@ -2554,6 +2573,48 @@ it.each(["darwin", "win32"] as const)( }, ); +it.each([ + ["darwin", "", "value", 650], + ["win32", "", "value", 650], + ["linux", "x11", "value", 650], + ["linux", "wayland", "value", 650], + ["darwin", "", "name", 100], +] as const)( + "preserves long text outside the scroll view on %s %s (%s)", + async (platform, session, field, lines) => { + const text = `${"Document line\n".repeat(lines)}End of document`; + vi.stubEnv("XDG_SESSION_TYPE", session); + const bounds = { x: 0, y: 0, width: 800, height: 600 }; + const window = { + role: "window", + name: "Editor", + bounds, + tree: async () => ({ name: "Editor", children: [{ [field]: text, children: [] }] }), + children: async () => [ + { role: "text_area", [field]: text, bounds, children: async () => [] }, + ], + }; + accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] }); + accessibilityForegroundMock + .mockReset() + .mockResolvedValue({ pid: 123, asElement: () => window }); + try { + const result = await readAccessibleWindowContext( + { title: "Editor", bounds, owner: { processId: 123 } }, + platform, + "Editor", + ); + assert.deepEqual(result?.accessibility, { + format: "flat-text", + text: `Editor\n${text}`, + truncated: false, + }); + } finally { + vi.unstubAllEnvs(); + } + }, +); + it.each([ { names: ["⠙ t3code"], expected: "Verified text" }, { names: ["⠋ t3code", "⠙ t3code"], expected: undefined }, @@ -2681,6 +2742,81 @@ it("falls back to completed flat text when rich traversal reaches the deadline", } }); +it("keeps a truncated element tree when the flat text read fails", async () => { + const bounds = { x: 0, y: 0, width: 800, height: 600 }; + accessibilityByPidMock.mockReset().mockResolvedValue({ + children: async () => [ + { + role: "window", + name: "Editor", + bounds, + tree: async () => { + throw new Error("Text read failed"); + }, + children: async () => [ + { + role: "text_area", + value: "x".repeat(8_001), + bounds: { x: 10, y: 50, width: 700, height: 500 }, + children: async () => [], + }, + ], + }, + ], + }); + + const result = await readAccessibleWindowContext( + { title: "Editor", bounds, owner: { processId: 123 } }, + "darwin", + "Editor", + ); + assert.equal(result?.accessibility?.format, "element-tree"); + assert.isTrue(result?.accessibility?.truncated); + assert.deepEqual( + result?.accessibility?.format === "element-tree" + ? result.accessibility.root.children[0]?.bounds + : undefined, + { x: 10, y: 50, width: 700, height: 500 }, + ); +}); + +it("keeps a truncated tree when flat text would not recover any text", async () => { + const bounds = { x: 0, y: 0, width: 800, height: 600 }; + accessibilityByPidMock.mockReset().mockResolvedValue({ + children: async () => [ + { + role: "window", + name: "Editor", + bounds, + tree: async () => ({ name: "Editor", children: [{ name: "Help", children: [] }] }), + children: async () => [ + { + role: "button", + name: "Help", + description: "Help text ".repeat(250), + bounds, + children: async () => [], + }, + ], + }, + ], + }); + + const result = await readAccessibleWindowContext( + { title: "Editor", bounds, owner: { processId: 123 } }, + "darwin", + "Editor", + ); + assert.equal(result?.accessibility?.format, "element-tree"); + assert.isTrue(result?.accessibility?.truncated); + assert.include( + result?.accessibility?.format === "element-tree" + ? result.accessibility.root.children[0]?.description + : undefined, + "Help text", + ); +}); + it("times out after three seconds without overlapping the outstanding accessibility read", async () => { vi.useFakeTimers(); accessibilityByPidMock.mockReset(); diff --git a/apps/desktop/src/snapShot/SnapShotAccessibility.ts b/apps/desktop/src/snapShot/SnapShotAccessibility.ts index bf20322e14a7..bb6cb95952b3 100644 --- a/apps/desktop/src/snapShot/SnapShotAccessibility.ts +++ b/apps/desktop/src/snapShot/SnapShotAccessibility.ts @@ -57,25 +57,27 @@ function accessibilityReadSnapshot( descendantLocationsReliable: progress.richLocationsReliable, }) : undefined; - const accessibleText = - progress.accessibleText ?? - (richTree - ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS) - : undefined); + const richText = richTree + ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS) + : undefined; + const accessibleText = progress.accessibleText ?? richText; + const richTruncated = progress.richTruncated || richTree?.truncated === true; const accessibility: SnapShotAccessibility | undefined = - progress.richComplete && richTree + progress.richComplete && + richTree && + (!richTruncated || !progress.accessibleText || progress.accessibleText === richText) ? { format: "element-tree", coordinateSpace: "captured-image", imageSize, - truncated: progress.richTruncated || richTree.truncated, + truncated: richTruncated, root: richTree.root, } - : progress.flatComplete && accessibleText + : progress.flatComplete && progress.accessibleText ? { format: "flat-text", - text: accessibleText, - truncated: accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS, + text: progress.accessibleText, + truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS, } : richTree ? { diff --git a/apps/desktop/src/snapShot/snapShot.test.ts b/apps/desktop/src/snapShot/snapShot.test.ts index ca9a97a2492a..6960718759aa 100644 --- a/apps/desktop/src/snapShot/snapShot.test.ts +++ b/apps/desktop/src/snapShot/snapShot.test.ts @@ -263,6 +263,23 @@ describe("accessibleWindowElementTree", () => { expect(tree?.root.children.length).toBeLessThan(10); expect(JSON.stringify(tree).length).toBeLessThanOrEqual(32_000); }); + + it.each(["name", "value", "description"] as const)( + "reports clipped %s text in partial trees", + async (field) => { + const tree = await accessibleWindowElementTree( + { + role: "window", + children: async () => [{ role: "text_area", [field]: "x".repeat(8_001) }], + }, + { x: 0, y: 0, width: 800, height: 600 }, + { width: 800, height: 600 }, + ); + + expect(tree?.truncated).toBe(true); + expect(tree?.root.children[0]?.[field]?.length).toBeLessThan(8_001); + }, + ); }); describe("compactAccessibilityTree", () => { diff --git a/apps/desktop/src/snapShot/snapShot.ts b/apps/desktop/src/snapShot/snapShot.ts index 4664d68a387f..35c240c6e09a 100644 --- a/apps/desktop/src/snapShot/snapShot.ts +++ b/apps/desktop/src/snapShot/snapShot.ts @@ -155,11 +155,16 @@ function safeProperty(read: () => T): T | undefined { } } -export function boundedSnapShotString(value: unknown, maxChars: number): string | undefined { +export function boundedSnapShotString( + value: unknown, + maxChars: number, + onTruncated?: () => void, +): string | undefined { if (typeof value !== "string") return undefined; const candidate = value.replaceAll("\0", "").trim(); if (!candidate) return undefined; if (candidate.length <= maxChars) return candidate; + onTruncated?.(); const end = /[\uD800-\uDBFF]/.test(candidate[maxChars - 1] ?? "") ? maxChars - 1 : maxChars; return candidate.slice(0, end).trimEnd(); } @@ -204,18 +209,22 @@ function accessibilityNode( imageSize: CapturedImageSize, isRoot: boolean, locationsReliable: boolean, + onTruncated: () => void, ): MutableAccessibilityNode { const name = boundedSnapShotString( safeProperty(() => element.name), 1_000, + onTruncated, ); const value = boundedSnapShotString( safeProperty(() => element.value), 8_000, + onTruncated, ); const description = boundedSnapShotString( safeProperty(() => element.description), 2_000, + onTruncated, ); const checked = safeProperty(() => element.checked); const expanded = safeProperty(() => element.expanded); @@ -389,6 +398,9 @@ export async function accessibleWindowElementTree( imageSize, required, options.locationsReliable !== false, + () => { + truncated = true; + }, ); nodes += 1; root ??= node;