diff --git a/apps/desktop/scripts/picker-frames.smoke.mjs b/apps/desktop/scripts/picker-frames.smoke.mjs new file mode 100644 index 000000000000..1fb6f445b4c8 --- /dev/null +++ b/apps/desktop/scripts/picker-frames.smoke.mjs @@ -0,0 +1,291 @@ +// Run explicitly with Chromium installed: node apps/desktop/scripts/picker-frames.smoke.mjs +// Optional arguments: live editor URL, evidence directory. Never launches the installed app. +import * as NodeAssert from "node:assert/strict"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; +import { build } from "vite-plus"; +import { chromium } from "playwright-core"; + +const entry = NodeURL.fileURLToPath(new URL("../src/preview/PickPreload.ts", import.meta.url)); +const result = await build({ + configFile: false, + logLevel: "error", + plugins: [ + { + name: "picker-ipc-harness", + enforce: "pre", + resolveId(id) { + if (id === "electron") return "\0picker-ipc"; + }, + load(id) { + if (id !== "\0picker-ipc") return; + return `const listeners = new Map(); + globalThis.pickerMessages = []; + globalThis.pickerEmit = (channel, ...args) => { + for (const fn of [...(listeners.get(channel) ?? [])]) fn({}, ...args); + }; + export const ipcRenderer = { + on(channel, fn) { listeners.set(channel, [...(listeners.get(channel) ?? []), fn]); return this; }, + off(channel, fn) { listeners.set(channel, (listeners.get(channel) ?? []).filter(x => x !== fn)); return this; }, + send(channel, ...args) { globalThis.pickerMessages.push({channel, args}); } + };`; + }, + }, + ], + build: { write: false, minify: false, lib: { entry, formats: ["iife"], name: "PickerSmoke" } }, +}); +const bundle = (Array.isArray(result) ? result[0] : result).output.find( + (x) => x.type === "chunk", +).code; +const browser = await chromium.launch({ headless: true }); +const [liveUrl, evidenceDir, liveTarget = ".document-title"] = process.argv.slice(2); +if (evidenceDir) await NodeFSP.mkdir(evidenceDir, { recursive: true }); + +async function install(page) { + // Retain a test-only reference to the closed root without changing its mode. + await page.evaluate(() => { + const original = Element.prototype.attachShadow; + Element.prototype.attachShadow = function (options) { + const root = original.call(this, options); + if (this.hasAttribute("data-t3code-annotation-ui")) globalThis.pickerRoot = root; + return root; + }; + }); + await page.evaluate(bundle); + await page.evaluate(() => pickerEmit("preview:start-pick")); +} + +async function attach(page, expected, name) { + NodeAssert.equal( + await page.evaluate( + () => + [...pickerRoot.querySelectorAll("button")].find((x) => x.textContent === "Attach").disabled, + ), + false, + `${name}: target selected`, + ); + await page.evaluate(() => { + pickerRoot.querySelector("textarea").value = "Frame annotation regression"; + [...pickerRoot.querySelectorAll("button")].find((x) => x.textContent === "Attach").click(); + }); + await page.waitForFunction(() => + pickerMessages.some((x) => x.channel === "preview:element-picked"), + ); + const message = await page.evaluate(() => + pickerMessages.find((x) => x.channel === "preview:element-picked"), + ); + const [annotation, crop, submission] = message.args; + NodeAssert.equal(submission, "attach"); + NodeAssert.equal(annotation.comment, "Frame annotation regression"); + NodeAssert.equal(annotation.elements.length, 1); + NodeAssert.match(annotation.elements[0].element.htmlPreview, expected); + NodeAssert.equal(annotation.pageUrl, page.url()); + NodeAssert.equal( + await page.evaluate((picked) => { + let owner = document; + for (const frame of picked.framePath ?? []) { + if (owner.URL !== frame.pageUrl) return false; + owner = owner.querySelector(frame.selector)?.contentDocument; + if (!owner) return false; + } + return ( + owner.URL === picked.pageUrl && + (owner.title.trim() || null) === picked.pageTitle && + (!picked.selector || owner.querySelector(picked.selector)?.localName === picked.tagName) + ); + }, annotation.elements[0].element), + true, + "frame path and selector resolve in the captured document", + ); + const rect = annotation.elements[0].rect; + NodeAssert.ok(crop.x <= rect.x && crop.y <= rect.y); + NodeAssert.ok(crop.x + crop.width >= rect.x + rect.width - 1); + NodeAssert.ok(crop.y + crop.height >= rect.y + rect.height - 1); + const screenshot = await page.screenshot({ clip: crop, timeout: 10000 }); + NodeAssert.ok(screenshot.length > 100); + if (evidenceDir) { + await NodeFSP.writeFile(NodePath.join(evidenceDir, `${name}.png`), screenshot); + await NodeFSP.writeFile( + NodePath.join(evidenceDir, `${name}.json`), + JSON.stringify({ annotation, crop, submission }, null, 2), + ); + } + await page.evaluate(() => pickerEmit("preview:annotation-captured")); + NodeAssert.equal(await page.locator("[data-t3code-annotation-ui]").count(), 0); + return rect; +} + +try { + const page = await browser.newPage({ viewport: { width: 1200, height: 900 } }); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.setContent(` + `); + await page.locator("#outer").evaluate((frame) => { + frame.srcdoc = ``; + }); + const target = page.frameLocator("#outer").frameLocator("#inner").locator("#target"); + await target.waitFor(); + await target.evaluate((element) => + element.addEventListener("click", () => (element.dataset.clicked = "yes")), + ); + if (evidenceDir) await page.screenshot({ path: NodePath.join(evidenceDir, "nested-before.png") }); + await install(page); + await target.click(); + NodeAssert.equal(await target.getAttribute("data-clicked"), null); + const expected = await target.boundingBox(); + const humanPointer = await page.evaluate( + () => + pickerMessages.find( + (message) => + message.channel === "preview:human-input" && message.args[0].kind === "pointer", + )?.args[0], + ); + NodeAssert.ok(humanPointer, "frame clicks report human control"); + NodeAssert.ok(Math.abs(humanPointer.x - (expected.x + expected.width / 2)) < 1); + NodeAssert.ok(Math.abs(humanPointer.y - (expected.y + expected.height / 2)) < 1); + const rect = await attach(page, /Nested frame target/, "nested-frames"); + for (const key of ["x", "y", "width", "height"]) + NodeAssert.ok( + Math.abs(rect[key] - expected[key]) < 1, + `${key}: ${rect[key]} vs ${expected[key]}`, + ); + await target.click(); + NodeAssert.equal( + await target.getAttribute("data-clicked"), + "yes", + "capture must restore page input", + ); + + await page.evaluate(() => { + pickerMessages.length = 0; + pickerEmit("preview:start-pick"); + }); + await page.locator("#top").click(); + await attach(page, /Top target/, "top-document"); + + await page.evaluate(() => { + pickerMessages.length = 0; + pickerEmit("preview:start-pick"); + [...pickerRoot.querySelectorAll("button")].find((x) => x.textContent === "Region").click(); + }); + const marqueeTarget = await target.boundingBox(); + await page.mouse.move(marqueeTarget.x - 2, marqueeTarget.y - 2); + await page.mouse.down(); + await page.mouse.move( + marqueeTarget.x + marqueeTarget.width + 2, + marqueeTarget.y + marqueeTarget.height + 2, + ); + await page.mouse.up(); + await attach(page, /Nested frame target/, "frame-marquee"); + + // Scroll and style editing operate across realms, and Escape restores page state. + await page.evaluate(() => { + pickerMessages.length = 0; + pickerEmit("preview:start-pick"); + }); + await target.click(); + await target.evaluate((element) => element.ownerDocument.defaultView.scrollTo(0, 30)); + const scrolled = await target.boundingBox(); + await page.waitForFunction( + (expected) => + [...pickerRoot.querySelectorAll("div")].some( + (node) => + node.style.border.startsWith("2px") && + node.style.display === "block" && + Math.abs(node.getBoundingClientRect().y - expected.y) < 1 && + Math.abs(node.getBoundingClientRect().width - expected.width) < 1, + ), + scrolled, + ); + const baseline = await target.evaluate((element) => element.style.fontSize); + await page.evaluate(() => { + const input = [...pickerRoot.querySelectorAll("label")] + .find((label) => label.firstChild.textContent === "Font size") + .querySelector("input"); + input.value = "24"; + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + NodeAssert.equal(await target.evaluate((element) => element.style.fontSize), "24px"); + await target.press("Escape"); + NodeAssert.equal(await page.locator("[data-t3code-annotation-ui]").count(), 0); + NodeAssert.equal(await target.evaluate((element) => element.style.fontSize), baseline); + + // Frames inserted and navigated during a session must acquire fresh handlers. + await page.evaluate(() => { + pickerMessages.length = 0; + pickerEmit("preview:start-pick"); + }); + await page.locator("#outer").evaluate((frame) => { + frame.srcdoc = ``; + }); + const replacement = page.frameLocator("#outer").locator("#replacement"); + await replacement.click(); + await attach(page, /Replacement document/, "frame-navigation"); + await page.evaluate(() => { + pickerMessages.length = 0; + pickerEmit("preview:start-pick"); + }); + await page.evaluate(() => { + const frame = document.createElement("iframe"); + frame.id = "dynamic"; + frame.srcdoc = ''; + document.body.prepend(frame); + }); + await page.frameLocator("#dynamic").locator("#added").click(); + await attach(page, /Dynamically inserted target/, "dynamic-frame"); + await page.evaluate(() => { + pickerMessages.length = 0; + pickerEmit("preview:start-pick"); + }); + await page.frameLocator("#dynamic").locator("#added").click(); + await page.locator("#dynamic").evaluate((frame) => frame.remove()); + await page.waitForFunction( + () => + [...pickerRoot.querySelectorAll("button")].find((x) => x.textContent === "Attach").disabled, + ); + await page.evaluate(() => pickerEmit("preview:cancel-pick")); + NodeAssert.deepEqual(errors, []); + console.log( + "PASS: nested srcdoc picking, translated geometry/crop, packaging, top-document input, frame navigation and capture cleanup", + ); + + const urls = await browser.newPage(); + await urls.route("http://picker.example/**", (route) => + route.fulfill({ + contentType: "text/html", + body: route.request().url().endsWith("/child") + ? 'Embedded preview' + : `Editor shell`, + }), + ); + await urls.goto("http://picker.example/"); + await install(urls); + await urls.frameLocator("#same").locator("#url-target").click(); + await attach(urls, /Same-origin URL target/, "same-origin-url"); + await urls.evaluate(() => pickerEmit("preview:start-pick")); + await urls.frameLocator("#opaque").locator("#outside").click(); + NodeAssert.equal(await urls.frameLocator("#opaque").locator("#outside").textContent(), "123"); + await urls.evaluate(() => pickerEmit("preview:cancel-pick")); + console.log("PASS: same-origin URL frames and opaque-origin exclusion"); + + if (liveUrl) { + const live = await browser.newPage({ viewport: { width: 1600, height: 1100 } }); + await live.goto(liveUrl); + await live.locator(".template-editor-workspace").waitFor(); + await live.locator(".template-editor-switch input").uncheck(); + const facsimile = live.frameLocator('iframe[title$="editable English Facsimile"]'); + const element = facsimile.locator(liveTarget); + await element.waitFor(); + await element.scrollIntoViewIfNeeded(); + if (evidenceDir) + await live.screenshot({ path: NodePath.join(evidenceDir, "facsimile-before.png") }); + await install(live); + await element.click(); + await attach(live, /data-binding-id/, "facsimile-after"); + console.log("PASS: populated live Facsimile selection, screenshot crop and annotation payload"); + } +} finally { + await browser.close(); +} diff --git a/apps/desktop/src/preview/FramePicking.test.ts b/apps/desktop/src/preview/FramePicking.test.ts new file mode 100644 index 000000000000..6e1a18d4c0a4 --- /dev/null +++ b/apps/desktop/src/preview/FramePicking.test.ts @@ -0,0 +1,335 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + frameDocument, + hasInlineStyle, + isElement, + observeFrameDocuments, + pickInDocument, + topViewportPoint, + topViewportRect, +} from "./FramePicking.ts"; + +class Rect { + x: number; + y: number; + width: number; + height: number; + constructor(x = 0, y = 0, width = 0, height = 0) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + get left() { + return this.x; + } + get top() { + return this.y; + } + get right() { + return this.x + this.width; + } + get bottom() { + return this.y + this.height; + } +} + +function fixture() { + const style = { + paddingLeft: "2px", + paddingTop: "2px", + paddingRight: "2px", + paddingBottom: "2px", + }; + const top = { + defaultView: { innerWidth: 800, innerHeight: 600, getComputedStyle: () => style }, + } as unknown as Document; + const frame = { + nodeType: 1, + localName: "iframe", + isConnected: true, + ownerDocument: top, + offsetWidth: 112, + offsetHeight: 112, + clientWidth: 104, + clientHeight: 104, + clientLeft: 4, + clientTop: 4, + getBoundingClientRect: () => new Rect(100, 50, 224, 224), + contentDocument: null as Document | null, + }; + const child = { + defaultView: { + innerWidth: 100, + innerHeight: 100, + frameElement: frame, + getComputedStyle: () => ({ + paddingLeft: "0px", + paddingTop: "0px", + paddingRight: "0px", + paddingBottom: "0px", + }), + }, + } as unknown as Document; + frame.contentDocument = child; + const element = { + nodeType: 1, + localName: "button", + style: {}, + isConnected: true, + ownerDocument: child, + getBoundingClientRect: () => new Rect(10, 20, 30, 40), + } as unknown as Element; + return { top, frame, child, element }; +} + +beforeEach(() => vi.stubGlobal("DOMRect", Rect)); +afterEach(() => vi.unstubAllGlobals()); + +describe("frame picking coordinates", () => { + it("includes scaled iframe borders and padding in selected rects and pointer coordinates", () => { + const { top, element } = fixture(); + expect(topViewportRect(element, top)).toEqual(new Rect(132, 102, 60, 80)); + expect( + topViewportPoint({ target: element, clientX: 15, clientY: 25 } as unknown as MouseEvent, top), + ).toEqual({ x: 142, y: 112 }); + }); + + it("clips through nested frame viewports before translating to the top viewport", () => { + const { top, child } = fixture(); + const nestedFrame = { + localName: "iframe", + isConnected: true, + ownerDocument: child, + offsetWidth: 44, + offsetHeight: 44, + clientWidth: 40, + clientHeight: 40, + clientLeft: 2, + clientTop: 2, + getBoundingClientRect: () => new Rect(20, 10, 22, 22), + contentDocument: null as Document | null, + }; + const nested = { + defaultView: { innerWidth: 40, innerHeight: 40, frameElement: nestedFrame }, + } as unknown as Document; + nestedFrame.contentDocument = nested; + const element = { + isConnected: true, + ownerDocument: nested, + getBoundingClientRect: () => new Rect(-10, 30, 50, 30), + } as unknown as Element; + expect(topViewportRect(element, top)).toEqual(new Rect(154, 114, 40, 10)); + }); + + it("drops detached, navigated, and now-inaccessible frame documents", () => { + const { top, frame, child, element } = fixture(); + frame.isConnected = false; + expect(topViewportRect(element, top)).toBeNull(); + frame.isConnected = true; + frame.contentDocument = null; + expect(topViewportRect(element, top)).toBeNull(); + Object.defineProperty(child.defaultView, "frameElement", { + get: () => { + throw new Error("SecurityError"); + }, + }); + expect(topViewportRect(element, top)).toBeNull(); + }); + + it("descends accessible frames using content coordinates and preserves inaccessible frame fallback", () => { + const { top, frame, child, element } = fixture(); + top.elementsFromPoint = () => [frame as unknown as Element]; + child.elementsFromPoint = vi.fn(() => [element]); + expect(pickInDocument(top, 142, 112, () => false)).toBe(element); + expect(child.elementsFromPoint).toHaveBeenCalledWith(15, 25); + frame.contentDocument = null; + expect(pickInDocument(top, 142, 112, () => false)).toBe(frame); + expect(frameDocument(frame as unknown as Element)).toBeNull(); + }); + + it.each([ + { transform: "matrix(0, 1, -1, 0, 0, 0)" }, + { transform: "matrix(1, 0, 0.25, 1, 0, 0)" }, + { transform: "matrix(-1, 0, 0, 1, 0, 0)" }, + { transform: "matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, -0.002, 0, 0, 0, 1)" }, + { perspective: "500px" }, + { rotate: "15deg" }, + { scale: "-1 1" }, + ])("falls back to the frame for unsupported transforms: %o", (unsupported) => { + const { top, frame, child, element } = fixture(); + const originalStyle = top.defaultView!.getComputedStyle(frame as unknown as Element); + top.defaultView!.getComputedStyle = () => ({ ...originalStyle, ...unsupported }); + top.elementsFromPoint = () => [frame as unknown as Element]; + child.elementsFromPoint = vi.fn(() => [element]); + expect(topViewportRect(element, top)).toBeNull(); + expect( + topViewportPoint({ target: element, clientX: 15, clientY: 25 } as unknown as MouseEvent, top), + ).toBeNull(); + expect(pickInDocument(top, 142, 112, () => false)).toBe(frame); + expect(child.elementsFromPoint).not.toHaveBeenCalled(); + }); + + it("rejects unsupported transforms on iframe ancestors", () => { + const { top, frame, element } = fixture(); + const ancestor = {} as Element; + Object.defineProperty(frame, "parentElement", { value: ancestor }); + const originalStyle = top.defaultView!.getComputedStyle(frame as unknown as Element); + top.defaultView!.getComputedStyle = (candidate) => + candidate === ancestor ? { ...originalStyle, rotate: "45deg" } : originalStyle; + expect(topViewportRect(element, top)).toBeNull(); + }); + + it.each([ + "matrix(2, 0, 0, 2, 10, 20)", + "matrix3d(2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 10, 20, 0, 1)", + ])("retains supported axis-aligned transforms: %s", (transform) => { + const { top, frame, element } = fixture(); + const originalStyle = top.defaultView!.getComputedStyle(frame as unknown as Element); + top.defaultView!.getComputedStyle = () => ({ ...originalStyle, transform }); + expect(topViewportRect(element, top)).toEqual(new Rect(132, 102, 60, 80)); + }); + + it("recognizes child-realm elements without top-window instanceof checks", () => { + const { element } = fixture(); + expect(isElement(element)).toBe(true); + expect(hasInlineStyle(element)).toBe(true); + expect(isElement(null)).toBe(false); + expect(isElement({ nodeType: 3 })).toBe(false); + }); +}); + +function observerFixture() { + const callbacks = new Map(); + class Observer { + callback: MutationCallback; + constructor(callback: MutationCallback) { + this.callback = callback; + } + observe(owner: Document) { + callbacks.set(owner, this.callback); + } + disconnect() {} + } + vi.stubGlobal("MutationObserver", Observer); + const frameList: HTMLIFrameElement[] = []; + const queryFrames = vi.fn(() => frameList); + const top = { querySelectorAll: queryFrames } as unknown as Document; + const child = { querySelectorAll: vi.fn(() => []) } as unknown as Document; + const frame = Object.assign(new EventTarget(), { + nodeType: 1, + localName: "iframe", + contentDocument: child, + }) as unknown as HTMLIFrameElement; + const unrelated = { + nodeType: 1, + localName: "div", + querySelector: vi.fn(() => null), + } as unknown as Element; + const lifecycle: string[] = []; + const attach = vi.fn((owner: Document) => { + lifecycle.push(owner === top ? "attach top" : "attach child"); + return () => lifecycle.push(owner === top ? "remove top" : "remove child"); + }); + const changed = vi.fn(); + const watch = observeFrameDocuments(top, attach, changed, () => false); + const mutate = (record: Partial) => + callbacks.get(top)!( + [ + { + target: unrelated, + addedNodes: [], + removedNodes: [], + ...record, + } as unknown as MutationRecord, + ], + {} as MutationObserver, + ); + return { + frameList, + queryFrames, + top, + child, + frame, + unrelated, + lifecycle, + attach, + changed, + watch, + mutate, + }; +} + +describe("frame document observation", () => { + it("does not rescan documents for animated attributes or unrelated text changes", async () => { + const fixture = observerFixture(); + for (let index = 0; index < 60; index += 1) { + fixture.mutate({ type: "attributes", attributeName: "style" }); + fixture.mutate({ type: "attributes", attributeName: "class" }); + fixture.mutate({ type: "childList", addedNodes: [{ nodeType: 3 }] as unknown as NodeList }); + await Promise.resolve(); + } + expect(fixture.queryFrames).toHaveBeenCalledTimes(1); + expect(fixture.attach).toHaveBeenCalledTimes(1); + expect(fixture.changed).toHaveBeenCalledTimes(60); + expect(fixture.unrelated.querySelector).not.toHaveBeenCalled(); + fixture.watch.dispose(); + }); + + it("batches frame-containing subtree insertion and removes old document listeners", async () => { + const fixture = observerFixture(); + fixture.frameList.push(fixture.frame); + const subtree = { + nodeType: 1, + localName: "section", + querySelector: () => fixture.frame, + } as unknown as Node; + fixture.mutate({ type: "childList", addedNodes: [subtree] as unknown as NodeList }); + fixture.mutate({ type: "childList", addedNodes: [fixture.frame] as unknown as NodeList }); + await Promise.resolve(); + expect(fixture.queryFrames).toHaveBeenCalledTimes(2); + expect(fixture.watch.documents()).toEqual([fixture.top, fixture.child]); + expect(fixture.changed).toHaveBeenCalledTimes(1); + fixture.frameList.length = 0; + fixture.mutate({ type: "childList", removedNodes: [subtree] as unknown as NodeList }); + await Promise.resolve(); + expect(fixture.queryFrames).toHaveBeenCalledTimes(3); + expect(fixture.watch.documents()).toEqual([fixture.top]); + expect(fixture.lifecycle).toEqual(["attach top", "attach child", "remove child"]); + fixture.watch.dispose(); + }); + + it("cleans the replaced document before attaching its successor on frame load", async () => { + const fixture = observerFixture(); + fixture.frameList.push(fixture.frame); + fixture.mutate({ type: "childList", addedNodes: [fixture.frame] as unknown as NodeList }); + await Promise.resolve(); + const successor = { querySelectorAll: () => [] } as unknown as Document; + Object.assign(fixture.frame, { contentDocument: successor }); + fixture.frame.dispatchEvent(new Event("load")); + fixture.frame.dispatchEvent(new Event("load")); + await Promise.resolve(); + expect(fixture.queryFrames).toHaveBeenCalledTimes(3); + expect(fixture.watch.documents()).toEqual([fixture.top, successor]); + expect(fixture.lifecycle).toEqual([ + "attach top", + "attach child", + "remove child", + "attach child", + ]); + fixture.watch.dispose(); + fixture.frame.dispatchEvent(new Event("load")); + await Promise.resolve(); + expect(fixture.queryFrames).toHaveBeenCalledTimes(3); + }); + + it("discards queued work after the annotation session is disposed", async () => { + const fixture = observerFixture(); + fixture.mutate({ type: "childList", addedNodes: [fixture.frame] as unknown as NodeList }); + fixture.watch.dispose(); + await Promise.resolve(); + expect(fixture.queryFrames).toHaveBeenCalledTimes(1); + expect(fixture.changed).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/desktop/src/preview/FramePicking.ts b/apps/desktop/src/preview/FramePicking.ts new file mode 100644 index 000000000000..599a545bd4a4 --- /dev/null +++ b/apps/desktop/src/preview/FramePicking.ts @@ -0,0 +1,282 @@ +/** DOM helpers shared by the top-level annotation preload and accessible child documents. */ +export function isElement(value: unknown): value is Element { + return typeof value === "object" && value !== null && "nodeType" in value && value.nodeType === 1; +} + +export function hasInlineStyle(element: Element): element is HTMLElement | SVGElement { + return "style" in element; +} + +export function frameDocument(element: Element): Document | null { + if (element.localName !== "iframe") return null; + try { + return (element as HTMLIFrameElement).contentDocument; + } catch { + return null; + } +} + +function owningFrame(owner: Document): HTMLIFrameElement | null { + try { + const frame = owner.defaultView?.frameElement; + return frame?.isConnected && frameDocument(frame) === owner + ? (frame as HTMLIFrameElement) + : null; + } catch { + // A retained document may already have navigated to an inaccessible origin. + return null; + } +} + +/** Rect-based conversion supports positive axis-aligned scaling and translation only. */ +function hasSupportedTransform(style: CSSStyleDeclaration): boolean { + if (style.perspective && style.perspective !== "none") return false; + if (style.rotate && style.rotate !== "none" && style.rotate !== "0deg") return false; + if ( + style.scale && + style.scale !== "none" && + style.scale.split(/\s+/).some((value) => !(Number.parseFloat(value) > 0)) + ) + return false; + if (!style.transform || style.transform === "none") return true; + const match = /^(matrix|matrix3d)\(([^)]+)\)$/.exec(style.transform); + if (!match) return false; + const values = match[2]!.split(",").map(Number); + if (values.some((value) => !Number.isFinite(value))) return false; + if (match[1] === "matrix") { + return ( + values.length === 6 && values[0]! > 0 && values[3]! > 0 && values[1] === 0 && values[2] === 0 + ); + } + return ( + values.length === 16 && + values[0]! > 0 && + values[5]! > 0 && + values[10]! > 0 && + values[15] === 1 && + [1, 2, 3, 4, 6, 7, 8, 9, 11].every((index) => values[index] === 0) + ); +} + +function frameGeometry(frame: HTMLIFrameElement) { + const view = frame.ownerDocument.defaultView; + if (!view) return null; + for (let ancestor: Element | null = frame; ancestor; ancestor = ancestor.parentElement) { + if (!hasSupportedTransform(view.getComputedStyle(ancestor))) return null; + } + const bounds = frame.getBoundingClientRect(); + const style = view.getComputedStyle(frame); + const scaleX = frame.offsetWidth ? bounds.width / frame.offsetWidth : 0; + const scaleY = frame.offsetHeight ? bounds.height / frame.offsetHeight : 0; + const paddingLeft = Number.parseFloat(style.paddingLeft) || 0; + const paddingTop = Number.parseFloat(style.paddingTop) || 0; + const width = frame.clientWidth - paddingLeft - (Number.parseFloat(style.paddingRight) || 0); + const height = frame.clientHeight - paddingTop - (Number.parseFloat(style.paddingBottom) || 0); + return { + x: bounds.left + (frame.clientLeft + paddingLeft) * scaleX, + y: bounds.top + (frame.clientTop + paddingTop) * scaleY, + width: Math.max(0, width) * scaleX, + height: Math.max(0, height) * scaleY, + scaleX, + scaleY, + }; +} + +function intersect(rect: DOMRect, clip: { x: number; y: number; width: number; height: number }) { + const x = Math.max(rect.x, clip.x); + const y = Math.max(rect.y, clip.y); + return new DOMRect( + x, + y, + Math.max(0, Math.min(rect.right, clip.x + clip.width) - x), + Math.max(0, Math.min(rect.bottom, clip.y + clip.height) - y), + ); +} + +/** Converts a child viewport rect through each frame's content box, clipping at every viewport. */ +export function topViewportRect( + element: Element, + topDocument: Document = document, +): DOMRect | null { + if (!element.isConnected) return null; + let owner = element.ownerDocument; + let rect = element.getBoundingClientRect(); + while (owner !== topDocument) { + const view = owner.defaultView; + const frame = owningFrame(owner); + if (!view || !frame) return null; + rect = intersect(rect, { x: 0, y: 0, width: view.innerWidth, height: view.innerHeight }); + const geometry = frameGeometry(frame); + if (!geometry) return null; + rect = intersect( + new DOMRect( + geometry.x + rect.x * geometry.scaleX, + geometry.y + rect.y * geometry.scaleY, + rect.width * geometry.scaleX, + rect.height * geometry.scaleY, + ), + geometry, + ); + owner = frame.ownerDocument; + } + const view = topDocument.defaultView; + return view + ? intersect(rect, { x: 0, y: 0, width: view.innerWidth, height: view.innerHeight }) + : null; +} + +export function topViewportPoint(event: MouseEvent, topDocument: Document = document) { + let owner = isElement(event.target) ? event.target.ownerDocument : topDocument; + let x = event.clientX; + let y = event.clientY; + while (owner !== topDocument) { + const frame = owningFrame(owner); + if (!frame) return null; + const geometry = frameGeometry(frame); + if (!geometry) return null; + x = geometry.x + x * geometry.scaleX; + y = geometry.y + y * geometry.scaleY; + owner = frame.ownerDocument; + } + return { x, y }; +} + +export function pickInDocument( + owner: Document, + x: number, + y: number, + ignore: (element: Element) => boolean, +): Element | null { + for (const candidate of owner.elementsFromPoint(x, y)) { + if (ignore(candidate) || candidate === owner.documentElement || candidate === owner.body) + continue; + const child = frameDocument(candidate); + if (child) { + const geometry = frameGeometry(candidate as HTMLIFrameElement); + if ( + geometry && + geometry.scaleX > 0 && + geometry.scaleY > 0 && + x >= geometry.x && + y >= geometry.y && + x < geometry.x + geometry.width && + y < geometry.y + geometry.height + ) { + const picked = pickInDocument( + child, + (x - geometry.x) / geometry.scaleX, + (y - geometry.y) / geometry.scaleY, + ignore, + ); + if (picked) return picked; + } + } + return candidate; + } + return null; +} + +/** Tracks frame loads and DOM replacement without enabling preloads or Node in subframes. */ +export function observeFrameDocuments( + topDocument: Document, + attach: (owner: Document) => () => void, + changed: () => void, + ignore: (element: Element) => boolean, +) { + const documents = new Map void>(); + const frames = new Map void>(); + let disposed = false; + let updateQueued = false; + let discoveryNeeded = false; + const queueUpdate = (discover: boolean) => { + discoveryNeeded ||= discover; + if (updateQueued || disposed) return; + updateQueued = true; + queueMicrotask(() => { + updateQueued = false; + if (disposed) return; + const discoverFrames = discoveryNeeded; + discoveryNeeded = false; + if (discoverFrames) refresh(); + else changed(); + }); + }; + const containsFrame = (node: Node) => + isElement(node) && (node.localName === "iframe" || node.querySelector("iframe") !== null); + const refresh = (notify = true) => { + if (disposed) return; + const foundDocuments = new Set(); + const foundFrames = new Set(); + const visit = (owner: Document) => { + foundDocuments.add(owner); + for (const frame of owner.querySelectorAll("iframe")) { + foundFrames.add(frame); + const child = frameDocument(frame); + if (child) visit(child); + } + }; + visit(topDocument); + for (const [owner, cleanup] of documents) { + if (!foundDocuments.has(owner)) { + cleanup(); + documents.delete(owner); + } + } + for (const [frame, cleanup] of frames) { + if (!foundFrames.has(frame)) { + cleanup(); + frames.delete(frame); + } + } + // Initial about:blank and srcdoc documents can share a Window. Remove the + // old document's listeners before adding the same callbacks to its successor. + for (const owner of foundDocuments) { + if (!documents.has(owner)) { + const cleanup = attach(owner); + const observer = new MutationObserver((records) => { + const pageChanges = records.filter( + (record) => !isElement(record.target) || !ignore(record.target), + ); + if (pageChanges.length === 0) return; + // Style animation and text updates can move a selection, but cannot + // introduce frame documents. Search only added/removed subtrees here. + const discover = pageChanges.some( + (record) => + record.type === "childList" && + [...record.addedNodes, ...record.removedNodes].some(containsFrame), + ); + queueUpdate(discover); + }); + observer.observe(owner, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ["style", "class", "hidden", "width", "height"], + }); + documents.set(owner, () => { + observer.disconnect(); + cleanup(); + }); + } + } + for (const frame of foundFrames) { + if (!frames.has(frame)) { + const loaded = () => queueUpdate(true); + frame.addEventListener("load", loaded); + frames.set(frame, () => frame.removeEventListener("load", loaded)); + } + } + if (notify) changed(); + }; + refresh(false); + return { + documents: () => Array.from(documents.keys()), + dispose: () => { + disposed = true; + for (const cleanup of documents.values()) cleanup(); + for (const cleanup of frames.values()) cleanup(); + documents.clear(); + frames.clear(); + }, + }; +} diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index 6155c4119ec8..cac878202a62 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -14,6 +14,15 @@ import type { PreviewAnnotationSubmission, } from "@t3tools/contracts"; +import { + hasInlineStyle, + isElement, + observeFrameDocuments, + pickInDocument, + topViewportPoint, + topViewportRect, +} from "./FramePicking.ts"; + import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts"; import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts"; import { @@ -85,10 +94,12 @@ const applyAnnotationTheme = ( const reportHumanPointerInput = (event: PointerEvent): void => { if (!event.isTrusted) return; + const point = topViewportPoint(event); + if (!point) return; ipcRenderer.send(HUMAN_INPUT_CHANNEL, { kind: "pointer", - x: event.clientX, - y: event.clientY, + x: point.x, + y: point.y, button: event.button, }); }; @@ -186,25 +197,19 @@ function unionRects( }; } -function isAnnotationNode(element: Element): boolean { - return element instanceof Element && element.closest(`[${OVERLAY_ATTRIBUTE}]`) !== null; +function isAnnotationNode(element: unknown): boolean { + return isElement(element) && element.closest(`[${OVERLAY_ATTRIBUTE}]`) !== null; } function pickFromPoint(clientX: number, clientY: number): Element | null { - for (const candidate of document.elementsFromPoint(clientX, clientY)) { - if (!(candidate instanceof Element)) continue; - if (isAnnotationNode(candidate)) continue; - if (candidate === document.documentElement || candidate === document.body) continue; - return candidate; - } - return null; + return pickInDocument(document, clientX, clientY, isAnnotationNode); } function describeRawElement(element: Element): string { const tag = element.tagName.toLowerCase(); const id = element.id ? `#${element.id}` : ""; const classes = - element instanceof HTMLElement && typeof element.className === "string" + typeof element.className === "string" ? element.className .trim() .split(/\s+/) @@ -255,12 +260,12 @@ function createLabel(): HTMLDivElement { } function updateSelectedVisual(target: SelectedElement): void { - if (!target.element.isConnected) { + const rect = topViewportRect(target.element); + if (!rect || !isUsableRect(rect)) { target.outline.style.display = "none"; target.label.style.display = "none"; return; } - const rect = target.element.getBoundingClientRect(); positionBox(target.outline, rectFromDomRect(rect)); target.label.textContent = describeRawElement(target.element); target.label.style.display = "block"; @@ -300,6 +305,28 @@ function withCaptureTimeout(promise: Promise, millis: number): Promise { + const path: { pageUrl: string; selector: string }[] = []; + let owner = element.ownerDocument; + while (owner !== document) { + const frame = owner.defaultView?.frameElement; + if (!frame) break; + const selectors: string[] = []; + for (let node: Element | null = frame; node; node = node.parentElement) { + const parent = node.parentElement; + selectors.unshift( + parent + ? `${node.localName}:nth-child(${Array.from(parent.children).indexOf(node) + 1})` + : ":root", + ); + } + owner = frame.ownerDocument; + path.unshift({ pageUrl: owner.URL, selector: selectors.join(" > ") }); + } + return path; +} + /** * Describes a picked element. The React context lookup can stall or throw on * some pages, so the element is never dropped: without context it still @@ -307,9 +334,12 @@ const HTML_PREVIEW_MAX_CHARS = 500; * pick instead of falling back to the whole viewport. */ async function captureElement(element: Element): Promise { + const owner = element.ownerDocument; + const framePath = captureFramePath(element); const base = { - pageUrl: location.href, - pageTitle: document.title?.trim() || null, + pageUrl: owner.URL, + pageTitle: owner.title?.trim() || null, + ...(framePath.length > 0 ? { framePath } : {}), tagName: element.tagName.toLowerCase(), pickedAt: new Date().toISOString(), }; @@ -566,11 +596,13 @@ function startAnnotation(): void { } if (tool !== "select") hoverOutline.style.display = "none"; if (tool !== "marquee") marqueeBox.style.display = "none"; - document.documentElement.setAttribute("data-t3code-annotation-tool", tool); + for (const owner of frameDocuments.documents()) { + owner.documentElement?.setAttribute("data-t3code-annotation-tool", tool); + } }; const removeSelected = (target: SelectedElement): void => { - if (target.element instanceof HTMLElement || target.element instanceof SVGElement) { + if (hasInlineStyle(target.element)) { for (const [property, baseline] of target.baselineStyles) { if (baseline) target.element.style.setProperty(property, baseline); else target.element.style.removeProperty(property); @@ -618,15 +650,17 @@ function startAnnotation(): void { const setStyleForSelected = (property: string, value: string): void => { for (const target of selected.values()) { - if (!(target.element instanceof HTMLElement || target.element instanceof SVGElement)) - continue; + if (!hasInlineStyle(target.element)) continue; if (!target.baselineStyles.has(property)) { target.baselineStyles.set(property, target.element.style.getPropertyValue(property)); } const key = `${target.id}:${property}`; const previousValue = styleChanges.get(key)?.previousValue ?? - getComputedStyle(target.element).getPropertyValue(property).trim(); + target.element.ownerDocument + .defaultView!.getComputedStyle(target.element) + .getPropertyValue(property) + .trim(); target.element.style.setProperty(property, value, "important"); styleChanges.set(key, { targetId: target.id, @@ -838,7 +872,7 @@ function startAnnotation(): void { const syncStyleControls = (): void => { const first = selected.values().next().value as SelectedElement | undefined; if (!first) return; - const computed = getComputedStyle(first.element); + const computed = first.element.ownerDocument.defaultView!.getComputedStyle(first.element); const rect = first.element.getBoundingClientRect(); aspectRatio = rect.height > 0 ? rect.width / rect.height : 1; widthInput.value = String(Math.round(rect.width)); @@ -906,9 +940,10 @@ function startAnnotation(): void { const getAnnotationBounds = (): PreviewAnnotationRect | null => unionRects( [ - ...Array.from(selected.values(), (target) => - rectFromDomRect(target.element.getBoundingClientRect()), - ), + ...Array.from(selected.values()).flatMap((target) => { + const rect = topViewportRect(target.element); + return rect && isUsableRect(rect) ? [rectFromDomRect(rect)] : []; + }), ...regions.map((region) => region.rect), ...strokes.map((stroke) => stroke.bounds), ], @@ -1016,14 +1051,19 @@ function startAnnotation(): void { dragHandle.addEventListener("pointercancel", onEditorPointerUp); const repaint = (): void => { - for (const target of selected.values()) updateSelectedVisual(target); + clearHoverOutline(); + for (const target of selected.values()) { + if (!topViewportRect(target.element)) removeSelected(target); + else updateSelectedVisual(target); + } + svg.setAttribute("viewBox", `0 0 ${window.innerWidth} ${window.innerHeight}`); queueEditorLayout(); }; const removeTargetAtPoint = (x: number, y: number): boolean => { for (const target of Array.from(selected.values()).toReversed()) { - const rect = target.element.getBoundingClientRect(); - if (x >= rect.left && x <= rect.right && y >= rect.top && y <= rect.bottom) { + const rect = topViewportRect(target.element); + if (rect && x >= rect.left && x <= rect.right && y >= rect.top && y <= rect.bottom) { removeSelected(target); return true; } @@ -1058,9 +1098,14 @@ function startAnnotation(): void { }; const selectElementsInRect = (rect: PreviewAnnotationRect): number => { - const candidates = Array.from(document.querySelectorAll("body *")) + const candidates = frameDocuments + .documents() + .flatMap((owner) => Array.from(owner.querySelectorAll("body *"))) .filter((element) => !isAnnotationNode(element)) - .map((element) => ({ element, rect: element.getBoundingClientRect() })) + .flatMap((element) => { + const rect = topViewportRect(element); + return rect ? [{ element, rect }] : []; + }) .filter(({ rect: candidate }) => { if (candidate.width < 2 || candidate.height < 2) return false; return !( @@ -1079,8 +1124,8 @@ function startAnnotation(): void { centerY >= rect.y && centerY <= rect.y + rect.height && (element.children.length === 0 || - element instanceof HTMLButtonElement || - element instanceof HTMLAnchorElement || + element.localName === "button" || + element.localName === "a" || element.getAttribute("role") === "button") ); }) @@ -1097,29 +1142,26 @@ function startAnnotation(): void { }; const onPointerMove = (event: PointerEvent): void => { + const point = topViewportPoint(event); + if (!point) return; if (isAnnotationNode(event.target as Element)) { clearHoverOutline(); return; } if (tool === "select" && dragStart === null) { - const target = pickFromPoint(event.clientX, event.clientY); - if (target) positionBox(hoverOutline, rectFromDomRect(target.getBoundingClientRect())); + const target = pickFromPoint(point.x, point.y); + const rect = target ? topViewportRect(target) : null; + if (rect && isUsableRect(rect)) positionBox(hoverOutline, rectFromDomRect(rect)); else clearHoverOutline(); return; } clearHoverOutline(); if (tool === "marquee" && dragStart) { - positionBox( - marqueeBox, - normalizeRect(dragStart.x, dragStart.y, event.clientX, event.clientY), - ); + positionBox(marqueeBox, normalizeRect(dragStart.x, dragStart.y, point.x, point.y)); return; } if (tool === "draw" && activeStroke) { - activeStroke.target.points = [ - ...activeStroke.target.points, - { x: event.clientX, y: event.clientY }, - ]; + activeStroke.target.points = [...activeStroke.target.points, { x: point.x, y: point.y }]; activeStroke.target.bounds = strokeBounds( activeStroke.target.points, activeStroke.target.width, @@ -1129,19 +1171,21 @@ function startAnnotation(): void { }; const onPointerDown = (event: PointerEvent): void => { + const point = topViewportPoint(event); + if (!point) return; if (event.button !== 0 || isAnnotationNode(event.target as Element)) return; event.preventDefault(); event.stopPropagation(); if (tool === "select") { - const target = pickFromPoint(event.clientX, event.clientY); + const target = pickFromPoint(point.x, point.y); if (target) toggleSelected(target, event.shiftKey); return; } if (tool === "erase") { - removeTargetAtPoint(event.clientX, event.clientY); + removeTargetAtPoint(point.x, point.y); return; } - dragStart = { x: event.clientX, y: event.clientY }; + dragStart = { x: point.x, y: point.y }; if (tool === "draw") { const stroke: PreviewAnnotationStrokeTarget = { id: nextId("stroke"), @@ -1164,11 +1208,13 @@ function startAnnotation(): void { }; const onPointerUp = (event: PointerEvent): void => { + const point = topViewportPoint(event); + if (!point) return; if (!dragStart) return; event.preventDefault(); event.stopPropagation(); if (tool === "marquee") { - const rect = normalizeRect(dragStart.x, dragStart.y, event.clientX, event.clientY); + const rect = normalizeRect(dragStart.x, dragStart.y, point.x, point.y); marqueeBox.style.display = "none"; if (isUsableRect(rect)) { const found = selectElementsInRect(rect); @@ -1199,6 +1245,14 @@ function startAnnotation(): void { event.stopPropagation(); }; + const cancelGesture = (): void => { + clearHoverOutline(); + dragStart = null; + marqueeBox.style.display = "none"; + activeStroke?.path.remove(); + activeStroke = null; + }; + const onPointerOut = (event: PointerEvent): void => { if (event.relatedTarget === null) clearHoverOutline(); }; @@ -1209,8 +1263,7 @@ function startAnnotation(): void { const restoreStyles = (): void => { for (const target of selected.values()) { - if (!(target.element instanceof HTMLElement || target.element instanceof SVGElement)) - continue; + if (!hasInlineStyle(target.element)) continue; for (const [property, baseline] of target.baselineStyles) { if (baseline) target.element.style.setProperty(property, baseline); else target.element.style.removeProperty(property); @@ -1222,15 +1275,7 @@ function startAnnotation(): void { if (finished) return; finished = true; restoreStyles(); - window.removeEventListener("pointermove", onPointerMove, true); - window.removeEventListener("pointerdown", onPointerDown, true); - window.removeEventListener("pointerup", onPointerUp, true); - window.removeEventListener("pointerout", onPointerOut, true); - window.removeEventListener("click", onClick, true); - window.removeEventListener("blur", onWindowBlur); - window.removeEventListener("keydown", onKeyDown, true); - window.removeEventListener("scroll", repaint, true); - window.removeEventListener("resize", repaint); + frameDocuments.dispose(); dragHandle.removeEventListener("pointerdown", onEditorPointerDown); dragHandle.removeEventListener("pointermove", onEditorPointerMove); dragHandle.removeEventListener("pointerup", onEditorPointerUp); @@ -1266,6 +1311,7 @@ function startAnnotation(): void { const submitAnnotation = (submission: PreviewAnnotationSubmission): void => { if (pendingCapture || (selected.size === 0 && regions.length === 0 && strokes.length === 0)) return; + repaint(); pendingCapture = true; submit.disabled = true; submit.textContent = "Capturing…"; @@ -1279,22 +1325,31 @@ function startAnnotation(): void { void Promise.all( Array.from(selected.values()).map(async (target) => { const element = await captureElement(target.element); - for (const change of submittedStyleChanges) { - if (change.targetId === target.id && element.selector !== null) { - change.selector = element.selector; - } - } - return { - id: target.id, - element, - rect: rectFromDomRect(target.element.getBoundingClientRect()), - }; + return { target, element }; }), ) - .then((elements) => { + .then((capturedElements) => { + const elements = capturedElements.flatMap(({ target, element }) => { + const rect = topViewportRect(target.element); + if (!rect || !isUsableRect(rect)) return []; + for (const change of submittedStyleChanges) { + if (change.targetId === target.id && element.selector !== null) { + change.selector = element.selector; + } + } + return [{ id: target.id, element, rect: rectFromDomRect(rect) }]; + }); // The overlay may have been cancelled or replaced while the capture // ran. A late submit must not deliver into the next pick's listener. if (finished) return; + if ( + elements.length === 0 && + submittedRegions.length === 0 && + submittedStrokes.length === 0 + ) { + teardown(true); + return; + } const annotation: PreviewAnnotationPayload = { id: nextId("annotation"), pageUrl: location.href, @@ -1303,7 +1358,9 @@ function startAnnotation(): void { elements, regions: submittedRegions, strokes: submittedStrokes, - styleChanges: submittedStyleChanges, + styleChanges: submittedStyleChanges.filter((change) => + elements.some((target) => target.id === change.targetId), + ), screenshot: null, createdAt: new Date().toISOString(), }; @@ -1335,15 +1392,42 @@ function startAnnotation(): void { submitAnnotation(submission); }); - window.addEventListener("pointermove", onPointerMove, { capture: true, passive: false }); - window.addEventListener("pointerdown", onPointerDown, { capture: true, passive: false }); - window.addEventListener("pointerup", onPointerUp, { capture: true, passive: false }); - window.addEventListener("pointerout", onPointerOut, { capture: true, passive: true }); - window.addEventListener("click", onClick, { capture: true, passive: false }); - window.addEventListener("blur", onWindowBlur); - window.addEventListener("keydown", onKeyDown, { capture: true }); - window.addEventListener("scroll", repaint, { capture: true, passive: true }); - window.addEventListener("resize", repaint, { passive: true }); + const frameDocuments = observeFrameDocuments( + document, + (owner) => { + const view = owner.defaultView; + if (!view) return () => {}; + const childCursorStyle = owner === document ? null : cursorStyle.cloneNode(true); + if (childCursorStyle) owner.documentElement?.appendChild(childCursorStyle); + owner.documentElement?.setAttribute("data-t3code-annotation-tool", tool); + const controller = new AbortController(); + const capture = { capture: true, passive: false, signal: controller.signal }; + const passive = { capture: true, passive: true, signal: controller.signal }; + if (owner !== document) { + view.addEventListener("pointerdown", reportHumanPointerInput, capture); + view.addEventListener("keydown", reportHumanKeyInput, capture); + } + view.addEventListener("pointermove", onPointerMove, capture); + view.addEventListener("pointerdown", onPointerDown, capture); + view.addEventListener("pointerup", onPointerUp, capture); + view.addEventListener("pointercancel", cancelGesture, capture); + view.addEventListener("pointerout", onPointerOut, passive); + view.addEventListener("click", onClick, capture); + view.addEventListener("blur", onWindowBlur, { signal: controller.signal }); + view.addEventListener("pagehide", cancelGesture, { signal: controller.signal }); + view.addEventListener("keydown", onKeyDown, capture); + view.addEventListener("scroll", repaint, passive); + view.addEventListener("resize", repaint, passive); + return () => { + controller.abort(); + cancelGesture(); + owner.documentElement?.removeAttribute("data-t3code-annotation-tool"); + childCursorStyle?.parentNode?.removeChild(childCursorStyle); + }; + }, + repaint, + isAnnotationNode, + ); ipcRenderer.on(CANCEL_PICK_CHANNEL, onCancel); ipcRenderer.on(ANNOTATION_CAPTURED_CHANNEL, onCaptured); document.documentElement.appendChild(host); diff --git a/apps/desktop/src/preview/PickedElementPayload.test.ts b/apps/desktop/src/preview/PickedElementPayload.test.ts index d7a967324771..eb1cfee2a64c 100644 --- a/apps/desktop/src/preview/PickedElementPayload.test.ts +++ b/apps/desktop/src/preview/PickedElementPayload.test.ts @@ -2,6 +2,23 @@ import { describe, expect, it } from "vite-plus/test"; import { isPickedElementPayload, isPreviewAnnotationPayload } from "./PickedElementPayload.ts"; +it("validates the document path carried by iframe selections", () => { + expect( + isPickedElementPayload( + validPayload({ framePath: [{ pageUrl: "https://example.com/", selector: "iframe" }] }), + ), + ).toBe(true); + for (const framePath of [ + null, + "iframe", + [null], + [{ selector: "iframe" }], + [{ pageUrl: 12, selector: "iframe" }], + ]) { + expect(isPickedElementPayload(validPayload({ framePath }))).toBe(false); + } +}); + function validPayload(overrides?: Record): Record { return { pageUrl: "https://example.com/", diff --git a/apps/desktop/src/preview/PickedElementPayload.ts b/apps/desktop/src/preview/PickedElementPayload.ts index e2d596120dba..a35082b7feb0 100644 --- a/apps/desktop/src/preview/PickedElementPayload.ts +++ b/apps/desktop/src/preview/PickedElementPayload.ts @@ -41,6 +41,20 @@ export function isPickedElementPayload(value: unknown): value is PickedElementPa if (typeof c["pickedAt"] !== "string") return false; if (!isStringOrNull(c["pageTitle"])) return false; if (!isStringOrNull(c["selector"])) return false; + if ( + c["framePath"] !== undefined && + (!Array.isArray(c["framePath"]) || + !c["framePath"].every( + (frame: unknown) => + typeof frame === "object" && + frame !== null && + "pageUrl" in frame && + typeof frame.pageUrl === "string" && + "selector" in frame && + typeof frame.selector === "string", + )) + ) + return false; if (!isStringOrNull(c["componentName"])) return false; if (c["source"] !== null && !isPickedStackFrame(c["source"])) return false; if (!Array.isArray(c["stack"])) return false; diff --git a/apps/web/src/lib/elementContext.test.ts b/apps/web/src/lib/elementContext.test.ts index a8c740741eb2..10142bd8bc94 100644 --- a/apps/web/src/lib/elementContext.test.ts +++ b/apps/web/src/lib/elementContext.test.ts @@ -1,4 +1,5 @@ -import type { PickedElementPayload } from "@t3tools/contracts"; +import { type PickedElementPayload, PickedElementPayloadSchema } from "@t3tools/contracts"; +import { Schema } from "effect"; import { describe, expect, it } from "vite-plus/test"; import { @@ -13,6 +14,30 @@ import { normalizeElementContextSelection, } from "./elementContext"; +it("preserves nested frame context in packaged prompts and distinguishes sibling srcdoc picks", () => { + const raw = makePayload({ + pageUrl: "about:srcdoc", + framePath: [ + { pageUrl: "https://example.com/editor", selector: "iframe:nth-child(1)" }, + { pageUrl: "about:srcdoc", selector: "iframe.preview" }, + ], + }); + const context = normalizeElementContextSelection( + Schema.decodeUnknownSync(PickedElementPayloadSchema)(raw), + )!; + expect(context.framePath).toEqual(raw.framePath); + const block = buildElementContextBlock([context]); + expect(block).toContain("url: about:srcdoc"); + expect(block).toContain("frame path (outer to inner"); + expect(block).toContain(JSON.stringify(raw.framePath![0])); + expect(block).toContain(JSON.stringify(raw.framePath![1])); + const sibling = normalizeElementContextSelection({ + ...raw, + framePath: [{ pageUrl: "https://example.com/editor", selector: "iframe:nth-child(2)" }], + })!; + expect(elementContextDedupKey(sibling)).not.toBe(elementContextDedupKey(context)); +}); + function makePayload(overrides?: Partial): PickedElementPayload { return { pageUrl: "https://example.com/dashboard", diff --git a/apps/web/src/lib/elementContext.ts b/apps/web/src/lib/elementContext.ts index 8064db710794..3a5f1c59102f 100644 --- a/apps/web/src/lib/elementContext.ts +++ b/apps/web/src/lib/elementContext.ts @@ -23,6 +23,8 @@ export interface ElementContextSelection { tagName: string; /** CSS selector — may be null when react-grab can't compute one. */ selector: string | null; + /** Outer-to-inner iframe selectors for elements in embedded documents. */ + framePath?: PickedElementPayload["framePath"]; /** Truncated outer-HTML preview. */ htmlPreview: string; /** Nearest React component display name, or null. */ @@ -80,6 +82,14 @@ export function normalizeElementContextSelection( pageTitle: raw.pageTitle?.trim() ?? null, tagName, selector: raw.selector?.trim() || null, + ...(raw.framePath?.length + ? { + framePath: raw.framePath.slice(0, 32).map((frame) => ({ + pageUrl: truncateString(frame.pageUrl.trim(), 2000), + selector: truncateString(frame.selector.trim(), 2000), + })), + } + : {}), htmlPreview: truncateString(normalizeText(raw.htmlPreview), ELEMENT_CONTEXT_HTML_PREVIEW_LIMIT), componentName: raw.componentName?.trim() || null, source: stackFrame @@ -99,7 +109,13 @@ export function normalizeElementContextSelection( * the same key, so we don't end up with a runaway chip row from spam-clicks. */ export function elementContextDedupKey(context: ElementContextSelection): string { - return [context.pageUrl, context.selector ?? "", context.tagName, context.componentName ?? ""] + return [ + context.pageUrl, + context.selector ?? "", + context.tagName, + context.componentName ?? "", + ...(context.framePath?.map((frame) => JSON.stringify(frame)) ?? []), + ] .join("|") .toLowerCase(); } @@ -150,6 +166,14 @@ function buildSingleContextLines(context: ElementContextSelection): string[] { if (context.selector) { lines.push(` selector: ${context.selector}`); } + if (context.framePath?.length) { + lines.push( + " frame path (outer to inner; enter each iframe's document before resolving the next selector):", + ); + for (const frame of context.framePath) { + lines.push(` ${JSON.stringify(frame)}`); + } + } if (context.source?.fileName) { const { fileName, lineNumber, columnNumber } = context.source; const location = diff --git a/docs/user/browser-annotations.md b/docs/user/browser-annotations.md new file mode 100644 index 000000000000..77fb484240c2 --- /dev/null +++ b/docs/user/browser-annotations.md @@ -0,0 +1,21 @@ +# Browser annotations + +Use the browser's annotation picker to select page elements, mark a region, or +draw feedback. Add a comment and attach the annotation to your conversation, or +send it using the annotation editor's send shortcut. The annotation includes the +selected content and a screenshot of the marked area. + +Elements inside same-origin iframes are selectable, including embedded `srcdoc` +previews and nested frames. Highlights follow scrolling within the frame and the +surrounding page. Screenshot crops use the element's visible position in the +browser. If a frame navigates or is removed, its old selections are discarded; +select the replacement content to annotate it. +The packaged element context includes the embedded document's URL and the path +through its containing frames, so selectors can be resolved inside the correct +preview, including nested `srcdoc` frames. + +Style adjustments inside frames remain temporary and are restored when the +annotation ends. Escape cancels the picker from either the page or a same-origin +frame. Cross-origin frame contents are not supported. Frames with rotation, skew, +mirroring, or perspective cannot be inspected internally; ordinary positioning +and positive scaling are supported. diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 5886dae78a17..a70f985c15d8 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -891,6 +891,8 @@ export interface PickedElementPayload { pageTitle: string | null; /** Lowercase tag name, e.g. `"button"`. */ tagName: string; + /** Outer-to-inner iframe selectors, each scoped to its parent document URL. */ + framePath?: ReadonlyArray<{ pageUrl: string; selector: string }>; /** CSS selector resolving back to the element on a re-render. */ selector: string | null; /** Truncated outer-HTML preview (matches react-grab's `htmlPreview`). */ @@ -911,6 +913,9 @@ export const PickedElementPayloadSchema: Schema.Codec = Sc pageUrl: Schema.String, pageTitle: Schema.NullOr(Schema.String), tagName: Schema.String, + framePath: Schema.optionalKey( + Schema.Array(Schema.Struct({ pageUrl: Schema.String, selector: Schema.String })), + ), selector: Schema.NullOr(Schema.String), htmlPreview: Schema.String, componentName: Schema.NullOr(Schema.String),