From f4e938fce4e05c5725cfa5b83fce1db16552db00 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 19:41:38 -0400 Subject: [PATCH 1/3] feat(studio): let an agent drive Studio's selection and playhead Adds `studio_select` and `studio_seek`, so an agent and the human are looking at the same element and the same instant. Selecting reveals the inspector, exactly as a click does, which is what makes the agent's move visible. Selection is shared state, not a per-call argument, and that is forced rather than chosen. Most of Studio's edit handlers read the ambient React selection, and `applyDomSelection` only schedules a state update, so selecting and committing inside ONE call would write to whatever was selected before. Two tool calls are separated by a render, so the contract is select first, then act. That is also how a human works: click, then type. `studio_seek` uses `requestSeek`, not `setCurrentTime`. The latter only moves the timeline's displayed number and leaves the composition where it was. Two things the tools refuse to fake: Seek does not clamp. `seek()` already clamps against the adapter's duration, which can differ from the store's, and clamping again would give that invariant two owners that can disagree. The tool reports where the playhead actually landed instead, read back afterwards. `requestSeek` is fire-and-forget, so it cannot report that no adapter was mounted to receive it. The tool compares the playhead before and after and fails rather than claiming a seek that never happened. Select separates three failures that a single message would have merged: the preview is not mounted yet (wait), no element matches the handle (re-read), and the element cannot be selected (try a neighbour). The agent's next move differs for each, so collapsing them would cost it a round trip or a retry loop. --- .../studio/src/webmcp/StudioAgentTools.tsx | 36 ++- packages/studio/src/webmcp/toolResult.ts | 2 +- .../src/webmcp/tools/selectionTools.test.ts | 214 ++++++++++++++++++ .../studio/src/webmcp/tools/selectionTools.ts | 154 +++++++++++++ .../src/webmcp/useStudioAgentTools.test.tsx | 63 ++++-- .../studio/src/webmcp/useStudioAgentTools.ts | 50 +++- 6 files changed, 487 insertions(+), 32 deletions(-) create mode 100644 packages/studio/src/webmcp/tools/selectionTools.test.ts create mode 100644 packages/studio/src/webmcp/tools/selectionTools.ts diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx index 8060d7382b9..0130ce1a9a1 100644 --- a/packages/studio/src/webmcp/StudioAgentTools.tsx +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -1,8 +1,8 @@ -import { useCallback } from "react"; -import { useDomEditSelectionContext } from "../contexts/DomEditContext"; +import { useCallback, useMemo } from "react"; +import { useDomEditActionsContext, useDomEditSelectionContext } from "../contexts/DomEditContext"; import { useStudioShellContext } from "../contexts/StudioContext"; import { usePlayerStore } from "../player"; -import { useStudioAgentTools } from "./useStudioAgentTools"; +import { useStudioAgentTools, type StudioAgentToolsDeps } from "./useStudioAgentTools"; import type { StudioLookSnapshot } from "./tools/lookTools"; /** @@ -12,14 +12,15 @@ import type { StudioLookSnapshot } from "./tools/lookTools"; * contexts are only readable below `DomEditProvider`, which `App` renders, and * `App.tsx` sits three lines under the 600-line cap. * - * The player store is read IMPERATIVELY through `getState()` inside the - * snapshot callback rather than subscribed to. Subscribing to `currentTime` - * would re-render this component on every animation frame during playback for - * a value nothing here displays. + * The player store is read IMPERATIVELY through `getState()` rather than + * subscribed to. Subscribing to `currentTime` would re-render this component on + * every animation frame during playback for a value nothing here displays. */ export function StudioAgentTools() { const { projectId, activeCompPath, editHistory } = useStudioShellContext(); const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext(); + const { previewIframeRef, buildDomSelectionFromTarget, applyDomSelection } = + useDomEditActionsContext(); const getSnapshot = useCallback((): StudioLookSnapshot => { const player = usePlayerStore.getState(); @@ -41,6 +42,25 @@ export function StudioAgentTools() { }; }, [projectId, activeCompPath, domEditSelection, selectedGsapAnimations, editHistory]); - useStudioAgentTools({ getSnapshot }); + const deps = useMemo( + () => ({ + getSnapshot, + getPreviewDocument: () => previewIframeRef.current?.contentDocument ?? null, + buildSelection: (element) => buildDomSelectionFromTarget(element), + applySelection: (selection) => applyDomSelection(selection, { revealPanel: true }), + requestSeek: (time) => usePlayerStore.getState().requestSeek(time), + readPlayhead: () => { + const player = usePlayerStore.getState(); + return { + currentTime: player.currentTime, + duration: player.duration, + isPlaying: player.isPlaying, + }; + }, + }), + [getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection], + ); + + useStudioAgentTools(deps); return null; } diff --git a/packages/studio/src/webmcp/toolResult.ts b/packages/studio/src/webmcp/toolResult.ts index fd803ec10ca..fcfdc7684a9 100644 --- a/packages/studio/src/webmcp/toolResult.ts +++ b/packages/studio/src/webmcp/toolResult.ts @@ -36,7 +36,7 @@ export function toolOk(value: T): { ok: true } & T { return { ok: true, ...value }; } -function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure { +export function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure { return hint ? { ok: false, kind, reason, hint } : { ok: false, kind, reason }; } diff --git a/packages/studio/src/webmcp/tools/selectionTools.test.ts b/packages/studio/src/webmcp/tools/selectionTools.test.ts new file mode 100644 index 00000000000..c6ec5fdd2a7 --- /dev/null +++ b/packages/studio/src/webmcp/tools/selectionTools.test.ts @@ -0,0 +1,214 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; +import type { DomEditSelection } from "../../components/editor/domEditingTypes"; +import { + studioSeek, + studioSelect, + type SelectionToolDeps, + type StudioSeekResult, + type StudioSelectResult, +} from "./selectionTools"; +import type { ToolFailure, ToolResult } from "../toolResult"; + +function previewDoc(html: string): Document { + const iframe = document.createElement("iframe"); + document.body.append(iframe); + const doc = iframe.contentDocument; + if (!doc) throw new Error("expected iframe document"); + doc.body.innerHTML = html; + return doc; +} + +function selectionFor(element: HTMLElement): DomEditSelection { + return { + id: element.id || undefined, + hfId: element.getAttribute("data-hf-id") ?? undefined, + element, + label: "Headline", + tagName: element.tagName.toLowerCase(), + sourceFile: "index.html", + compositionPath: "index.html", + isCompositionHost: false, + isInsideLockedComposition: false, + boundingBox: { x: 40, y: 12, width: 880, height: 96 }, + textContent: element.textContent, + dataAttributes: {}, + inlineStyles: {}, + computedStyles: {}, + textFields: [], + capabilities: { + canSelect: true, + canEditStyles: true, + canCrop: true, + canMove: true, + canResize: true, + canApplyManualOffset: true, + canApplyManualSize: true, + canApplyManualRotation: true, + }, + }; +} + +function selectionDeps(overrides: Partial = {}): SelectionToolDeps { + return { + getPreviewDocument: () => null, + buildSelection: async (element) => selectionFor(element), + applySelection: () => undefined, + requestSeek: () => undefined, + readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }), + ...overrides, + }; +} + +function expectFailure(result: ToolResult): ToolFailure { + if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`); + return result; +} + +function expectOk(result: ToolResult): { ok: true } & T { + if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`); + return result; +} + +describe("studioSelect", () => { + it("applies the selection a click would produce and reports it back", async () => { + const doc = previewDoc('

Ship it

'); + const applySelection = vi.fn(); + + const result = await studioSelect( + selectionDeps({ getPreviewDocument: () => doc, applySelection }), + "hf:abc", + ); + + const ok = expectOk(result); + expect(ok.handle).toBe("hf:abc"); + expect(ok.label).toBe("Headline"); + expect(ok.box.width).toBe(880); + // Reveals the inspector, which is what makes the human see what the agent did. + expect(applySelection).toHaveBeenCalledTimes(1); + }); + + it("distinguishes a preview that is not mounted from a handle that does not match", async () => { + const notMounted = expectFailure(await studioSelect(selectionDeps(), "dom:headline")); + expect(notMounted.kind).toBe("blocked"); + expect(notMounted.reason).toMatch(/not mounted/); + + const doc = previewDoc('

Ship it

'); + const noMatch = expectFailure( + await studioSelect(selectionDeps({ getPreviewDocument: () => doc }), "dom:missing"), + ); + expect(noMatch.kind).toBe("invalid"); + expect(noMatch.reason).toMatch(/no element matches/); + // The two must not be the same message: waiting and re-reading are different fixes. + expect(noMatch.reason).not.toBe(notMounted.reason); + }); + + it("reports an element Studio cannot build a selection for, as a third case", async () => { + const doc = previewDoc('

Ship it

'); + + const result = expectFailure( + await studioSelect( + selectionDeps({ getPreviewDocument: () => doc, buildSelection: async () => null }), + "dom:headline", + ), + ); + + expect(result.kind).toBe("blocked"); + expect(result.reason).toMatch(/cannot select/); + }); + + it("rejects a missing handle without touching the preview", async () => { + const getPreviewDocument = vi.fn(() => null); + + const result = expectFailure(await studioSelect(selectionDeps({ getPreviewDocument }), " ")); + + expect(result.kind).toBe("invalid"); + expect(getPreviewDocument).not.toHaveBeenCalled(); + }); + + it("leaves the existing selection alone when it fails", async () => { + const doc = previewDoc('

Ship it

'); + const applySelection = vi.fn(); + + await studioSelect( + selectionDeps({ getPreviewDocument: () => doc, applySelection }), + "dom:missing", + ); + + expect(applySelection).not.toHaveBeenCalled(); + }); +}); + +describe("studioSeek", () => { + it("reports where the playhead landed, not what was requested", () => { + // The player clamps against the ADAPTER's duration, which the wrapper + // deliberately does not second-guess. + let currentTime = 0; + const result = studioSeek( + selectionDeps({ + requestSeek: () => { + currentTime = 10; + }, + readPlayhead: () => ({ currentTime, duration: 10, isPlaying: false }), + }), + 999, + ); + + const ok = expectOk(result); + expect(ok.playhead).toBe(10); + expect(ok.moved).toBe(true); + }); + + it("reports that playback stopped", () => { + let isPlaying = true; + let currentTime = 0; + const result = studioSeek( + selectionDeps({ + requestSeek: () => { + currentTime = 2; + isPlaying = false; + }, + readPlayhead: () => ({ currentTime, duration: 10, isPlaying }), + }), + 2, + ); + + expect(expectOk(result).isPlaying).toBe(false); + }); + + it("fails rather than claiming a seek the player never received", () => { + // `requestSeek` is fire-and-forget: with no adapter mounted it silently does + // nothing, and reporting ok would be a lie the agent builds on. + const result = expectFailure( + studioSeek( + selectionDeps({ readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }) }), + 5, + ), + ); + + expect(result.kind).toBe("blocked"); + expect(result.reason).toMatch(/did not move/); + }); + + it("succeeds when asked to seek to where the playhead already is", () => { + const result = studioSeek( + selectionDeps({ readPlayhead: () => ({ currentTime: 3, duration: 10, isPlaying: false }) }), + 3, + ); + + // Nothing moved, but nothing failed either, and `moved` says which. + const ok = expectOk(result); + expect(ok.moved).toBe(false); + expect(ok.playhead).toBe(3); + }); + + it("rejects a non-finite time without calling the player", () => { + const requestSeek = vi.fn(); + + for (const time of [Number.NaN, Number.POSITIVE_INFINITY]) { + const result = expectFailure(studioSeek(selectionDeps({ requestSeek }), time)); + expect(result.kind).toBe("invalid"); + } + expect(requestSeek).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/studio/src/webmcp/tools/selectionTools.ts b/packages/studio/src/webmcp/tools/selectionTools.ts new file mode 100644 index 00000000000..fa40f52d8f8 --- /dev/null +++ b/packages/studio/src/webmcp/tools/selectionTools.ts @@ -0,0 +1,154 @@ +/** + * `studio_select` and `studio_seek`: pointing the human and the agent at the + * same thing. + * + * Selection is shared state, not a per-call argument. That is deliberate and it + * is also forced: most of Studio's edit handlers read the ambient React + * selection, and `applyDomSelection` only schedules a state update, so + * selecting and committing inside ONE call would write to whatever was selected + * before. Two tool calls are separated by a render, so the contract is select + * first, then act, which is also how a human works: click, then type. + */ + +import type { DomEditSelection } from "../../components/editor/domEditingTypes"; +import { mintElementHandle, patchTargetAddress, resolveElementHandle } from "../handles"; +import { toolFailure, toolOk, type ToolResult } from "../toolResult"; + +export interface SelectionToolDeps { + /** The preview iframe's document, or null before it mounts. */ + getPreviewDocument: () => Document | null; + buildSelection: (element: HTMLElement) => Promise; + applySelection: (selection: DomEditSelection) => void; + /** Out-of-loop seek. `requestSeek`, not `setCurrentTime`. */ + requestSeek: (time: number) => void; + readPlayhead: () => { currentTime: number; duration: number; isPlaying: boolean }; +} + +export interface StudioSelectResult { + handle: string | null; + label: string; + tagName: string; + box: { x: number; y: number; width: number; height: number }; +} + +export async function studioSelect( + deps: SelectionToolDeps, + handle: string, +): Promise> { + if (typeof handle !== "string" || !handle.trim()) { + return toolFailure("invalid", "handle must be a non-empty string", "Call studio_look first."); + } + + // Three distinct failures, deliberately not collapsed: "the preview is not up + // yet" is a wait, "no such element" is a stale handle, and "could not build a + // selection" is an element Studio cannot drive. The agent's next move differs + // for each. + const doc = deps.getPreviewDocument(); + if (!doc) { + return toolFailure( + "blocked", + "the preview is not mounted yet", + "Wait for the composition to load, then retry.", + ); + } + + const element = resolveElementHandle(doc, handle); + if (!element) { + return toolFailure( + "invalid", + `no element matches handle ${handle}`, + "The composition may have changed. Call studio_look for current handles.", + ); + } + + const selection = await deps.buildSelection(element); + if (!selection) { + return toolFailure( + "blocked", + `${handle} resolved to an element Studio cannot select`, + "Try a parent or child element from studio_look.", + ); + } + + deps.applySelection(selection); + return toolOk({ + handle: mintElementHandle(patchTargetAddress(selection)), + label: selection.label, + tagName: selection.tagName, + box: selection.boundingBox, + }); +} + +export interface StudioSeekResult { + /** Where the playhead ACTUALLY landed, which may differ from the request. */ + playhead: number; + duration: number; + isPlaying: boolean; + moved: boolean; +} + +export function studioSeek(deps: SelectionToolDeps, time: number): ToolResult { + if (typeof time !== "number" || !Number.isFinite(time)) { + return toolFailure("invalid", "time must be a finite number of seconds"); + } + + const before = deps.readPlayhead(); + // Deliberately NOT clamped here. `seek()` already clamps against the + // adapter's duration, which can differ from the store's, and a second clamp + // would give that invariant two owners that can disagree. Report where it + // landed instead. + deps.requestSeek(time); + const after = deps.readPlayhead(); + + // `requestSeek` is fire-and-forget: it cannot report that no adapter was + // mounted to receive it. Reading back is the only way to avoid claiming a + // seek that never happened. + const moved = after.currentTime !== before.currentTime; + if (!moved && before.currentTime !== time) { + return toolFailure( + "blocked", + `the playhead did not move; it is still at ${after.currentTime}`, + "The preview may not be ready. Check studio_look, then retry.", + ); + } + + return toolOk({ + playhead: after.currentTime, + duration: after.duration, + isPlaying: after.isPlaying, + moved, + }); +} + +export const STUDIO_SELECT_INPUT_SCHEMA = { + type: "object", + properties: { + handle: { type: "string", description: "An element handle from studio_look." }, + }, + required: ["handle"], + additionalProperties: false, +} as const; + +export const STUDIO_SELECT_DESCRIPTION = [ + "Select an element in HyperFrames Studio, exactly as clicking it would:", + "the human sees the same selection box and inspector.", + "Takes a handle from studio_look. Most editing tools act on the CURRENT selection,", + "so call this first, then the edit.", + "Returns `ok: true` with the resulting selection, or `ok: false` with `kind`, `reason` and a `hint`.", +].join(" "); + +export const STUDIO_SEEK_INPUT_SCHEMA = { + type: "object", + properties: { + time: { type: "number", minimum: 0, description: "Playhead position in seconds." }, + }, + required: ["time"], + additionalProperties: false, +} as const; + +export const STUDIO_SEEK_DESCRIPTION = [ + "Move the playhead to a time in seconds. Pauses playback.", + "Out-of-range times are clamped by the player, so check the returned `playhead`", + "for where it actually landed rather than assuming it matched your request.", + "Returns `ok: true`, or `ok: false` with `kind`, `reason` and a `hint`.", +].join(" "); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx index c809516b8aa..d27665c4a75 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -29,6 +29,19 @@ function snapshot(overrides: Partial = {}): StudioLookSnapsh }; } +/** Full deps with inert defaults; override only what the test is about. */ +function deps(overrides: Partial = {}): StudioAgentToolsDeps { + return { + getSnapshot: () => snapshot(), + getPreviewDocument: () => null, + buildSelection: async () => null, + applySelection: () => undefined, + requestSeek: () => undefined, + readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }), + ...overrides, + }; +} + /** Install a fake `document.modelContext` and report what got registered. */ function installModelContext() { const registered: ModelContextTool[] = []; @@ -50,12 +63,12 @@ function removeModelContext() { Reflect.deleteProperty(document, "modelContext"); } -function mountTools(deps: StudioAgentToolsDeps) { +function mountTools(initial: StudioAgentToolsDeps) { function Probe({ current }: { current: StudioAgentToolsDeps }) { useStudioAgentTools(current); return null; } - const root = mountReactHarness(); + const root = mountReactHarness(); cleanup = () => act(() => root.unmount()); return { rerenderWith(next: StudioAgentToolsDeps) { @@ -82,10 +95,14 @@ describe("useStudioAgentTools", () => { const { registered } = installModelContext(); await act(async () => { - mountTools({ getSnapshot: () => snapshot() }); + mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registered.map((tool) => tool.name)).toEqual(["studio_look"]); + expect(registered.map((tool) => tool.name)).toEqual([ + "studio_look", + "studio_select", + "studio_seek", + ]); expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present"); }); @@ -97,16 +114,16 @@ describe("useStudioAgentTools", () => { let harness: ReturnType | null = null; await act(async () => { - harness = mountTools({ getSnapshot: () => snapshot() }); + harness = mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(1); + expect(registerTool).toHaveBeenCalledTimes(3); await act(async () => { - harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 5 }) }); - harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 6 }) }); + harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) })); + harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) })); }); - expect(registerTool).toHaveBeenCalledTimes(1); + expect(registerTool).toHaveBeenCalledTimes(3); }); it("executes against the LATEST deps, not the ones present at registration", async () => { @@ -116,11 +133,11 @@ describe("useStudioAgentTools", () => { let harness: ReturnType | null = null; await act(async () => { - harness = mountTools({ getSnapshot: () => snapshot({ currentTime: 1 }) }); + harness = mountTools(deps({ getSnapshot: () => snapshot({ currentTime: 1 }) })); }); await act(async () => { - harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 42 }) }); + harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 42 }) })); }); const look = registered[0]; @@ -138,7 +155,7 @@ describe("useStudioAgentTools", () => { const { registerTool } = installModelContext(); await act(async () => { - mountTools({ getSnapshot: () => snapshot() }); + mountTools(deps({ getSnapshot: () => snapshot() })); }); const signal = registerTool.mock.calls[0]?.[1]?.signal; expect(signal?.aborted).toBe(false); @@ -153,7 +170,7 @@ describe("useStudioAgentTools", () => { removeModelContext(); await act(async () => { - mountTools({ getSnapshot: () => snapshot() }); + mountTools(deps({ getSnapshot: () => snapshot() })); }); // The assertion is that mounting did not throw; a browser without the API @@ -166,7 +183,7 @@ describe("useStudioAgentTools", () => { const { registerTool } = installModelContext(); await act(async () => { - mountTools({ getSnapshot: () => snapshot() }); + mountTools(deps({ getSnapshot: () => snapshot() })); }); expect(registerTool).not.toHaveBeenCalled(); @@ -176,10 +193,10 @@ describe("useStudioAgentTools", () => { const { registerTool } = installModelContext(); await act(async () => { - mountTools({ getSnapshot: () => snapshot() }); + mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(1); + expect(registerTool).toHaveBeenCalledTimes(3); }); it("reports a non-abort registration failure through production telemetry", async () => { @@ -187,7 +204,7 @@ describe("useStudioAgentTools", () => { registerTool.mockRejectedValue(new DOMException("blocked", "NotAllowedError")); await act(async () => { - mountTools({ getSnapshot: () => snapshot() }); + mountTools(deps({ getSnapshot: () => snapshot() })); }); expect(trackEvent).toHaveBeenCalledWith("webmcp_registration_failed", { @@ -200,11 +217,13 @@ describe("useStudioAgentTools", () => { const { registered } = installModelContext(); await act(async () => { - mountTools({ - getSnapshot: () => { - throw new TypeError("handler signature moved"); - }, - }); + mountTools( + deps({ + getSnapshot: () => { + throw new TypeError("handler signature moved"); + }, + }), + ); }); vi.spyOn(console, "error").mockImplementation(() => {}); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.ts b/packages/studio/src/webmcp/useStudioAgentTools.ts index 6f4fa42e84b..2c67d233289 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.ts +++ b/packages/studio/src/webmcp/useStudioAgentTools.ts @@ -14,6 +14,17 @@ import { type StudioLookInput, type StudioLookSnapshot, } from "./tools/lookTools"; +import { + studioSeek, + studioSelect, + STUDIO_SEEK_DESCRIPTION, + STUDIO_SEEK_INPUT_SCHEMA, + STUDIO_SELECT_DESCRIPTION, + STUDIO_SELECT_INPUT_SCHEMA, + type SelectionToolDeps, + type StudioSeekResult, + type StudioSelectResult, +} from "./tools/selectionTools"; const log = makeStudioDebugLogger("webmcp"); @@ -27,7 +38,7 @@ function reportRegistration(report: ToolRegistrationReport, native: boolean): vo } } -export interface StudioAgentToolsDeps { +export interface StudioAgentToolsDeps extends SelectionToolDeps { /** Read Studio's current state. Called per tool invocation, never cached. */ getSnapshot: () => StudioLookSnapshot; } @@ -57,9 +68,46 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): buildStudioLook(depsRef.current.getSnapshot(), input as StudioLookInput), ), }, + { + name: "studio_select", + title: "Select an element", + description: STUDIO_SELECT_DESCRIPTION, + inputSchema: STUDIO_SELECT_INPUT_SCHEMA, + annotations: { readOnlyHint: false, untrustedContentHint: true }, + execute: (input): Promise> => + runToolBody("studio_select", () => + studioSelect(depsRef.current, readStringInput(input, "handle")), + ), + }, + { + name: "studio_seek", + title: "Move the playhead", + description: STUDIO_SEEK_DESCRIPTION, + inputSchema: STUDIO_SEEK_INPUT_SCHEMA, + annotations: { readOnlyHint: false }, + execute: (input): Promise> => + runToolBody("studio_seek", async () => + studioSeek(depsRef.current, readNumberInput(input, "time")), + ), + }, ]; } +/** + * Nothing in the platform validates the input object against `inputSchema`, so + * a tool receives whatever the agent sent. These read a field without asserting + * its type; the tools themselves reject what they cannot use. + */ +function readStringInput(input: object, key: string): string { + const value = Reflect.get(input, key); + return typeof value === "string" ? value : ""; +} + +function readNumberInput(input: object, key: string): number { + const value = Reflect.get(input, key); + return typeof value === "number" ? value : Number.NaN; +} + /** * Register Studio's tools with the browser, exactly once per mount. * From 57c9bb0d602f31e1d9b7ec1688eb93e30b20e872 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Wed, 26 Aug 2026 19:52:09 -0400 Subject: [PATCH 2/3] feat(studio): give an agent eyes with studio_frame Renders the composition to a PNG at a given time and returns the URL. This is what turns the tool set from a remote control into a loop: author a change, capture the instant it affects, look, adjust. No agent can judge motion from source, because "what does this look like at 2.4 seconds" is not a question a file answers. Reuses Studio's existing capture endpoint via `buildFrameCaptureUrl` rather than inventing a second one. Two things this does not fake: It reports the time the playhead LANDED on, not the time requested. The player clamps, so those differ at the ends, and attaching the wrong time to a frame is how an agent draws a confident wrong conclusion about motion. It waits before capturing, by default 150ms. The frame is rendered from the file on disk, and the render cache is cleared by a file watcher with a 40ms write-stability threshold, so a capture that beats the watcher renders the PRE-edit composition. That exact staleness was a real bug here once. An agent reading a stale frame as "my edit failed" would thrash, so the wait is on by default, `settleMs` makes it tunable, and the tool description names the failure rather than leaving it to be rediscovered. It probes with HEAD before returning, so a URL that 404s comes back as a failure with a hint instead of as a link the agent cannot render. --- .../studio/src/webmcp/StudioAgentTools.tsx | 23 ++- .../src/webmcp/tools/frameTools.test.ts | 146 ++++++++++++++++++ .../studio/src/webmcp/tools/frameTools.ts | 133 ++++++++++++++++ .../src/webmcp/useStudioAgentTools.test.tsx | 11 +- .../studio/src/webmcp/useStudioAgentTools.ts | 19 ++- 5 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 packages/studio/src/webmcp/tools/frameTools.test.ts create mode 100644 packages/studio/src/webmcp/tools/frameTools.ts diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx index 0130ce1a9a1..e26612c1752 100644 --- a/packages/studio/src/webmcp/StudioAgentTools.tsx +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -57,8 +57,29 @@ export function StudioAgentTools() { isPlaying: player.isPlaying, }; }, + getProjectId: () => projectId, + getCompositionPath: () => activeCompPath, + // HEAD, not GET: the tool only needs to know the frame renders. Pulling + // the PNG here would download it once for nothing, since the agent + // fetches the URL itself. + probeFrame: async (url) => { + try { + const response = await fetch(url, { method: "HEAD" }); + return { ok: response.ok, status: response.status }; + } catch { + return { ok: false, status: 0 }; + } + }, + wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), }), - [getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection], + [ + getSnapshot, + previewIframeRef, + buildDomSelectionFromTarget, + applyDomSelection, + projectId, + activeCompPath, + ], ); useStudioAgentTools(deps); diff --git a/packages/studio/src/webmcp/tools/frameTools.test.ts b/packages/studio/src/webmcp/tools/frameTools.test.ts new file mode 100644 index 00000000000..e98d5c2e46c --- /dev/null +++ b/packages/studio/src/webmcp/tools/frameTools.test.ts @@ -0,0 +1,146 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; +import { studioFrame, type FrameToolDeps, type StudioFrameResult } from "./frameTools"; +import type { ToolFailure, ToolResult } from "../toolResult"; + +function frameDeps(overrides: Partial = {}): FrameToolDeps { + return { + getProjectId: () => "demo", + getCompositionPath: () => "index.html", + readPlayhead: () => ({ currentTime: 2.4, duration: 10, isPlaying: false }), + requestSeek: () => undefined, + probeFrame: async () => ({ ok: true, status: 200 }), + wait: async () => undefined, + ...overrides, + }; +} + +function expectOk(result: ToolResult): { ok: true } & T { + if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`); + return result; +} + +function expectFailure(result: ToolResult): ToolFailure { + if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`); + return result; +} + +describe("studioFrame", () => { + it("returns a URL for the composition at the playhead", async () => { + const result = await studioFrame(frameDeps()); + + const ok = expectOk(result); + expect(ok.time).toBe(2.4); + expect(ok.compositionPath).toBe("index.html"); + expect(ok.url).toContain("/thumbnail/"); + expect(ok.url).toContain("t=2.400"); + expect(ok.url).toContain("format=png"); + }); + + it("seeks first when given a time", async () => { + const requestSeek = vi.fn(); + + await studioFrame(frameDeps({ requestSeek }), { time: 5 }); + + expect(requestSeek).toHaveBeenCalledWith(5); + }); + + it("captures where the playhead LANDED, not what was asked for", async () => { + // The player clamps. Reporting the request would attach the wrong time to + // the frame, and an agent judging motion would draw the wrong conclusion. + const result = await studioFrame( + frameDeps({ readPlayhead: () => ({ currentTime: 10, duration: 10, isPlaying: false }) }), + { time: 999 }, + ); + + const ok = expectOk(result); + expect(ok.time).toBe(10); + expect(ok.url).toContain("t=10.000"); + }); + + it("waits before capturing, so a just-made edit is in the frame", async () => { + // The render cache is cleared by a file watcher with a write-stability + // threshold. Capturing faster than that renders the PRE-edit composition. + const wait = vi.fn(async () => undefined); + const order: string[] = []; + + await studioFrame( + frameDeps({ + wait: async (ms) => { + order.push(`wait:${ms}`); + await wait(); + }, + probeFrame: async () => { + order.push("probe"); + return { ok: true, status: 200 }; + }, + }), + ); + + expect(order).toEqual(["wait:150", "probe"]); + }); + + it("honours a caller-supplied settle time and reports it", async () => { + const result = await studioFrame(frameDeps(), { settleMs: 800 }); + + expect(expectOk(result).settledMs).toBe(800); + }); + + it("clamps an absurd settle time rather than hanging", async () => { + const result = await studioFrame(frameDeps(), { settleMs: 10 * 60 * 1000 }); + + expect(expectOk(result).settledMs).toBe(5000); + }); + + it("falls back to the default for a nonsense settle time", async () => { + for (const settleMs of [-1, Number.NaN]) { + const result = await studioFrame(frameDeps(), { settleMs }); + expect(expectOk(result).settledMs).toBe(150); + } + }); + + it("skips the wait entirely when asked for zero", async () => { + const wait = vi.fn(async () => undefined); + + await studioFrame(frameDeps({ wait }), { settleMs: 0 }); + + expect(wait).not.toHaveBeenCalled(); + }); + + it("reports a renderer failure instead of handing back a dead URL", async () => { + const result = expectFailure( + await studioFrame(frameDeps({ probeFrame: async () => ({ ok: false, status: 500 }) })), + ); + + expect(result.kind).toBe("failed"); + expect(result.reason).toContain("500"); + expect(result.hint).toBeDefined(); + }); + + it("fails when no project is open, before touching the renderer", async () => { + const probeFrame = vi.fn(); + + const result = expectFailure( + await studioFrame(frameDeps({ getProjectId: () => null, probeFrame })), + ); + + expect(result.kind).toBe("blocked"); + expect(probeFrame).not.toHaveBeenCalled(); + }); + + it("rejects a negative or non-finite time without seeking", async () => { + const requestSeek = vi.fn(); + + for (const time of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + const result = expectFailure(await studioFrame(frameDeps({ requestSeek }), { time })); + expect(result.kind).toBe("invalid"); + } + expect(requestSeek).not.toHaveBeenCalled(); + }); + + it("captures the master composition when no path is active", async () => { + const result = await studioFrame(frameDeps({ getCompositionPath: () => null })); + + expect(expectOk(result).compositionPath).toBe("index.html"); + }); +}); diff --git a/packages/studio/src/webmcp/tools/frameTools.ts b/packages/studio/src/webmcp/tools/frameTools.ts new file mode 100644 index 00000000000..839e4c231de --- /dev/null +++ b/packages/studio/src/webmcp/tools/frameTools.ts @@ -0,0 +1,133 @@ +/** + * `studio_frame`: the eyes. + * + * Without this the tool set is a remote control. With it an agent can author a + * change, look at the instant it affects, judge it, and adjust. That loop is the + * one thing source alone cannot support, because "what does this look like at + * 2.4 seconds" is not a question a file can answer. + * + * Reuses Studio's existing capture endpoint (`utils/frameCapture`) rather than + * inventing a second one. The server renders the composition with Puppeteer, so + * the frame reflects the file on disk, not the live preview DOM. + */ + +import { buildFrameCaptureUrl } from "../../utils/frameCapture"; +import { toolFailure, toolOk, type ToolResult } from "../toolResult"; + +export interface FrameToolDeps { + getProjectId: () => string | null; + getCompositionPath: () => string | null; + readPlayhead: () => { currentTime: number; duration: number; isPlaying: boolean }; + requestSeek: (time: number) => void; + /** Confirms the URL renders. Injected so tests need no network. */ + probeFrame: (url: string) => Promise<{ ok: boolean; status: number }>; + wait: (ms: number) => Promise; +} + +export interface StudioFrameResult { + /** Fetch this to see the frame. A PNG of the composition at `time`. */ + url: string; + time: number; + compositionPath: string; + /** How long the tool waited for a pending write to settle before capturing. */ + settledMs: number; +} + +export interface StudioFrameInput { + /** Seconds. Omit to capture wherever the playhead already is. */ + time?: number; + /** + * Milliseconds to wait before capturing, so a just-written edit is visible. + * See the staleness note in the description. + */ + settleMs?: number; +} + +/** + * Long enough to cover the project watcher's 40ms write-stability threshold + * plus filesystem latency, short enough not to be felt. This is the mitigation + * for a real, previously-fixed bug: the preview signature is invalidated by a + * file watcher, and a capture that beats the watcher renders the PRE-edit + * composition. An agent reading that as "my edit failed" would thrash. + */ +const DEFAULT_SETTLE_MS = 150; +const MAX_SETTLE_MS = 5_000; + +export async function studioFrame( + deps: FrameToolDeps, + input: StudioFrameInput = {}, +): Promise> { + const projectId = deps.getProjectId(); + if (!projectId) { + return toolFailure("blocked", "no project is open"); + } + + if (input.time !== undefined) { + if (typeof input.time !== "number" || !Number.isFinite(input.time) || input.time < 0) { + return toolFailure("invalid", "time must be a non-negative, finite number of seconds"); + } + deps.requestSeek(input.time); + } + + const settledMs = clampSettle(input.settleMs); + if (settledMs > 0) await deps.wait(settledMs); + + // Capture whatever the playhead now reads, rather than what was requested: + // the player clamps, so those can differ and the frame belongs to the former. + const { currentTime } = deps.readPlayhead(); + const compositionPath = deps.getCompositionPath(); + const url = buildFrameCaptureUrl({ projectId, compositionPath, currentTime }); + + const probe = await deps.probeFrame(url); + if (!probe.ok) { + return toolFailure( + "failed", + `the renderer returned ${probe.status} for this frame`, + "The composition may not build. Try `hyperframes check`.", + ); + } + + return toolOk({ + url, + time: currentTime, + compositionPath: compositionPath ?? "index.html", + settledMs, + }); +} + +function clampSettle(requested: number | undefined): number { + if (requested === undefined) return DEFAULT_SETTLE_MS; + if (typeof requested !== "number" || !Number.isFinite(requested) || requested < 0) { + return DEFAULT_SETTLE_MS; + } + return Math.min(requested, MAX_SETTLE_MS); +} + +export const STUDIO_FRAME_INPUT_SCHEMA = { + type: "object", + properties: { + time: { + type: "number", + minimum: 0, + description: "Seconds. Omit to capture wherever the playhead already is.", + }, + settleMs: { + type: "integer", + minimum: 0, + maximum: MAX_SETTLE_MS, + description: `Wait this long before capturing so a just-made edit is included. Default ${DEFAULT_SETTLE_MS}.`, + }, + }, + additionalProperties: false, +} as const; + +export const STUDIO_FRAME_DESCRIPTION = [ + "Render the composition to a PNG at a given time and return its URL, so you can", + "SEE the result instead of inferring it from source. Use this to judge a change:", + "edit, capture the instant it affects, look, adjust.", + "The frame is rendered from the file on disk, not the live preview.", + "A capture taken immediately after an edit can therefore predate that edit, because", + "the render cache is cleared by a file watcher. The tool waits briefly to cover that;", + "raise `settleMs` if a frame still looks stale, rather than concluding the edit failed.", + "Returns `ok: true` with `url` and the `time` actually captured, or `ok: false`.", +].join(" "); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx index d27665c4a75..d9c7dd40819 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -38,6 +38,10 @@ function deps(overrides: Partial = {}): StudioAgentToolsDe applySelection: () => undefined, requestSeek: () => undefined, readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }), + getProjectId: () => "demo", + getCompositionPath: () => "index.html", + probeFrame: async () => ({ ok: true, status: 200 }), + wait: async () => undefined, ...overrides, }; } @@ -102,6 +106,7 @@ describe("useStudioAgentTools", () => { "studio_look", "studio_select", "studio_seek", + "studio_frame", ]); expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present"); }); @@ -116,14 +121,14 @@ describe("useStudioAgentTools", () => { await act(async () => { harness = mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(4); await act(async () => { harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) })); harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(4); }); it("executes against the LATEST deps, not the ones present at registration", async () => { @@ -196,7 +201,7 @@ describe("useStudioAgentTools", () => { mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(3); + expect(registerTool).toHaveBeenCalledTimes(4); }); it("reports a non-abort registration failure through production telemetry", async () => { diff --git a/packages/studio/src/webmcp/useStudioAgentTools.ts b/packages/studio/src/webmcp/useStudioAgentTools.ts index 2c67d233289..9af5b8cc619 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.ts +++ b/packages/studio/src/webmcp/useStudioAgentTools.ts @@ -25,6 +25,14 @@ import { type StudioSeekResult, type StudioSelectResult, } from "./tools/selectionTools"; +import { + studioFrame, + STUDIO_FRAME_DESCRIPTION, + STUDIO_FRAME_INPUT_SCHEMA, + type FrameToolDeps, + type StudioFrameInput, + type StudioFrameResult, +} from "./tools/frameTools"; const log = makeStudioDebugLogger("webmcp"); @@ -38,7 +46,7 @@ function reportRegistration(report: ToolRegistrationReport, native: boolean): vo } } -export interface StudioAgentToolsDeps extends SelectionToolDeps { +export interface StudioAgentToolsDeps extends SelectionToolDeps, FrameToolDeps { /** Read Studio's current state. Called per tool invocation, never cached. */ getSnapshot: () => StudioLookSnapshot; } @@ -90,6 +98,15 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): studioSeek(depsRef.current, readNumberInput(input, "time")), ), }, + { + name: "studio_frame", + title: "See the composition", + description: STUDIO_FRAME_DESCRIPTION, + inputSchema: STUDIO_FRAME_INPUT_SCHEMA, + annotations: { readOnlyHint: true, untrustedContentHint: true }, + execute: (input): Promise> => + runToolBody("studio_frame", () => studioFrame(depsRef.current, input as StudioFrameInput)), + }, ]; } From 7e03e0b6f79c33d50b1b4882f9fd368c6cfb22ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 30 Aug 2026 01:12:01 -0400 Subject: [PATCH 3/3] feat(studio): add studio_inspect, so an agent reads before it writes (#3517) Everything about one element in one call: resolved styles, text fields, box, data attributes, GSAP animations, and what the element will and will not accept. The point is to prevent a failed write rather than to satisfy curiosity. `can.reasonIfDisabled` is passed through verbatim from Studio's own capabilities, so an agent that reads first should never attempt an edit the element would refuse. Three things it refuses to get wrong: Animations are reported ONLY for the current selection, because that is the only element Studio parses them for. Attributing them to any other element would be reporting the wrong element's motion, which is worse than reporting none. When a handle names something else the field is empty and `animationEditingBlocked` says why. `animationEditingBlocked` also carries the two states where animation editing is off entirely, multiple timelines and an unsupported timeline pattern. Both live on the selection context. Learning them from a read costs one call; learning them from a failed write costs a retry loop. Inspecting a handle does NOT change what is selected. It is a read, and stealing the human's selection would be a side effect they did not ask for. There is a test asserting `applySelection` is never called. Nothing selected and no handle given is a failure, not an empty result. An empty result would assert "this element has nothing", which is a different and false claim. --- .../studio/src/webmcp/StudioAgentTools.tsx | 17 +- .../src/webmcp/tools/frameTools.test.ts | 12 +- .../src/webmcp/tools/inspectTools.test.ts | 197 +++++++++++++++++ .../studio/src/webmcp/tools/inspectTools.ts | 208 ++++++++++++++++++ .../src/webmcp/tools/selectionTools.test.ts | 52 +---- .../src/webmcp/useStudioAgentTools.test.tsx | 13 +- .../studio/src/webmcp/useStudioAgentTools.ts | 21 +- packages/studio/src/webmcp/webmcpTestUtils.ts | 91 ++++++++ 8 files changed, 544 insertions(+), 67 deletions(-) create mode 100644 packages/studio/src/webmcp/tools/inspectTools.test.ts create mode 100644 packages/studio/src/webmcp/tools/inspectTools.ts create mode 100644 packages/studio/src/webmcp/webmcpTestUtils.ts diff --git a/packages/studio/src/webmcp/StudioAgentTools.tsx b/packages/studio/src/webmcp/StudioAgentTools.tsx index e26612c1752..4fbf55e689e 100644 --- a/packages/studio/src/webmcp/StudioAgentTools.tsx +++ b/packages/studio/src/webmcp/StudioAgentTools.tsx @@ -18,7 +18,12 @@ import type { StudioLookSnapshot } from "./tools/lookTools"; */ export function StudioAgentTools() { const { projectId, activeCompPath, editHistory } = useStudioShellContext(); - const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext(); + const { + domEditSelection, + selectedGsapAnimations, + gsapMultipleTimelines, + gsapUnsupportedTimelinePattern, + } = useDomEditSelectionContext(); const { previewIframeRef, buildDomSelectionFromTarget, applyDomSelection } = useDomEditActionsContext(); @@ -71,6 +76,12 @@ export function StudioAgentTools() { } }, wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + getCurrentSelection: () => domEditSelection, + getGsapDiagnostics: () => ({ + animations: selectedGsapAnimations, + multipleTimelines: gsapMultipleTimelines, + unsupportedTimelinePattern: gsapUnsupportedTimelinePattern, + }), }), [ getSnapshot, @@ -79,6 +90,10 @@ export function StudioAgentTools() { applyDomSelection, projectId, activeCompPath, + domEditSelection, + selectedGsapAnimations, + gsapMultipleTimelines, + gsapUnsupportedTimelinePattern, ], ); diff --git a/packages/studio/src/webmcp/tools/frameTools.test.ts b/packages/studio/src/webmcp/tools/frameTools.test.ts index e98d5c2e46c..8ac0b5a432e 100644 --- a/packages/studio/src/webmcp/tools/frameTools.test.ts +++ b/packages/studio/src/webmcp/tools/frameTools.test.ts @@ -1,7 +1,7 @@ // @vitest-environment jsdom import { describe, expect, it, vi } from "vitest"; import { studioFrame, type FrameToolDeps, type StudioFrameResult } from "./frameTools"; -import type { ToolFailure, ToolResult } from "../toolResult"; +import { expectFailure, expectOk } from "../webmcpTestUtils"; function frameDeps(overrides: Partial = {}): FrameToolDeps { return { @@ -15,16 +15,6 @@ function frameDeps(overrides: Partial = {}): FrameToolDeps { }; } -function expectOk(result: ToolResult): { ok: true } & T { - if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`); - return result; -} - -function expectFailure(result: ToolResult): ToolFailure { - if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`); - return result; -} - describe("studioFrame", () => { it("returns a URL for the composition at the playhead", async () => { const result = await studioFrame(frameDeps()); diff --git a/packages/studio/src/webmcp/tools/inspectTools.test.ts b/packages/studio/src/webmcp/tools/inspectTools.test.ts new file mode 100644 index 00000000000..d59034a45f2 --- /dev/null +++ b/packages/studio/src/webmcp/tools/inspectTools.test.ts @@ -0,0 +1,197 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; +import type { GsapAnimation } from "@hyperframes/parsers/gsap-parser"; +import { studioInspect, type InspectToolDeps, type StudioInspectResult } from "./inspectTools"; +import { + expectFailure, + expectOk, + previewDoc, + previewElement, + selectionFor, +} from "../webmcpTestUtils"; + +function animation(overrides: Partial = {}): GsapAnimation { + return { + id: "anim-1", + targetSelector: "#headline", + method: "from", + position: 0, + properties: { y: -50, opacity: 0 }, + duration: 1, + ease: "power2.out", + ...overrides, + } as GsapAnimation; +} + +function inspectDeps(overrides: Partial = {}): InspectToolDeps { + return { + getPreviewDocument: () => null, + buildSelection: async (element) => selectionFor(element), + applySelection: () => undefined, + requestSeek: () => undefined, + readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }), + getCurrentSelection: () => null, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), + ...overrides, + }; +} + +describe("studioInspect", () => { + it("returns the resolved styles, not the authored ones", async () => { + const element = previewElement('

Ship it

', "headline"); + const selection = selectionFor(element); + + const result = await studioInspect(inspectDeps({ getCurrentSelection: () => selection })); + + const ok = expectOk(result); + // The authored value is a clamp(); the resolved one is what actually renders. + expect(ok.styles["font-size"]).toBe("42.7px"); + expect(ok.inlineStyles.color).toBe("red"); + expect(ok.box.width).toBe(880); + }); + + it("reports capabilities and the disabled reason verbatim", async () => { + const element = previewElement('

Ship it

', "headline"); + const locked = selectionFor(element, { + capabilities: { + canSelect: true, + canEditStyles: false, + canCrop: false, + canMove: false, + canResize: false, + canApplyManualOffset: false, + canApplyManualSize: false, + canApplyManualRotation: false, + reasonIfDisabled: "Element is inside a locked composition", + }, + }); + + const result = await studioInspect(inspectDeps({ getCurrentSelection: () => locked })); + + const ok = expectOk(result); + expect(ok.can.editStyles).toBe(false); + expect(ok.can.move).toBe(false); + expect(ok.can.reasonIfDisabled).toBe("Element is inside a locked composition"); + }); + + it("lists the animations on the current selection", async () => { + const element = previewElement('

Ship it

', "headline"); + + const result = await studioInspect( + inspectDeps({ + getCurrentSelection: () => selectionFor(element), + getGsapDiagnostics: () => ({ + animations: [animation()], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), + }), + ); + + const ok = expectOk(result); + expect(ok.animations).toHaveLength(1); + expect(ok.animations[0]?.animationId).toBe("anim-1"); + expect(ok.animations[0]?.ease).toBe("power2.out"); + expect(ok.animationEditingBlocked).toBeNull(); + }); + + it("says WHY animation editing is unavailable, so a write is not attempted", async () => { + const element = previewElement('

Ship it

', "headline"); + const base = { + getCurrentSelection: () => selectionFor(element), + }; + + const multiple = await studioInspect( + inspectDeps({ + ...base, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: true, + unsupportedTimelinePattern: false, + }), + }), + ); + const unsupported = await studioInspect( + inspectDeps({ + ...base, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: false, + unsupportedTimelinePattern: true, + }), + }), + ); + + expect(expectOk(multiple).animationEditingBlocked).toMatch( + /multiple GSAP timelines/, + ); + expect(expectOk(unsupported).animationEditingBlocked).toMatch( + /not editable/, + ); + }); + + it("does not attribute the selection's animations to a different element", async () => { + // Studio only parses animations for the CURRENT selection. Reporting them + // against another element would report the wrong element's motion. + const headline = previewElement('

A

B

', "headline"); + const doc = headline.ownerDocument; + + const result = await studioInspect( + inspectDeps({ + getPreviewDocument: () => doc, + getCurrentSelection: () => selectionFor(headline), + getGsapDiagnostics: () => ({ + animations: [animation()], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), + }), + { handle: "dom:body" }, + ); + + const ok = expectOk(result); + expect(ok.isCurrentSelection).toBe(false); + expect(ok.animations).toEqual([]); + expect(ok.animationEditingBlocked).toMatch(/only readable for the current selection/); + }); + + it("inspects a handle without changing what is selected", async () => { + const doc = previewDoc('

A

'); + const applySelection = vi.fn(); + + const result = await studioInspect( + inspectDeps({ getPreviewDocument: () => doc, applySelection }), + { handle: "dom:headline" }, + ); + + expect(result.ok).toBe(true); + // Inspecting is a read. It must not steal the human's selection. + expect(applySelection).not.toHaveBeenCalled(); + }); + + it("fails rather than returning an empty result when nothing is selected", async () => { + const result = expectFailure(await studioInspect(inspectDeps())); + + // An empty result would assert "this element has nothing", a different and + // false claim from "you did not say which element". + expect(result.kind).toBe("invalid"); + expect(result.reason).toMatch(/nothing is selected/); + expect(result.hint).toMatch(/studio_select/); + }); + + it("reports an unknown handle distinctly from an unmounted preview", async () => { + const notMounted = expectFailure(await studioInspect(inspectDeps(), { handle: "dom:x" })); + expect(notMounted.kind).toBe("blocked"); + + const doc = previewDoc('

A

'); + const unknown = expectFailure( + await studioInspect(inspectDeps({ getPreviewDocument: () => doc }), { handle: "dom:x" }), + ); + expect(unknown.kind).toBe("invalid"); + expect(unknown.reason).not.toBe(notMounted.reason); + }); +}); diff --git a/packages/studio/src/webmcp/tools/inspectTools.ts b/packages/studio/src/webmcp/tools/inspectTools.ts new file mode 100644 index 00000000000..f31b9fc8347 --- /dev/null +++ b/packages/studio/src/webmcp/tools/inspectTools.ts @@ -0,0 +1,208 @@ +/** + * `studio_inspect`: everything about one element, in one call. + * + * The point is to prevent a failed write. Every field here either tells the + * agent what it can change (`can`, with `reasonIfDisabled` verbatim) or what it + * would be changing (the resolved styles, the text fields, the animations). + * An agent that reads this first should never attempt an edit the element will + * refuse. + * + * The GSAP diagnostics are here for the same reason: `multipleTimelines` and + * `unsupportedTimelinePattern` are states where animation editing is off, and + * learning that from a read is cheaper than learning it from a failed write. + */ + +import type { GsapAnimation } from "@hyperframes/parsers/gsap-parser"; +import type { DomEditSelection } from "../../components/editor/domEditingTypes"; +import { mintElementHandle, patchTargetAddress, resolveElementHandle } from "../handles"; +import { toolFailure, toolOk, type ToolResult } from "../toolResult"; +import type { SelectionToolDeps } from "./selectionTools"; + +export interface InspectToolDeps extends SelectionToolDeps { + /** What the human currently has selected, used when no handle is given. */ + getCurrentSelection: () => DomEditSelection | null; + getGsapDiagnostics: () => { + animations: readonly GsapAnimation[]; + multipleTimelines: boolean; + unsupportedTimelinePattern: boolean; + }; +} + +interface InspectAnimation { + animationId: string; + method: string; + target: string; + position: number | string; + duration: number | null; + ease: string | null; + properties: Record; + hasKeyframes: boolean; + hasArcPath: boolean; +} + +interface InspectTextField { + key: string; + label: string; + value: string; + tagName: string; +} + +export interface StudioInspectResult { + handle: string | null; + label: string; + tagName: string; + sourceFile: string; + box: { x: number; y: number; width: number; height: number }; + text: string | null; + textFields: InspectTextField[]; + /** The styles Studio itself surfaces, resolved, not as authored. */ + styles: Record; + inlineStyles: Record; + dataAttributes: Record; + can: { + editStyles: boolean; + move: boolean; + resize: boolean; + rotate: boolean; + crop: boolean; + editText: boolean; + reasonIfDisabled: string | null; + }; + animations: InspectAnimation[]; + /** Present only when animation editing is unavailable, with the reason. */ + animationEditingBlocked: string | null; + /** True when this element is the one the human currently has selected. */ + isCurrentSelection: boolean; +} + +export interface StudioInspectInput { + /** Omit to inspect the current selection. */ + handle?: string; +} + +function describeAnimation(animation: GsapAnimation): InspectAnimation { + return { + animationId: animation.id, + method: animation.method, + target: animation.targetSelector, + position: animation.position, + duration: animation.duration ?? null, + ease: animation.ease ?? null, + properties: animation.properties, + hasKeyframes: animation.keyframes !== undefined, + hasArcPath: animation.arcPath !== undefined, + }; +} + +function describe( + selection: DomEditSelection, + deps: InspectToolDeps, + isCurrentSelection: boolean, +): ToolResult { + const { capabilities } = selection; + const gsap = deps.getGsapDiagnostics(); + + // Only the CURRENT selection's animations are parsed by Studio. Reporting + // them for some other element would be reporting the wrong element's motion, + // which is worse than reporting none. + const animations = isCurrentSelection ? gsap.animations.map(describeAnimation) : []; + + let animationEditingBlocked: string | null = null; + if (!isCurrentSelection) { + animationEditingBlocked = "animations are only readable for the current selection"; + } else if (gsap.multipleTimelines) { + animationEditingBlocked = "this composition has multiple GSAP timelines"; + } else if (gsap.unsupportedTimelinePattern) { + animationEditingBlocked = "this composition's timeline pattern is not editable by Studio"; + } + + return toolOk({ + handle: mintElementHandle(patchTargetAddress(selection)), + label: selection.label, + tagName: selection.tagName, + sourceFile: selection.sourceFile, + box: selection.boundingBox, + text: selection.textContent, + textFields: selection.textFields.map((field) => ({ + key: field.key, + label: field.label, + value: field.value, + tagName: field.tagName, + })), + styles: selection.computedStyles, + inlineStyles: selection.inlineStyles, + dataAttributes: selection.dataAttributes, + can: { + editStyles: capabilities.canEditStyles, + move: capabilities.canMove || capabilities.canApplyManualOffset, + resize: capabilities.canResize || capabilities.canApplyManualSize, + rotate: capabilities.canApplyManualRotation, + crop: capabilities.canCrop, + editText: selection.textFields.length > 0, + reasonIfDisabled: capabilities.reasonIfDisabled ?? null, + }, + animations, + animationEditingBlocked, + isCurrentSelection, + }); +} + +export async function studioInspect( + deps: InspectToolDeps, + input: StudioInspectInput = {}, +): Promise> { + const current = deps.getCurrentSelection(); + + if (!input.handle) { + // An empty result here would assert "this element has nothing", which is a + // different and false claim from "you did not tell me which element". + if (!current) { + return toolFailure( + "invalid", + "nothing is selected and no handle was given", + "Pass a handle from studio_look, or call studio_select first.", + ); + } + return describe(current, deps, true); + } + + const doc = deps.getPreviewDocument(); + if (!doc) return toolFailure("blocked", "the preview is not mounted yet"); + + const element = resolveElementHandle(doc, input.handle); + if (!element) { + return toolFailure( + "invalid", + `no element matches handle ${input.handle}`, + "Call studio_look for current handles.", + ); + } + + const selection = await deps.buildSelection(element); + if (!selection) { + return toolFailure("blocked", `${input.handle} resolved to an element Studio cannot inspect`); + } + + return describe(selection, deps, current?.element === element); +} + +export const STUDIO_INSPECT_INPUT_SCHEMA = { + type: "object", + properties: { + handle: { + type: "string", + description: "An element handle from studio_look. Omit to inspect the current selection.", + }, + }, + additionalProperties: false, +} as const; + +export const STUDIO_INSPECT_DESCRIPTION = [ + "Everything about one element: its resolved styles, its text fields, its box,", + "its GSAP animations, and crucially what it will and will not accept.", + "Read this BEFORE editing. `can` tells you which edits are possible and", + "`can.reasonIfDisabled` says why one is not, so you can avoid a write that would be refused.", + "Animations are only readable for the CURRENT selection; `animationEditingBlocked` says when", + "and why animation editing is unavailable.", + "Returns `ok: true`, or `ok: false` with `kind`, `reason` and a `hint`.", +].join(" "); diff --git a/packages/studio/src/webmcp/tools/selectionTools.test.ts b/packages/studio/src/webmcp/tools/selectionTools.test.ts index c6ec5fdd2a7..07168529aec 100644 --- a/packages/studio/src/webmcp/tools/selectionTools.test.ts +++ b/packages/studio/src/webmcp/tools/selectionTools.test.ts @@ -1,6 +1,5 @@ // @vitest-environment jsdom import { describe, expect, it, vi } from "vitest"; -import type { DomEditSelection } from "../../components/editor/domEditingTypes"; import { studioSeek, studioSelect, @@ -8,46 +7,7 @@ import { type StudioSeekResult, type StudioSelectResult, } from "./selectionTools"; -import type { ToolFailure, ToolResult } from "../toolResult"; - -function previewDoc(html: string): Document { - const iframe = document.createElement("iframe"); - document.body.append(iframe); - const doc = iframe.contentDocument; - if (!doc) throw new Error("expected iframe document"); - doc.body.innerHTML = html; - return doc; -} - -function selectionFor(element: HTMLElement): DomEditSelection { - return { - id: element.id || undefined, - hfId: element.getAttribute("data-hf-id") ?? undefined, - element, - label: "Headline", - tagName: element.tagName.toLowerCase(), - sourceFile: "index.html", - compositionPath: "index.html", - isCompositionHost: false, - isInsideLockedComposition: false, - boundingBox: { x: 40, y: 12, width: 880, height: 96 }, - textContent: element.textContent, - dataAttributes: {}, - inlineStyles: {}, - computedStyles: {}, - textFields: [], - capabilities: { - canSelect: true, - canEditStyles: true, - canCrop: true, - canMove: true, - canResize: true, - canApplyManualOffset: true, - canApplyManualSize: true, - canApplyManualRotation: true, - }, - }; -} +import { expectFailure, expectOk, previewDoc, selectionFor } from "../webmcpTestUtils"; function selectionDeps(overrides: Partial = {}): SelectionToolDeps { return { @@ -60,16 +20,6 @@ function selectionDeps(overrides: Partial = {}): SelectionToo }; } -function expectFailure(result: ToolResult): ToolFailure { - if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`); - return result; -} - -function expectOk(result: ToolResult): { ok: true } & T { - if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`); - return result; -} - describe("studioSelect", () => { it("applies the selection a click would produce and reports it back", async () => { const doc = previewDoc('

Ship it

'); diff --git a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx index d9c7dd40819..3a9a3ee988e 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.test.tsx +++ b/packages/studio/src/webmcp/useStudioAgentTools.test.tsx @@ -42,6 +42,12 @@ function deps(overrides: Partial = {}): StudioAgentToolsDe getCompositionPath: () => "index.html", probeFrame: async () => ({ ok: true, status: 200 }), wait: async () => undefined, + getCurrentSelection: () => null, + getGsapDiagnostics: () => ({ + animations: [], + multipleTimelines: false, + unsupportedTimelinePattern: false, + }), ...overrides, }; } @@ -107,6 +113,7 @@ describe("useStudioAgentTools", () => { "studio_select", "studio_seek", "studio_frame", + "studio_inspect", ]); expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present"); }); @@ -121,14 +128,14 @@ describe("useStudioAgentTools", () => { await act(async () => { harness = mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(4); + expect(registerTool).toHaveBeenCalledTimes(5); await act(async () => { harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) })); harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) })); }); - expect(registerTool).toHaveBeenCalledTimes(4); + expect(registerTool).toHaveBeenCalledTimes(5); }); it("executes against the LATEST deps, not the ones present at registration", async () => { @@ -201,7 +208,7 @@ describe("useStudioAgentTools", () => { mountTools(deps({ getSnapshot: () => snapshot() })); }); - expect(registerTool).toHaveBeenCalledTimes(4); + expect(registerTool).toHaveBeenCalledTimes(5); }); it("reports a non-abort registration failure through production telemetry", async () => { diff --git a/packages/studio/src/webmcp/useStudioAgentTools.ts b/packages/studio/src/webmcp/useStudioAgentTools.ts index 9af5b8cc619..c5e1a444eb8 100644 --- a/packages/studio/src/webmcp/useStudioAgentTools.ts +++ b/packages/studio/src/webmcp/useStudioAgentTools.ts @@ -33,6 +33,14 @@ import { type StudioFrameInput, type StudioFrameResult, } from "./tools/frameTools"; +import { + studioInspect, + STUDIO_INSPECT_DESCRIPTION, + STUDIO_INSPECT_INPUT_SCHEMA, + type InspectToolDeps, + type StudioInspectInput, + type StudioInspectResult, +} from "./tools/inspectTools"; const log = makeStudioDebugLogger("webmcp"); @@ -46,7 +54,7 @@ function reportRegistration(report: ToolRegistrationReport, native: boolean): vo } } -export interface StudioAgentToolsDeps extends SelectionToolDeps, FrameToolDeps { +export interface StudioAgentToolsDeps extends SelectionToolDeps, FrameToolDeps, InspectToolDeps { /** Read Studio's current state. Called per tool invocation, never cached. */ getSnapshot: () => StudioLookSnapshot; } @@ -107,6 +115,17 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): execute: (input): Promise> => runToolBody("studio_frame", () => studioFrame(depsRef.current, input as StudioFrameInput)), }, + { + name: "studio_inspect", + title: "Inspect one element", + description: STUDIO_INSPECT_DESCRIPTION, + inputSchema: STUDIO_INSPECT_INPUT_SCHEMA, + annotations: { readOnlyHint: true, untrustedContentHint: true }, + execute: (input): Promise> => + runToolBody("studio_inspect", () => + studioInspect(depsRef.current, input as StudioInspectInput), + ), + }, ]; } diff --git a/packages/studio/src/webmcp/webmcpTestUtils.ts b/packages/studio/src/webmcp/webmcpTestUtils.ts new file mode 100644 index 00000000000..8f4f5e96b00 --- /dev/null +++ b/packages/studio/src/webmcp/webmcpTestUtils.ts @@ -0,0 +1,91 @@ +/** + * Shared fixtures for the WebMCP tool tests. + * + * Not a `.test` file so vitest does not collect it as a suite. Mirrors the + * existing `hooks/domSelectionTestHarness.ts` convention. + */ + +import { expect } from "vitest"; +import type { DomEditSelection } from "../components/editor/domEditingTypes"; +import type { ToolFailure, ToolResult } from "./toolResult"; + +/** + * An element inside a real iframe, which is where Studio's chrome expects to + * find preview elements. The separate realm matters: a preview element is not + * an instance of Studio's own `HTMLElement`. + */ +export function previewDoc(html: string): Document { + const iframe = document.createElement("iframe"); + document.body.append(iframe); + const doc = iframe.contentDocument; + if (!doc) throw new Error("expected iframe document"); + doc.body.innerHTML = html; + return doc; +} + +export function previewElement(html: string, id: string): HTMLElement { + const doc = previewDoc(html); + const element = doc.getElementById(id); + const HTMLElementCtor = doc.defaultView?.HTMLElement; + if (!HTMLElementCtor || !(element instanceof HTMLElementCtor)) { + throw new Error(`expected preview element #${id}`); + } + return element; +} + +export function selectionFor( + element: HTMLElement, + overrides: Partial = {}, +): DomEditSelection { + return { + id: element.id || undefined, + hfId: element.getAttribute("data-hf-id") ?? undefined, + element, + label: "Headline", + tagName: element.tagName.toLowerCase(), + sourceFile: "index.html", + compositionPath: "index.html", + isCompositionHost: false, + isInsideLockedComposition: false, + boundingBox: { x: 40, y: 12, width: 880, height: 96 }, + textContent: element.textContent, + dataAttributes: { "data-role": "title" }, + inlineStyles: { color: "red" }, + computedStyles: { "font-size": "42.7px", color: "rgb(255, 0, 0)" }, + textFields: [ + { + key: "self", + label: "Text", + value: element.textContent ?? "", + tagName: element.tagName.toLowerCase(), + attributes: [], + inlineStyles: {}, + computedStyles: {}, + source: "self", + }, + ], + capabilities: { + canSelect: true, + canEditStyles: true, + canCrop: true, + canMove: true, + canResize: true, + canApplyManualOffset: true, + canApplyManualSize: true, + canApplyManualRotation: true, + }, + ...overrides, + }; +} + +export function expectOk(result: ToolResult): { ok: true } & T { + expect(result.ok, `expected ok, got ${JSON.stringify(result)}`).toBe(true); + if (!result.ok) throw new Error("unreachable"); + return result; +} + +export function expectFailure(result: ToolResult): ToolFailure { + expect(result.ok, `expected failure, got ${JSON.stringify(result)}`).toBe(false); + if (result.ok) throw new Error("unreachable"); + return result; +}