From 414dc278dd38541b817ccf2be00eacc8d96eceb4 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 10 Sep 2026 02:34:36 +0000 Subject: [PATCH 1/6] fix(preview): render website favicons for browser tool activity --- apps/server/src/mcp/McpHttpServer.test.ts | 25 ++- .../server/src/mcp/PreviewAutomationBroker.ts | 42 +++- .../src/mcp/toolkits/preview/handlers.ts | 43 +++-- apps/server/src/mcp/toolkits/preview/tools.ts | 25 ++- .../ActivityPayloadProjection.test.ts | 181 ++++++++++++++++++ .../ActivityPayloadProjection.ts | 98 +++++++++- .../preview/PreviewAutomationHosts.tsx | 65 ++++++- .../previewAutomationRequestConsumer.test.ts | 22 ++- .../previewAutomationRequestConsumer.ts | 12 +- packages/contracts/src/previewAutomation.ts | 2 + 10 files changed, 455 insertions(+), 60 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 885629930706..c0ae2cd0571f 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -615,6 +615,11 @@ it.effect("registers annotated tools and preserves authenticated request context Effect.gen(function* () { const server = yield* McpServer.McpServer; const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const toolIcon = { + _tag: "website" as const, + pageUrl: "http://example.test/", + faviconUrl: "data:image/png;base64,aWNvbg==", + }; const routedRequests: Array<{ readonly operation: string; readonly tabId?: string | undefined; @@ -631,6 +636,7 @@ it.effect("registers annotated tools and preserves authenticated request context connectionId: event.connectionId, requestId: event.request.requestId, ok: true, + toolIcon, result: event.request.operation === "snapshot" ? snapshotResult @@ -664,7 +670,7 @@ it.effect("registers annotated tools and preserves authenticated request context expect(clickTool?.tool.annotations?.readOnlyHint).toBe(false); expect(clickTool?.tool.annotations?.destructiveHint).toBe(true); expect(clickTool?.tool.annotations?.openWorldHint).toBe(true); - expect(clickTool?.tool.outputSchema).toEqual({ + expect(clickTool?.tool.outputSchema).toMatchObject({ type: "object", additionalProperties: false, description: "The preview action completed successfully.", @@ -684,6 +690,7 @@ it.effect("registers annotated tools and preserves authenticated request context expect(status.structuredContent).toMatchObject({ available: true, tabId, + toolIcon, }); const malformed = yield* server @@ -704,6 +711,7 @@ it.effect("registers annotated tools and preserves authenticated request context expect(snapshot.isError).toBe(false); expect(snapshot.content.some((content) => content.type === "image")).toBe(true); expect(snapshot.structuredContent).toMatchObject({ + toolIcon, screenshot: { mimeType: "image/png", width: 10, height: 5 }, }); expect(routedRequests.find(({ operation }) => operation === "snapshot")?.tabId).toBe( @@ -721,10 +729,12 @@ it.effect("registers annotated tools and preserves authenticated request context Effect.provideService(McpSchema.McpServerClient, client), ); expect(evaluated.isError).toBe(false); - expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"] }); - expect(evaluated.content).toEqual([ - { type: "text", text: '{"value":["Connect","Continue"]}' }, - ]); + expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"], toolIcon }); + const evaluatedText = evaluated.content[0]; + expect(evaluatedText?.type === "text" ? decodeJsonText(evaluatedText.text) : null).toEqual({ + toolIcon, + value: ["Connect", "Continue"], + }); const actionRequests = [ { name: "preview_click", arguments: { x: 10, y: 10 } }, @@ -741,8 +751,9 @@ it.effect("registers annotated tools and preserves authenticated request context Effect.provideService(McpSchema.McpServerClient, client), ); expect(result.isError).toBe(false); - expect(result.structuredContent).toEqual({}); - expect(result.content).toEqual([{ type: "text", text: "{}" }]); + expect(result.structuredContent).toEqual({ toolIcon }); + const text = result.content[0]; + expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual({ toolIcon }); } }), ).pipe(Effect.provide(TestLayer)), diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index d8f17973c218..9c1a146c61a8 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -46,6 +46,11 @@ export interface PreviewAutomationInvokeInput { readonly timeoutMs?: number; } +export interface PreviewAutomationResult { + readonly result: A; + readonly toolIcon?: PreviewAutomationResponse["toolIcon"]; +} + export class PreviewAutomationBroker extends Context.Service< PreviewAutomationBroker, { @@ -59,6 +64,9 @@ export class PreviewAutomationBroker extends Context.Service< readonly invoke: ( request: PreviewAutomationInvokeInput, ) => Effect.Effect; + readonly invokeWithPresentation: ( + request: PreviewAutomationInvokeInput, + ) => Effect.Effect, PreviewAutomationError>; } >()("t3/mcp/PreviewAutomationBroker") {} @@ -74,7 +82,7 @@ interface ClientConnection { interface PendingRequest { readonly queue: ClientConnection["queue"]; - readonly deferred: Deferred.Deferred; + readonly deferred: Deferred.Deferred, PreviewAutomationError>; readonly context: PreviewAutomationRequestErrorContext; } @@ -436,7 +444,10 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { }); if (!pending) return; if (response.ok) { - yield* Deferred.succeed(pending.deferred, response.result); + yield* Deferred.succeed(pending.deferred, { + result: response.result, + ...(response.toolIcon ? { toolIcon: response.toolIcon } : {}), + }); } else { yield* Deferred.fail( pending.deferred, @@ -447,11 +458,16 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { } }); - const invoke = Effect.fn("PreviewAutomationBroker.invoke")(function* ( + const invokeWithPresentation = Effect.fn("PreviewAutomationBroker.invoke")(function* < + A = unknown, + >( input: Parameters[0], - ): Effect.fn.Return { + ): Effect.fn.Return, PreviewAutomationError> { const timeoutMs = input.timeoutMs ?? 15_000; - const deferred = yield* Deferred.make(); + const deferred = yield* Deferred.make< + PreviewAutomationResult, + PreviewAutomationError + >(); const route = yield* SynchronizedRef.modify(state, (current) => { const assignments = new Map( Array.from(current.assignments).filter(([, assignment]) => { @@ -564,18 +580,18 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { if (!offered) { const completion = yield* Deferred.poll(deferred); if (Option.isSome(completion)) { - return (yield* completion.value) as A; + return (yield* completion.value) as PreviewAutomationResult; } return yield* new PreviewAutomationRequestQueueClosedError(requestContext); } const result = yield* Deferred.await(deferred).pipe(Effect.timeoutOption(timeoutMs)); return yield* Option.match(result, { onNone: () => Effect.fail(new PreviewAutomationTimeoutError(requestContext)), - onSome: (value) => Effect.succeed(value as A), + onSome: (value) => Effect.succeed(value as PreviewAutomationResult), }); }); const result = yield* awaitResponse().pipe(Effect.ensuring(removePending)); - const responseTabId = readResultTabId(result); + const responseTabId = readResultTabId(result.result); const resultTabId = responseTabId === undefined ? input.tabId : responseTabId; if (resultTabId === undefined) return result; const assignmentKey = hostAssignmentKey(input.scope); @@ -605,7 +621,15 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { return result; }); - return PreviewAutomationBroker.of({ connect, focusHost, respond, invoke }); + const invoke = (input: PreviewAutomationInvokeInput) => + invokeWithPresentation(input).pipe(Effect.map(({ result }) => result)); + return PreviewAutomationBroker.of({ + connect, + focusHost, + respond, + invoke, + invokeWithPresentation, + }); }).pipe(Effect.withSpan("PreviewAutomationBroker.make")); export const layer = Layer.effect(PreviewAutomationBroker, make); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index d34c2d3ba3af..8f008f4566ff 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -55,13 +55,13 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( timeoutMs?: number, tabId?: PreviewTabId, ): Effect.fn.Return< - A, + PreviewAutomationBroker.PreviewAutomationResult, import("@t3tools/contracts").PreviewAutomationError, McpInvocationContext.McpInvocationContext | PreviewAutomationBroker.PreviewAutomationBroker > { const scope = yield* McpInvocationContext.requireMcpCapability("preview"); const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; - return yield* broker.invoke({ + return yield* broker.invokeWithPresentation({ scope, operation, input, @@ -70,7 +70,7 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( }); }); -const invokeTargeted = ( +const invokeTargeted = ( operation: PreviewAutomationOperation, input: { readonly tabId?: PreviewTabId | undefined; @@ -79,7 +79,12 @@ const invokeTargeted = ( timeoutMs?: number, ) => { const { tabId, ...operationInput } = input; - return invoke(operation, operationInput, timeoutMs, tabId); + return invoke(operation, operationInput, timeoutMs, tabId).pipe( + Effect.map(({ result, toolIcon }) => ({ + ...result, + ...(toolIcon ? { toolIcon } : {}), + })), + ); }; const UploadedRecordingArtifact = Schema.Struct({ @@ -170,28 +175,32 @@ const handlers = { const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {}; return invokeTargeted("snapshot", operationInput); }, - preview_click: (input) => - invokeTargeted("click", input, input.timeoutMs).pipe(Effect.as({})), - preview_type: (input) => invokeTargeted("type", input, input.timeoutMs).pipe(Effect.as({})), - preview_press: (input) => invokeTargeted("press", input).pipe(Effect.as({})), - preview_scroll: (input) => invokeTargeted("scroll", input).pipe(Effect.as({})), - preview_evaluate: (input) => - invokeTargeted("evaluate", input).pipe( - Effect.map((result) => ({ value: result ?? null })), + preview_click: (input) => invokeTargeted("click", input, input.timeoutMs), + preview_type: (input) => invokeTargeted("type", input, input.timeoutMs), + preview_press: (input) => invokeTargeted("press", input), + preview_scroll: (input) => invokeTargeted("scroll", input), + preview_evaluate: ({ tabId, ...input }) => + invoke("evaluate", input, undefined, tabId).pipe( + Effect.map(({ result, toolIcon }) => ({ + value: result ?? null, + ...(toolIcon ? { toolIcon } : {}), + })), ), - preview_wait_for: (input) => - invokeTargeted("waitFor", input, input.timeoutMs).pipe(Effect.as({})), + preview_wait_for: (input) => invokeTargeted("waitFor", input, input.timeoutMs), preview_recording_start: (input) => invokeTargeted("recordingStart", input ?? {}), preview_recording_stop: (input) => Effect.gen(function* () { const scope = yield* McpInvocationContext.requireMcpCapability("preview"); - const response = yield* invokeTargeted( + const { tabId, ...operationInput } = input; + const response = yield* invoke( "recordingStop", - { ...input, transferToEnvironment: true }, + { ...operationInput, transferToEnvironment: true }, PREVIEW_RECORDING_STOP_TIMEOUT_MS, + tabId, ); - return yield* claimPreviewRecording(scope.threadId, response); + const artifact = yield* claimPreviewRecording(scope.threadId, response.result); + return { ...artifact, ...(response.toolIcon ? { toolIcon: response.toolIcon } : {}) }; }), } satisfies Parameters[0]; diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 28a2b96228b5..38786b34686b 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -1,4 +1,5 @@ import { + ToolActivityIcon, PreviewAutomationClickInput, PreviewAutomationError, PreviewAutomationEvaluateInput, @@ -31,7 +32,9 @@ const dependencies = [ PreviewAutomationBroker.PreviewAutomationBroker, ]; -const PreviewActionResult = Schema.Record(Schema.String, Schema.Never).annotate({ +const presentationFields = { toolIcon: Schema.optional(ToolActivityIcon) }; + +const PreviewActionResult = Schema.Struct(presentationFields).annotate({ description: "The preview action completed successfully.", }); @@ -51,7 +54,7 @@ const PreviewStatusTool = Tool.make("preview_status", { description: "Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.", parameters: PreviewAutomationTabTargetInput, - success: PreviewAutomationStatus, + success: Schema.Struct({ ...PreviewAutomationStatus.fields, ...presentationFields }), failure: PreviewAutomationError, dependencies, }) @@ -65,7 +68,7 @@ const PreviewOpenTool = browserTool( description: "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", parameters: PreviewAutomationOpenInput, - success: PreviewAutomationStatus, + success: Schema.Struct({ ...PreviewAutomationStatus.fields, ...presentationFields }), failure: PreviewAutomationError, dependencies, }) @@ -78,7 +81,7 @@ const PreviewNavigateTool = safeBrowserTool( description: "Navigate a collaborative browser tab. Pass tabId to target a specific tab, plus {url:'https://t3.chat'} for a website or {target:{kind:'environment-port',port:5173}} for a dev server. Exactly one of url or target is required.", parameters: PreviewAutomationNavigateInput, - success: PreviewAutomationStatus, + success: Schema.Struct({ ...PreviewAutomationStatus.fields, ...presentationFields }), failure: PreviewAutomationError, dependencies, }).annotate(Tool.Title, "Navigate browser preview"), @@ -89,7 +92,7 @@ const PreviewResizeTool = safeBrowserTool( description: "Resize a collaborative browser tab, optionally selected by tabId. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.", parameters: PreviewAutomationResizeInput, - success: PreviewAutomationResizeResult, + success: Schema.Struct({ ...PreviewAutomationResizeResult.fields, ...presentationFields }), failure: PreviewAutomationError, dependencies, }) @@ -102,7 +105,10 @@ const PreviewSetAppearanceTool = safeBrowserTool( description: "Emulate prefers-color-scheme in a collaborative browser tab, optionally selected by tabId. Use {colorScheme:'dark'} or {colorScheme:'light'} to preview the page in that appearance, and {colorScheme:'system'} to clear the override and follow the OS appearance.", parameters: PreviewAutomationSetColorSchemeInput, - success: PreviewAutomationSetColorSchemeResult, + success: Schema.Struct({ + ...PreviewAutomationSetColorSchemeResult.fields, + ...presentationFields, + }), failure: PreviewAutomationError, dependencies, }) @@ -129,7 +135,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool( }), ), }), - success: PreviewAutomationSnapshot, + success: Schema.Struct({ ...PreviewAutomationSnapshot.fields, ...presentationFields }), failure: PreviewAutomationError, dependencies, }).annotate(Tool.Title, "Inspect browser page"), @@ -185,6 +191,7 @@ const PreviewScrollTool = safeBrowserTool( * null valid instead of failing only for non-object expressions. */ export const PreviewEvaluateResult = Schema.Struct({ + ...presentationFields, value: Schema.Unknown.annotate({ description: "The JSON-serializable value the expression produced, or null.", }), @@ -217,7 +224,7 @@ const PreviewRecordingStartTool = safeBrowserTool( description: "Start recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted.", parameters: PreviewAutomationTabTargetInput, - success: PreviewAutomationRecordingStatus, + success: Schema.Struct({ ...PreviewAutomationRecordingStatus.fields, ...presentationFields }), failure: PreviewAutomationError, dependencies, }).annotate(Tool.Title, "Start browser recording"), @@ -228,7 +235,7 @@ const PreviewRecordingStopTool = safeBrowserTool( description: "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and transfer the compressed recording once (up to 50 MiB) to an evidence file readable in this agent's environment. Returns its environment-local path after transfer succeeds.", parameters: PreviewAutomationTabTargetInput, - success: PreviewAutomationRecordingArtifact, + success: Schema.Struct({ ...PreviewAutomationRecordingArtifact.fields, ...presentationFields }), failure: PreviewAutomationError, dependencies: [...dependencies, FileSystem.FileSystem, ServerConfig.ServerConfig], }).annotate(Tool.Title, "Stop browser recording"), diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts index bf09ed959e17..67885bc3ec6f 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts @@ -249,6 +249,187 @@ describe("projectActivityPayload", () => { expect(JSON.stringify(projected.payload).length).toBeLessThan(500); }); + it("preserves preview website metadata through result slimming and repeated projection", () => { + const toolIcon = { + _tag: "website", + pageUrl: "https://example.com/page", + faviconUrl: "https://example.com/icon.png", + }; + const projected = projectActivityPayload( + activity({ + itemType: "mcp_tool_call", + data: { + item: { + server: "t3-code", + tool: "preview_click", + result: { + structuredContent: { toolIcon }, + content: [ + { + type: "text", + text: JSON.stringify({ + toolIcon: { + _tag: "website", + pageUrl: "https://other.example/", + }, + }), + }, + ], + }, + }, + }, + }), + ); + expect(projected.payload).toMatchObject({ toolSurface: "browser", toolIcon }); + expect(projectActivityPayload(projected).payload).toMatchObject({ + toolSurface: "browser", + toolIcon, + }); + expect(projected.payload).not.toMatchObject({ + data: { item: { result: { structuredContent: expect.anything() } } }, + }); + }); + + it.each([ + { + toolName: "mcp__t3-code__preview_snapshot", + result: { + type: "tool_result", + content: [ + { + type: "text", + text: JSON.stringify({ + toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, + }), + }, + ], + }, + }, + { + tool: "t3-code_preview_snapshot", + state: { + status: "completed", + output: JSON.stringify({ + toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, + }), + }, + }, + ])("reads preview metadata from provider-preserved JSON output", (data) => { + const projected = projectActivityPayload(activity({ itemType: "dynamic_tool_call", data })); + expect(projected.payload).toMatchObject({ + toolSurface: "browser", + toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, + }); + }); + + it.each(["t3-code", "t3_code", "t3code"])("recognizes the %s preview server alias", (server) => { + const result = { content: JSON.stringify({ url: "https://example.com/" }) }; + for (const data of [ + { item: { server, tool: "preview_open", result } }, + { toolName: `mcp__${server}__preview_open`, result }, + { tool: `${server}_preview_open`, state: { output: result.content } }, + ]) { + const projected = projectActivityPayload(activity({ itemType: "mcp_tool_call", data })); + expect(projected.payload).toMatchObject({ + toolSurface: "browser", + toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, + }); + } + }); + + it("infers legacy preview URLs and prefers the returned page after redirects", () => { + const projected = projectActivityPayload( + activity({ + itemType: "mcp_tool_call", + data: { + toolName: "mcp__t3-code__preview_navigate", + input: { url: "https://before.example/" }, + result: { content: JSON.stringify({ url: "https://after.example/" }) }, + }, + }), + ); + expect(projected.payload).toMatchObject({ + toolIcon: { _tag: "website", pageUrl: "https://after.example/" }, + }); + const started = projectActivityPayload( + activity({ + itemType: "mcp_tool_call", + data: { + toolName: "mcp__t3-code__preview_navigate", + input: { target: { kind: "url", url: "https://before.example/" } }, + }, + }), + ); + expect(started.payload).toMatchObject({ + toolIcon: { _tag: "website", pageUrl: "https://before.example/" }, + }); + }); + + it.each([ + { toolName: "mcp__other__preview_click" }, + { item: { server: "other", tool: "preview_click" } }, + { toolName: "preview_click" }, + ])("ignores preview metadata from unrelated tools", (identity) => { + const result = { + structuredContent: { toolIcon: { _tag: "website", pageUrl: "https://example.com/" } }, + }; + const projected = projectActivityPayload( + activity({ + itemType: "mcp_tool_call", + data: { + ...identity, + result, + ...("item" in identity ? { item: { ...identity.item, result } } : {}), + }, + }), + ); + expect(projected.payload).not.toHaveProperty("toolIcon"); + expect(projected.payload).not.toHaveProperty("toolSurface"); + }); + + it.each([ + { result: { isError: true, content: JSON.stringify({ url: "https://stale.example/" }) } }, + { result: { is_error: true, content: "navigation failed" } }, + { result: { structuredContent: { toolIcon: { _tag: "website", pageUrl: 42 } } } }, + { + result: { + structuredContent: { + toolIcon: { _tag: "themed-logo", logoUrl: "https://example.com/icon.png" }, + }, + }, + }, + { result: { content: "malformed JSON" } }, + { result: { content: JSON.stringify({ url: "about:blank" }) } }, + ])("does not invent a page icon for failures or invalid results", (output) => { + const projected = projectActivityPayload( + activity({ + itemType: "mcp_tool_call", + data: { + toolName: "mcp__t3-code__preview_navigate", + input: { url: "https://attempted.example/" }, + ...output, + }, + }), + ); + expect(projected.payload).toMatchObject({ toolSurface: "browser" }); + expect(projected.payload).not.toHaveProperty("toolIcon"); + }); + + it("preserves explicit activity icons", () => { + const toolIcon = { _tag: "website", pageUrl: "https://explicit.example/" }; + const projected = projectActivityPayload( + activity({ + itemType: "mcp_tool_call", + toolIcon, + data: { + toolName: "mcp__t3-code__preview_status", + result: { content: JSON.stringify({ url: "https://inferred.example/" }) }, + }, + }), + ); + expect(projected.payload).toMatchObject({ toolIcon }); + }); + it("passes task lifecycle payloads (no data field) through untouched", () => { const source = activity({ taskId: "task-9", diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.ts b/apps/server/src/orchestration/ActivityPayloadProjection.ts index 98294e63b35c..172d0d4bb31e 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.ts @@ -1,9 +1,11 @@ -import type { - OrchestrationEvent, - OrchestrationThreadActivity, - OrchestrationThreadDetailSnapshot, +import { + ToolActivityIcon, + type OrchestrationEvent, + type OrchestrationThreadActivity, + type OrchestrationThreadDetailSnapshot, } from "@t3tools/contracts"; import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; +import * as Schema from "effect/Schema"; function asRecord(value: unknown): Record | null { return value !== null && typeof value === "object" && !Array.isArray(value) @@ -240,6 +242,88 @@ function summarizeMcpResult(result: unknown): Record | undefine return summary ? { content: summary } : undefined; } +const decodeToolActivityIcon = Schema.decodeUnknownOption(ToolActivityIcon); +const PREVIEW_TOOL_NAME = + /^preview_(?:status|open|navigate|resize|set_appearance|snapshot|click|type|press|scroll|evaluate|wait_for|recording_start|recording_stop)$/u; + +function parsePreviewResult(value: unknown): Record | null { + if (typeof value !== "string") return asRecord(value); + // Snapshot text can be large; never parse arbitrary unbounded tool output. + if (value.length > 2 * 1024 * 1024) return null; + try { + return asRecord(JSON.parse(value)); + } catch { + return null; + } +} + +function projectPreviewToolMetadata(data: Record, status: unknown) { + const item = asRecord(data.item); + const qualifiedName = asTrimmedString(data.toolName) ?? asTrimmedString(data.tool); + const tool = item + ? /^(?:t3-code|t3_code|t3code)$/u.test(asTrimmedString(item.server) ?? "") + ? asTrimmedString(item.tool) + : null + : qualifiedName?.replace( + /^(?:mcp__(?:t3-code|t3_code|t3code)__|(?:t3-code|t3_code|t3code)_)/u, + "", + ); + if (!tool || !PREVIEW_TOOL_NAME.test(tool) || (!item && tool === qualifiedName)) { + return {}; + } + + const state = asRecord(data.state); + const result = item?.result ?? data.result ?? state?.output; + const resultRecord = asRecord(result); + const failed = + status === "failed" || + status === "declined" || + state?.status === "error" || + item?.error != null || + resultRecord?.isError === true || + resultRecord?.is_error === true; + if (failed) return { toolSurface: "browser" }; + + // Prefer MCP structured content, then the JSON text preserved by Claude/OpenCode. + const structuredContent = asRecord(resultRecord?.structuredContent); + const structuredIcon = decodeToolActivityIcon(structuredContent?.toolIcon); + if (structuredIcon._tag === "Some" && structuredIcon.value._tag === "website") { + return { toolSurface: "browser", toolIcon: structuredIcon.value }; + } + const candidates = [structuredContent, parsePreviewResult(result)]; + if (typeof resultRecord?.content === "string") { + candidates.push(parsePreviewResult(resultRecord.content)); + } else if (Array.isArray(resultRecord?.content)) { + for (const entry of resultRecord.content.slice(0, 32)) { + const block = asRecord(entry); + if (block?.type === "text") candidates.push(parsePreviewResult(block.text)); + } + } + for (const candidate of candidates) { + const decoded = decodeToolActivityIcon(candidate?.toolIcon); + if (decoded._tag === "Some" && decoded.value._tag === "website") { + return { toolSurface: "browser", toolIcon: decoded.value }; + } + } + + // Older preview hosts return the page URL without icon metadata. + if (/^preview_(?:status|open|navigate|snapshot)$/u.test(tool)) { + const input = asRecord(item?.arguments ?? data.input ?? state?.input); + const pageUrl = + candidates.find((candidate) => typeof candidate?.url === "string")?.url ?? + (result == null ? (input?.url ?? asRecord(input?.target)?.url) : undefined); + const decoded = decodeToolActivityIcon({ _tag: "website", pageUrl }); + if ( + decoded._tag === "Some" && + decoded.value._tag === "website" && + /^https?:\/\//iu.test(decoded.value.pageUrl) + ) { + return { toolSurface: "browser", toolIcon: decoded.value }; + } + } + return { toolSurface: "browser" }; +} + /** * MCP tool calls carry full tool results (`data.item.result` on Codex, * `data.result` on Claude/OpenCode) that used to bypass slimming entirely to @@ -366,10 +450,14 @@ export function projectActivityPayload( } const itemStatus = asRecord(data.item)?.status; - const projectedPayload = + const statusPayload = payload.status === "completed" && (itemStatus === "failed" || itemStatus === "declined") ? { ...payload, status: itemStatus } : payload; + const projectedPayload = { + ...projectPreviewToolMetadata(data, statusPayload.status), + ...statusPayload, + }; if (payload.itemType === "mcp_tool_call") { return { diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index d1fc12821730..308ac347850b 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -14,6 +14,7 @@ import { type PreviewAutomationSetColorSchemeResult, type PreviewAutomationHost as PreviewAutomationHostState, type PreviewAutomationRequest, + type PreviewAutomationResponse, type PreviewAutomationStatus, type PreviewRenderedViewportSize, type PreviewViewportSetting, @@ -75,7 +76,10 @@ import { assertPreviewRuntimeCurrent, waitForNavigationReadiness, } from "./previewNavigationReadiness"; -import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationRequestConsumer"; +import { + createPreviewAutomationRequestConsumerAtom, + type PreviewAutomationHandledResult, +} from "./previewAutomationRequestConsumer"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { needsPreviewAutomationSessionSync, @@ -324,7 +328,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const presentationSuppressedRuntimeTabsRef = useRef(new Map>()); const handleRequest = useCallback( - async (request: PreviewAutomationRequest): Promise => { + async (request: PreviewAutomationRequest): Promise => { // Session sync and tab creation consume the same budget as overlay registration. const hostDeadlineMs = Date.now() + resolveHostWaitBudgetMs(request.timeoutMs); const threadRef: ScopedThreadRef = { @@ -333,7 +337,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; let tabId = request.tabId ?? null; const browserActivity = { release: null as (() => void) | null }; - try { + const execute = async () => { let state = readThreadPreviewState(threadRef); const needsSessionSync = needsPreviewAutomationSessionSync(state, request.tabId); if (needsSessionSync) { @@ -746,6 +750,61 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; } } + }; + try { + const result: unknown = await execute(); + let toolIcon: PreviewAutomationResponse["toolIcon"]; + let iconTimeout: ReturnType | undefined; + try { + // Read the resolved target, including tabs created or selected by this operation. + const resultHasPageUrl = + ["status", "open", "navigate", "snapshot"].includes(request.operation) && + typeof result === "object" && + result !== null && + "url" in result; + const pageUrl = resultHasPageUrl + ? result.url + : tabId && previewBridge && Date.now() < hostDeadlineMs + ? ( + await Promise.race([ + previewBridge.automation.status( + previewRuntimeTabId( + threadRef, + readThreadPreviewState(threadRef).serverEpoch, + tabId, + ), + ), + new Promise((resolve) => { + iconTimeout = setTimeout( + () => resolve(null), + Math.min(300, Math.max(0, hostDeadlineMs - Date.now())), + ); + }), + ]) + )?.url + : null; + if (typeof pageUrl === "string" && pageUrl.length <= 4096) { + const url = new URL(pageUrl); + if (url.protocol === "http:" || url.protocol === "https:") { + toolIcon = { _tag: "website", pageUrl }; + const favicon = tabId + ? readThreadPreviewState(threadRef).desktopByTabId[tabId]?.favicon + : null; + if ( + favicon && + favicon.dataUrl.length <= 4096 && + new URL(favicon.pageUrl).origin === url.origin + ) { + toolIcon = { ...toolIcon, faviconUrl: favicon.dataUrl }; + } + } + } + } catch { + // Icon lookup must not turn a successful browser action into a failure. + } finally { + clearTimeout(iconTimeout); + } + return { result, ...(toolIcon ? { toolIcon } : {}) }; } catch (cause) { throw PreviewAutomationOperationError.fromCause({ requestId: request.requestId, diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index af3a95c32c78..05569bbf5c9f 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -17,6 +17,7 @@ import { import { createPreviewAutomationRequestConsumerAtom, serializePreviewAutomationError, + type PreviewAutomationHandledResult, } from "./previewAutomationRequestConsumer"; const environmentId = EnvironmentId.make("environment-1"); @@ -47,7 +48,9 @@ const requestEvent = ( request: request(requestId, overrides), }); -const consumerState = (handleRequest: (request: PreviewAutomationRequest) => Promise) => ({ +const consumerState = ( + handleRequest: (request: PreviewAutomationRequest) => Promise, +) => ({ connectionAtom: Atom.make(null), requestHandlerAtom: Atom.make({ handle: handleRequest }), }); @@ -60,7 +63,7 @@ describe("previewAutomationRequestConsumer", () => { connectionId, }), ); - const handleRequest = vi.fn(async () => undefined); + const handleRequest = vi.fn(async () => ({ result: undefined })); const respond = vi.fn(async () => undefined); const state = consumerState(handleRequest); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ @@ -90,7 +93,7 @@ describe("previewAutomationRequestConsumer", () => { connectionId: "connection-2", }), ); - const handleRequest = vi.fn(async () => undefined); + const handleRequest = vi.fn(async () => ({ result: undefined })); const respond = vi.fn(async () => undefined); const state = consumerState(handleRequest); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ @@ -121,7 +124,8 @@ describe("previewAutomationRequestConsumer", () => { AsyncResult.initial(false), ); const handleRequest = vi.fn(async (value: PreviewAutomationRequest) => ({ - requestId: value.requestId, + result: { requestId: value.requestId }, + toolIcon: { _tag: "website" as const, pageUrl: `https://${value.requestId}.example` }, })); const responses: PreviewAutomationResponse[] = []; const respond = vi.fn(async (response: PreviewAutomationResponse) => { @@ -149,6 +153,10 @@ describe("previewAutomationRequestConsumer", () => { "request-2", ]); expect(responses.map((response) => response.requestId)).toEqual(["request-1", "request-2"]); + expect(responses.map(({ toolIcon }) => toolIcon)).toEqual([ + { _tag: "website", pageUrl: "https://request-1.example" }, + { _tag: "website", pageUrl: "https://request-2.example" }, + ]); registry.dispose(); }); @@ -156,8 +164,8 @@ describe("previewAutomationRequestConsumer", () => { const requestsAtom = Atom.make>( AsyncResult.initial(false), ); - const firstHandler = vi.fn(async () => "first"); - const secondHandler = vi.fn(async () => "second"); + const firstHandler = vi.fn(async () => ({ result: "first" })); + const secondHandler = vi.fn(async () => ({ result: "second" })); const respond = vi.fn(async (_response: PreviewAutomationResponse) => undefined); const state = consumerState(firstHandler); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ @@ -189,7 +197,7 @@ describe("previewAutomationRequestConsumer", () => { AsyncResult.success(requestEvent("request-ready")), ); const respond = vi.fn(async (_response: PreviewAutomationResponse) => undefined); - const state = consumerState(async () => undefined); + const state = consumerState(async () => ({ result: undefined })); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ requestsAtom, clientId, diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index 89a9387e4af6..8c7001932ad3 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -12,6 +12,11 @@ import { serializePreviewAutomationHostError, } from "./previewAutomationErrors"; +export interface PreviewAutomationHandledResult { + readonly result: unknown; + readonly toolIcon?: PreviewAutomationResponse["toolIcon"]; +} + type AutomationStreamResult = AsyncResult.AsyncResult; export function serializePreviewAutomationError( @@ -29,7 +34,7 @@ export function createPreviewAutomationRequestConsumerAtom(options: { readonly connectionAtom: Atom.Writable; readonly environmentId: PreviewAutomationHost["environmentId"]; readonly requestHandlerAtom: Atom.Atom<{ - readonly handle: (request: PreviewAutomationRequest) => Promise; + readonly handle: (request: PreviewAutomationRequest) => Promise; }>; readonly respond: (response: PreviewAutomationResponse) => Promise; readonly label: string; @@ -67,13 +72,14 @@ export function createPreviewAutomationRequestConsumerAtom(options: { .once(options.requestHandlerAtom) .handle(request) .then( - (value) => + ({ result, toolIcon }) => options.respond({ clientId: options.clientId, connectionId: event.connectionId, requestId: request.requestId, ok: true, - ...(value === undefined ? {} : { result: value }), + ...(result === undefined ? {} : { result }), + ...(toolIcon ? { toolIcon } : {}), }), (error) => options.respond({ diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 59387fdacfd9..ec0200e6e21e 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -10,6 +10,7 @@ import { PreviewViewportSize, } from "./preview.ts"; import { ProviderInstanceId } from "./providerInstance.ts"; +import { ToolActivityIcon } from "./providerRuntime.ts"; const BoundedUrl = Schema.String.check(Schema.isTrimmed()) .check(Schema.isNonEmpty()) @@ -623,6 +624,7 @@ export const PreviewAutomationResponse = Schema.Struct({ requestId: TrimmedNonEmptyString, ok: Schema.Boolean, result: Schema.optional(Schema.Unknown), + toolIcon: Schema.optional(ToolActivityIcon), error: Schema.optional( Schema.Struct({ _tag: TrimmedNonEmptyString, From d79f180405963b3cd39c924bc0bd1094918eb7c7 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 10 Sep 2026 02:39:22 +0000 Subject: [PATCH 2/6] test(preview): allow favicon metadata in action result schemas --- apps/server/src/mcp/toolkits/preview/tools.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/mcp/toolkits/preview/tools.test.ts b/apps/server/src/mcp/toolkits/preview/tools.test.ts index 2cdc67ad7d92..3deef11f671e 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.test.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.test.ts @@ -67,6 +67,7 @@ it("exports exact object result schemas for preview actions", () => { for (const name of actionNames) { expect(Tool.getJsonSchemaFromSchema(PreviewToolkit.tools[name].successSchema)).toEqual({ type: "object", + properties: { toolIcon: expect.any(Object) }, additionalProperties: false, description: "The preview action completed successfully.", }); From b79dd1a3ecfa1caad757303de507727cd68f91b0 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 10 Sep 2026 05:24:31 +0000 Subject: [PATCH 3/6] refactor(preview): reuse browser status for activity favicons --- apps/server/src/mcp/McpHttpServer.test.ts | 5 +- .../server/src/mcp/PreviewAutomationBroker.ts | 42 +--- .../src/mcp/toolkits/preview/handlers.ts | 25 ++- apps/server/src/mcp/toolkits/preview/tools.ts | 8 +- .../ActivityPayloadProjection.test.ts | 201 +++--------------- .../ActivityPayloadProjection.ts | 112 ++++------ .../preview/PreviewAutomationHosts.tsx | 65 +----- .../previewAutomationRequestConsumer.test.ts | 22 +- .../previewAutomationRequestConsumer.ts | 12 +- packages/contracts/src/previewAutomation.ts | 2 - 10 files changed, 121 insertions(+), 373 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index c0ae2cd0571f..ee2d7543bc29 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -618,7 +618,6 @@ it.effect("registers annotated tools and preserves authenticated request context const toolIcon = { _tag: "website" as const, pageUrl: "http://example.test/", - faviconUrl: "data:image/png;base64,aWNvbg==", }; const routedRequests: Array<{ readonly operation: string; @@ -636,7 +635,6 @@ it.effect("registers annotated tools and preserves authenticated request context connectionId: event.connectionId, requestId: event.request.requestId, ok: true, - toolIcon, result: event.request.operation === "snapshot" ? snapshotResult @@ -690,7 +688,6 @@ it.effect("registers annotated tools and preserves authenticated request context expect(status.structuredContent).toMatchObject({ available: true, tabId, - toolIcon, }); const malformed = yield* server @@ -711,7 +708,6 @@ it.effect("registers annotated tools and preserves authenticated request context expect(snapshot.isError).toBe(false); expect(snapshot.content.some((content) => content.type === "image")).toBe(true); expect(snapshot.structuredContent).toMatchObject({ - toolIcon, screenshot: { mimeType: "image/png", width: 10, height: 5 }, }); expect(routedRequests.find(({ operation }) => operation === "snapshot")?.tabId).toBe( @@ -752,6 +748,7 @@ it.effect("registers annotated tools and preserves authenticated request context ); expect(result.isError).toBe(false); expect(result.structuredContent).toEqual({ toolIcon }); + expect(routedRequests.at(-1)?.operation).toBe("status"); const text = result.content[0]; expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual({ toolIcon }); } diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index 9c1a146c61a8..d8f17973c218 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -46,11 +46,6 @@ export interface PreviewAutomationInvokeInput { readonly timeoutMs?: number; } -export interface PreviewAutomationResult { - readonly result: A; - readonly toolIcon?: PreviewAutomationResponse["toolIcon"]; -} - export class PreviewAutomationBroker extends Context.Service< PreviewAutomationBroker, { @@ -64,9 +59,6 @@ export class PreviewAutomationBroker extends Context.Service< readonly invoke: ( request: PreviewAutomationInvokeInput, ) => Effect.Effect; - readonly invokeWithPresentation: ( - request: PreviewAutomationInvokeInput, - ) => Effect.Effect, PreviewAutomationError>; } >()("t3/mcp/PreviewAutomationBroker") {} @@ -82,7 +74,7 @@ interface ClientConnection { interface PendingRequest { readonly queue: ClientConnection["queue"]; - readonly deferred: Deferred.Deferred, PreviewAutomationError>; + readonly deferred: Deferred.Deferred; readonly context: PreviewAutomationRequestErrorContext; } @@ -444,10 +436,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { }); if (!pending) return; if (response.ok) { - yield* Deferred.succeed(pending.deferred, { - result: response.result, - ...(response.toolIcon ? { toolIcon: response.toolIcon } : {}), - }); + yield* Deferred.succeed(pending.deferred, response.result); } else { yield* Deferred.fail( pending.deferred, @@ -458,16 +447,11 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { } }); - const invokeWithPresentation = Effect.fn("PreviewAutomationBroker.invoke")(function* < - A = unknown, - >( + const invoke = Effect.fn("PreviewAutomationBroker.invoke")(function* ( input: Parameters[0], - ): Effect.fn.Return, PreviewAutomationError> { + ): Effect.fn.Return { const timeoutMs = input.timeoutMs ?? 15_000; - const deferred = yield* Deferred.make< - PreviewAutomationResult, - PreviewAutomationError - >(); + const deferred = yield* Deferred.make(); const route = yield* SynchronizedRef.modify(state, (current) => { const assignments = new Map( Array.from(current.assignments).filter(([, assignment]) => { @@ -580,18 +564,18 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { if (!offered) { const completion = yield* Deferred.poll(deferred); if (Option.isSome(completion)) { - return (yield* completion.value) as PreviewAutomationResult; + return (yield* completion.value) as A; } return yield* new PreviewAutomationRequestQueueClosedError(requestContext); } const result = yield* Deferred.await(deferred).pipe(Effect.timeoutOption(timeoutMs)); return yield* Option.match(result, { onNone: () => Effect.fail(new PreviewAutomationTimeoutError(requestContext)), - onSome: (value) => Effect.succeed(value as PreviewAutomationResult), + onSome: (value) => Effect.succeed(value as A), }); }); const result = yield* awaitResponse().pipe(Effect.ensuring(removePending)); - const responseTabId = readResultTabId(result.result); + const responseTabId = readResultTabId(result); const resultTabId = responseTabId === undefined ? input.tabId : responseTabId; if (resultTabId === undefined) return result; const assignmentKey = hostAssignmentKey(input.scope); @@ -621,15 +605,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { return result; }); - const invoke = (input: PreviewAutomationInvokeInput) => - invokeWithPresentation(input).pipe(Effect.map(({ result }) => result)); - return PreviewAutomationBroker.of({ - connect, - focusHost, - respond, - invoke, - invokeWithPresentation, - }); + return PreviewAutomationBroker.of({ connect, focusHost, respond, invoke }); }).pipe(Effect.withSpan("PreviewAutomationBroker.make")); export const layer = Layer.effect(PreviewAutomationBroker, make); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 8f008f4566ff..5954a5a361ec 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -7,6 +7,7 @@ import { PreviewAutomationRecordingTransferError, PreviewAutomationRecordingDesktopUpdateRequiredError, PreviewAutomationRecordingArtifact, + type ToolActivityIcon, type ThreadId, type PreviewAutomationOperation, type PreviewAutomationOpenInput, @@ -55,19 +56,39 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( timeoutMs?: number, tabId?: PreviewTabId, ): Effect.fn.Return< - PreviewAutomationBroker.PreviewAutomationResult, + { result: A; toolIcon?: ToolActivityIcon }, import("@t3tools/contracts").PreviewAutomationError, McpInvocationContext.McpInvocationContext | PreviewAutomationBroker.PreviewAutomationBroker > { const scope = yield* McpInvocationContext.requireMcpCapability("preview"); const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; - return yield* broker.invokeWithPresentation({ + const result = yield* broker.invoke({ scope, operation, input, ...(timeoutMs === undefined ? {} : { timeoutMs }), ...(tabId === undefined ? {} : { tabId }), }); + if (["status", "open", "navigate", "snapshot"].includes(operation)) return { result }; + const statusTabId = + (operation !== "evaluate" && typeof result === "object" && result !== null + ? (result as { tabId?: PreviewTabId }).tabId + : undefined) ?? tabId; + const page = yield* broker + .invoke({ + scope, + operation: "status", + input: {}, + timeoutMs: 500, + ...(statusTabId === undefined ? {} : { tabId: statusTabId }), + }) + .pipe(Effect.catch(() => Effect.succeed(null))); + return { + result, + ...(page?.url && /^https?:\/\//i.test(page.url) && page.url.length <= 4096 + ? { toolIcon: { _tag: "website" as const, pageUrl: page.url } } + : {}), + }; }); const invokeTargeted = ( diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 38786b34686b..3f80e84e9a59 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -54,7 +54,7 @@ const PreviewStatusTool = Tool.make("preview_status", { description: "Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.", parameters: PreviewAutomationTabTargetInput, - success: Schema.Struct({ ...PreviewAutomationStatus.fields, ...presentationFields }), + success: PreviewAutomationStatus, failure: PreviewAutomationError, dependencies, }) @@ -68,7 +68,7 @@ const PreviewOpenTool = browserTool( description: "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", parameters: PreviewAutomationOpenInput, - success: Schema.Struct({ ...PreviewAutomationStatus.fields, ...presentationFields }), + success: PreviewAutomationStatus, failure: PreviewAutomationError, dependencies, }) @@ -81,7 +81,7 @@ const PreviewNavigateTool = safeBrowserTool( description: "Navigate a collaborative browser tab. Pass tabId to target a specific tab, plus {url:'https://t3.chat'} for a website or {target:{kind:'environment-port',port:5173}} for a dev server. Exactly one of url or target is required.", parameters: PreviewAutomationNavigateInput, - success: Schema.Struct({ ...PreviewAutomationStatus.fields, ...presentationFields }), + success: PreviewAutomationStatus, failure: PreviewAutomationError, dependencies, }).annotate(Tool.Title, "Navigate browser preview"), @@ -135,7 +135,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool( }), ), }), - success: Schema.Struct({ ...PreviewAutomationSnapshot.fields, ...presentationFields }), + success: PreviewAutomationSnapshot, failure: PreviewAutomationError, dependencies, }).annotate(Tool.Title, "Inspect browser page"), diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts index 67885bc3ec6f..ad2141c74f87 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts @@ -249,185 +249,50 @@ describe("projectActivityPayload", () => { expect(JSON.stringify(projected.payload).length).toBeLessThan(500); }); - it("preserves preview website metadata through result slimming and repeated projection", () => { - const toolIcon = { - _tag: "website", - pageUrl: "https://example.com/page", - faviconUrl: "https://example.com/icon.png", - }; - const projected = projectActivityPayload( - activity({ - itemType: "mcp_tool_call", - data: { - item: { - server: "t3-code", - tool: "preview_click", - result: { - structuredContent: { toolIcon }, - content: [ - { - type: "text", - text: JSON.stringify({ - toolIcon: { - _tag: "website", - pageUrl: "https://other.example/", - }, - }), - }, - ], - }, - }, - }, - }), - ); - expect(projected.payload).toMatchObject({ toolSurface: "browser", toolIcon }); - expect(projectActivityPayload(projected).payload).toMatchObject({ - toolSurface: "browser", - toolIcon, - }); - expect(projected.payload).not.toMatchObject({ - data: { item: { result: { structuredContent: expect.anything() } } }, - }); - }); - it.each([ { - toolName: "mcp__t3-code__preview_snapshot", - result: { - type: "tool_result", - content: [ - { - type: "text", - text: JSON.stringify({ - toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, - }), - }, - ], + item: { + server: "t3-code", + tool: "preview_open", + result: { structuredContent: { url: "https://example.com/" } }, }, }, { - tool: "t3-code_preview_snapshot", - state: { - status: "completed", - output: JSON.stringify({ - toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, - }), - }, + toolName: "mcp__t3-code__preview_navigate", + result: { content: '{"url":"https://example.com/"}' }, }, - ])("reads preview metadata from provider-preserved JSON output", (data) => { - const projected = projectActivityPayload(activity({ itemType: "dynamic_tool_call", data })); - expect(projected.payload).toMatchObject({ - toolSurface: "browser", - toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, - }); - }); - - it.each(["t3-code", "t3_code", "t3code"])("recognizes the %s preview server alias", (server) => { - const result = { content: JSON.stringify({ url: "https://example.com/" }) }; - for (const data of [ - { item: { server, tool: "preview_open", result } }, - { toolName: `mcp__${server}__preview_open`, result }, - { tool: `${server}_preview_open`, state: { output: result.content } }, - ]) { - const projected = projectActivityPayload(activity({ itemType: "mcp_tool_call", data })); - expect(projected.payload).toMatchObject({ - toolSurface: "browser", - toolIcon: { _tag: "website", pageUrl: "https://example.com/" }, - }); - } - }); - - it("infers legacy preview URLs and prefers the returned page after redirects", () => { - const projected = projectActivityPayload( - activity({ - itemType: "mcp_tool_call", - data: { - toolName: "mcp__t3-code__preview_navigate", - input: { url: "https://before.example/" }, - result: { content: JSON.stringify({ url: "https://after.example/" }) }, - }, - }), - ); - expect(projected.payload).toMatchObject({ - toolIcon: { _tag: "website", pageUrl: "https://after.example/" }, - }); - const started = projectActivityPayload( - activity({ - itemType: "mcp_tool_call", - data: { - toolName: "mcp__t3-code__preview_navigate", - input: { target: { kind: "url", url: "https://before.example/" } }, - }, - }), - ); - expect(started.payload).toMatchObject({ - toolIcon: { _tag: "website", pageUrl: "https://before.example/" }, - }); - }); - - it.each([ - { toolName: "mcp__other__preview_click" }, - { item: { server: "other", tool: "preview_click" } }, - { toolName: "preview_click" }, - ])("ignores preview metadata from unrelated tools", (identity) => { - const result = { - structuredContent: { toolIcon: { _tag: "website", pageUrl: "https://example.com/" } }, - }; - const projected = projectActivityPayload( - activity({ - itemType: "mcp_tool_call", - data: { - ...identity, - result, - ...("item" in identity ? { item: { ...identity.item, result } } : {}), - }, - }), - ); - expect(projected.payload).not.toHaveProperty("toolIcon"); - expect(projected.payload).not.toHaveProperty("toolSurface"); + { tool: "t3-code_preview_status", state: { output: '{"url":"https://example.com/"}' } }, + { + toolName: "mcp__t3_code__preview_snapshot", + result: { content: [{ type: "text", text: '{"url":"https://example.com/"}' }] }, + }, + { + toolName: "mcp__t3-code__preview_click", + result: { content: '{"toolIcon":{"_tag":"website","pageUrl":"https://example.com/"}}' }, + }, + ])("preserves the preview page favicon through result slimming", (data) => { + const projected = projectActivityPayload(activity({ itemType: "mcp_tool_call", data })); + const icon = { _tag: "website", pageUrl: "https://example.com/" }; + expect(projected.payload).toMatchObject({ toolIcon: icon }); + expect(projectActivityPayload(projected).payload).toMatchObject({ toolIcon: icon }); }); it.each([ - { result: { isError: true, content: JSON.stringify({ url: "https://stale.example/" }) } }, - { result: { is_error: true, content: "navigation failed" } }, - { result: { structuredContent: { toolIcon: { _tag: "website", pageUrl: 42 } } } }, + { toolName: "mcp__other__preview_open", result: { content: '{"url":"https://example.com/"}' } }, { - result: { - structuredContent: { - toolIcon: { _tag: "themed-logo", logoUrl: "https://example.com/icon.png" }, - }, - }, + toolName: "mcp__t3-code__preview_evaluate", + result: { content: '{"url":"https://example.com/"}' }, }, - { result: { content: "malformed JSON" } }, - { result: { content: JSON.stringify({ url: "about:blank" }) } }, - ])("does not invent a page icon for failures or invalid results", (output) => { - const projected = projectActivityPayload( - activity({ - itemType: "mcp_tool_call", - data: { - toolName: "mcp__t3-code__preview_navigate", - input: { url: "https://attempted.example/" }, - ...output, - }, - }), - ); - expect(projected.payload).toMatchObject({ toolSurface: "browser" }); - expect(projected.payload).not.toHaveProperty("toolIcon"); - }); - - it("preserves explicit activity icons", () => { - const toolIcon = { _tag: "website", pageUrl: "https://explicit.example/" }; - const projected = projectActivityPayload( - activity({ - itemType: "mcp_tool_call", - toolIcon, - data: { - toolName: "mcp__t3-code__preview_status", - result: { content: JSON.stringify({ url: "https://inferred.example/" }) }, - }, - }), - ); - expect(projected.payload).toMatchObject({ toolIcon }); + { + toolName: "mcp__t3-code__preview_open", + result: { isError: true, content: '{"url":"https://example.com/"}' }, + }, + { toolName: "mcp__t3-code__preview_open", result: { content: "malformed JSON" } }, + { toolName: "mcp__t3-code__preview_open", result: { content: '{"url":"about:blank"}' } }, + ])("keeps the fallback for unrelated tools, failed navigation, and missing page URLs", (data) => { + expect( + projectActivityPayload(activity({ itemType: "mcp_tool_call", data })).payload, + ).not.toHaveProperty("toolIcon"); }); it("passes task lifecycle payloads (no data field) through untouched", () => { diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.ts b/apps/server/src/orchestration/ActivityPayloadProjection.ts index 172d0d4bb31e..64b0f2e5f2f5 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.ts @@ -1,11 +1,9 @@ -import { - ToolActivityIcon, - type OrchestrationEvent, - type OrchestrationThreadActivity, - type OrchestrationThreadDetailSnapshot, +import type { + OrchestrationEvent, + OrchestrationThreadActivity, + OrchestrationThreadDetailSnapshot, } from "@t3tools/contracts"; import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; -import * as Schema from "effect/Schema"; function asRecord(value: unknown): Record | null { return value !== null && typeof value === "object" && !Array.isArray(value) @@ -242,86 +240,52 @@ function summarizeMcpResult(result: unknown): Record | undefine return summary ? { content: summary } : undefined; } -const decodeToolActivityIcon = Schema.decodeUnknownOption(ToolActivityIcon); -const PREVIEW_TOOL_NAME = - /^preview_(?:status|open|navigate|resize|set_appearance|snapshot|click|type|press|scroll|evaluate|wait_for|recording_start|recording_stop)$/u; - -function parsePreviewResult(value: unknown): Record | null { - if (typeof value !== "string") return asRecord(value); - // Snapshot text can be large; never parse arbitrary unbounded tool output. - if (value.length > 2 * 1024 * 1024) return null; - try { - return asRecord(JSON.parse(value)); - } catch { - return null; - } -} - +/** Reuse the page URL already returned by preview tools before slimming their output. */ function projectPreviewToolMetadata(data: Record, status: unknown) { const item = asRecord(data.item); - const qualifiedName = asTrimmedString(data.toolName) ?? asTrimmedString(data.tool); - const tool = item - ? /^(?:t3-code|t3_code|t3code)$/u.test(asTrimmedString(item.server) ?? "") - ? asTrimmedString(item.tool) - : null - : qualifiedName?.replace( - /^(?:mcp__(?:t3-code|t3_code|t3code)__|(?:t3-code|t3_code|t3code)_)/u, - "", - ); - if (!tool || !PREVIEW_TOOL_NAME.test(tool) || (!item && tool === qualifiedName)) { + const name = item ? `mcp__${item.server}__${item.tool}` : (data.toolName ?? data.tool); + if ( + typeof name !== "string" || + !/^(?:mcp__)?(?:t3-code|t3_code|t3code)_{1,2}preview_(?:open|navigate|status|snapshot|click|type|press|scroll|resize|set_appearance|evaluate|wait_for|recording_start|recording_stop)$/.test( + name, + ) + ) return {}; - } - const state = asRecord(data.state); const result = item?.result ?? data.result ?? state?.output; - const resultRecord = asRecord(result); - const failed = + const record = asRecord(result); + if ( status === "failed" || status === "declined" || state?.status === "error" || item?.error != null || - resultRecord?.isError === true || - resultRecord?.is_error === true; - if (failed) return { toolSurface: "browser" }; - - // Prefer MCP structured content, then the JSON text preserved by Claude/OpenCode. - const structuredContent = asRecord(resultRecord?.structuredContent); - const structuredIcon = decodeToolActivityIcon(structuredContent?.toolIcon); - if (structuredIcon._tag === "Some" && structuredIcon.value._tag === "website") { - return { toolSurface: "browser", toolIcon: structuredIcon.value }; - } - const candidates = [structuredContent, parsePreviewResult(result)]; - if (typeof resultRecord?.content === "string") { - candidates.push(parsePreviewResult(resultRecord.content)); - } else if (Array.isArray(resultRecord?.content)) { - for (const entry of resultRecord.content.slice(0, 32)) { - const block = asRecord(entry); - if (block?.type === "text") candidates.push(parsePreviewResult(block.text)); - } - } - for (const candidate of candidates) { - const decoded = decodeToolActivityIcon(candidate?.toolIcon); - if (decoded._tag === "Some" && decoded.value._tag === "website") { - return { toolSurface: "browser", toolIcon: decoded.value }; - } - } + record?.isError === true || + record?.is_error === true + ) + return {}; - // Older preview hosts return the page URL without icon metadata. - if (/^preview_(?:status|open|navigate|snapshot)$/u.test(tool)) { - const input = asRecord(item?.arguments ?? data.input ?? state?.input); - const pageUrl = - candidates.find((candidate) => typeof candidate?.url === "string")?.url ?? - (result == null ? (input?.url ?? asRecord(input?.target)?.url) : undefined); - const decoded = decodeToolActivityIcon({ _tag: "website", pageUrl }); - if ( - decoded._tag === "Some" && - decoded.value._tag === "website" && - /^https?:\/\//iu.test(decoded.value.pageUrl) - ) { - return { toolSurface: "browser", toolIcon: decoded.value }; + let page = asRecord(record?.structuredContent); + if (!page) { + const text = extractMcpResultText(result); + if (!text || text.length > 2 * 1024 * 1024) return {}; + try { + page = asRecord(JSON.parse(text)); + } catch { + return {}; } } - return { toolSurface: "browser" }; + const rawUrl = asTrimmedString( + asRecord(page?.toolIcon)?.pageUrl ?? + (/preview_(?:open|navigate|status|snapshot)$/.test(name) ? page?.url : undefined), + ); + if (!rawUrl || rawUrl.length > 4096) return {}; + try { + const url = new URL(rawUrl); + if (url.protocol !== "http:" && url.protocol !== "https:") return {}; + return { toolIcon: { _tag: "website", pageUrl: url.href } }; + } catch { + return {}; + } } /** diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 308ac347850b..d1fc12821730 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -14,7 +14,6 @@ import { type PreviewAutomationSetColorSchemeResult, type PreviewAutomationHost as PreviewAutomationHostState, type PreviewAutomationRequest, - type PreviewAutomationResponse, type PreviewAutomationStatus, type PreviewRenderedViewportSize, type PreviewViewportSetting, @@ -76,10 +75,7 @@ import { assertPreviewRuntimeCurrent, waitForNavigationReadiness, } from "./previewNavigationReadiness"; -import { - createPreviewAutomationRequestConsumerAtom, - type PreviewAutomationHandledResult, -} from "./previewAutomationRequestConsumer"; +import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationRequestConsumer"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { needsPreviewAutomationSessionSync, @@ -328,7 +324,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const presentationSuppressedRuntimeTabsRef = useRef(new Map>()); const handleRequest = useCallback( - async (request: PreviewAutomationRequest): Promise => { + async (request: PreviewAutomationRequest): Promise => { // Session sync and tab creation consume the same budget as overlay registration. const hostDeadlineMs = Date.now() + resolveHostWaitBudgetMs(request.timeoutMs); const threadRef: ScopedThreadRef = { @@ -337,7 +333,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; let tabId = request.tabId ?? null; const browserActivity = { release: null as (() => void) | null }; - const execute = async () => { + try { let state = readThreadPreviewState(threadRef); const needsSessionSync = needsPreviewAutomationSessionSync(state, request.tabId); if (needsSessionSync) { @@ -750,61 +746,6 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; } } - }; - try { - const result: unknown = await execute(); - let toolIcon: PreviewAutomationResponse["toolIcon"]; - let iconTimeout: ReturnType | undefined; - try { - // Read the resolved target, including tabs created or selected by this operation. - const resultHasPageUrl = - ["status", "open", "navigate", "snapshot"].includes(request.operation) && - typeof result === "object" && - result !== null && - "url" in result; - const pageUrl = resultHasPageUrl - ? result.url - : tabId && previewBridge && Date.now() < hostDeadlineMs - ? ( - await Promise.race([ - previewBridge.automation.status( - previewRuntimeTabId( - threadRef, - readThreadPreviewState(threadRef).serverEpoch, - tabId, - ), - ), - new Promise((resolve) => { - iconTimeout = setTimeout( - () => resolve(null), - Math.min(300, Math.max(0, hostDeadlineMs - Date.now())), - ); - }), - ]) - )?.url - : null; - if (typeof pageUrl === "string" && pageUrl.length <= 4096) { - const url = new URL(pageUrl); - if (url.protocol === "http:" || url.protocol === "https:") { - toolIcon = { _tag: "website", pageUrl }; - const favicon = tabId - ? readThreadPreviewState(threadRef).desktopByTabId[tabId]?.favicon - : null; - if ( - favicon && - favicon.dataUrl.length <= 4096 && - new URL(favicon.pageUrl).origin === url.origin - ) { - toolIcon = { ...toolIcon, faviconUrl: favicon.dataUrl }; - } - } - } - } catch { - // Icon lookup must not turn a successful browser action into a failure. - } finally { - clearTimeout(iconTimeout); - } - return { result, ...(toolIcon ? { toolIcon } : {}) }; } catch (cause) { throw PreviewAutomationOperationError.fromCause({ requestId: request.requestId, diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 05569bbf5c9f..af3a95c32c78 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -17,7 +17,6 @@ import { import { createPreviewAutomationRequestConsumerAtom, serializePreviewAutomationError, - type PreviewAutomationHandledResult, } from "./previewAutomationRequestConsumer"; const environmentId = EnvironmentId.make("environment-1"); @@ -48,9 +47,7 @@ const requestEvent = ( request: request(requestId, overrides), }); -const consumerState = ( - handleRequest: (request: PreviewAutomationRequest) => Promise, -) => ({ +const consumerState = (handleRequest: (request: PreviewAutomationRequest) => Promise) => ({ connectionAtom: Atom.make(null), requestHandlerAtom: Atom.make({ handle: handleRequest }), }); @@ -63,7 +60,7 @@ describe("previewAutomationRequestConsumer", () => { connectionId, }), ); - const handleRequest = vi.fn(async () => ({ result: undefined })); + const handleRequest = vi.fn(async () => undefined); const respond = vi.fn(async () => undefined); const state = consumerState(handleRequest); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ @@ -93,7 +90,7 @@ describe("previewAutomationRequestConsumer", () => { connectionId: "connection-2", }), ); - const handleRequest = vi.fn(async () => ({ result: undefined })); + const handleRequest = vi.fn(async () => undefined); const respond = vi.fn(async () => undefined); const state = consumerState(handleRequest); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ @@ -124,8 +121,7 @@ describe("previewAutomationRequestConsumer", () => { AsyncResult.initial(false), ); const handleRequest = vi.fn(async (value: PreviewAutomationRequest) => ({ - result: { requestId: value.requestId }, - toolIcon: { _tag: "website" as const, pageUrl: `https://${value.requestId}.example` }, + requestId: value.requestId, })); const responses: PreviewAutomationResponse[] = []; const respond = vi.fn(async (response: PreviewAutomationResponse) => { @@ -153,10 +149,6 @@ describe("previewAutomationRequestConsumer", () => { "request-2", ]); expect(responses.map((response) => response.requestId)).toEqual(["request-1", "request-2"]); - expect(responses.map(({ toolIcon }) => toolIcon)).toEqual([ - { _tag: "website", pageUrl: "https://request-1.example" }, - { _tag: "website", pageUrl: "https://request-2.example" }, - ]); registry.dispose(); }); @@ -164,8 +156,8 @@ describe("previewAutomationRequestConsumer", () => { const requestsAtom = Atom.make>( AsyncResult.initial(false), ); - const firstHandler = vi.fn(async () => ({ result: "first" })); - const secondHandler = vi.fn(async () => ({ result: "second" })); + const firstHandler = vi.fn(async () => "first"); + const secondHandler = vi.fn(async () => "second"); const respond = vi.fn(async (_response: PreviewAutomationResponse) => undefined); const state = consumerState(firstHandler); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ @@ -197,7 +189,7 @@ describe("previewAutomationRequestConsumer", () => { AsyncResult.success(requestEvent("request-ready")), ); const respond = vi.fn(async (_response: PreviewAutomationResponse) => undefined); - const state = consumerState(async () => ({ result: undefined })); + const state = consumerState(async () => undefined); const consumerAtom = createPreviewAutomationRequestConsumerAtom({ requestsAtom, clientId, diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index 8c7001932ad3..89a9387e4af6 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -12,11 +12,6 @@ import { serializePreviewAutomationHostError, } from "./previewAutomationErrors"; -export interface PreviewAutomationHandledResult { - readonly result: unknown; - readonly toolIcon?: PreviewAutomationResponse["toolIcon"]; -} - type AutomationStreamResult = AsyncResult.AsyncResult; export function serializePreviewAutomationError( @@ -34,7 +29,7 @@ export function createPreviewAutomationRequestConsumerAtom(options: { readonly connectionAtom: Atom.Writable; readonly environmentId: PreviewAutomationHost["environmentId"]; readonly requestHandlerAtom: Atom.Atom<{ - readonly handle: (request: PreviewAutomationRequest) => Promise; + readonly handle: (request: PreviewAutomationRequest) => Promise; }>; readonly respond: (response: PreviewAutomationResponse) => Promise; readonly label: string; @@ -72,14 +67,13 @@ export function createPreviewAutomationRequestConsumerAtom(options: { .once(options.requestHandlerAtom) .handle(request) .then( - ({ result, toolIcon }) => + (value) => options.respond({ clientId: options.clientId, connectionId: event.connectionId, requestId: request.requestId, ok: true, - ...(result === undefined ? {} : { result }), - ...(toolIcon ? { toolIcon } : {}), + ...(value === undefined ? {} : { result: value }), }), (error) => options.respond({ diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index ec0200e6e21e..59387fdacfd9 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -10,7 +10,6 @@ import { PreviewViewportSize, } from "./preview.ts"; import { ProviderInstanceId } from "./providerInstance.ts"; -import { ToolActivityIcon } from "./providerRuntime.ts"; const BoundedUrl = Schema.String.check(Schema.isTrimmed()) .check(Schema.isNonEmpty()) @@ -624,7 +623,6 @@ export const PreviewAutomationResponse = Schema.Struct({ requestId: TrimmedNonEmptyString, ok: Schema.Boolean, result: Schema.optional(Schema.Unknown), - toolIcon: Schema.optional(ToolActivityIcon), error: Schema.optional( Schema.Struct({ _tag: TrimmedNonEmptyString, From ee14be4d78995ed30f86154684df52aa116e1aaa Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 10 Sep 2026 05:27:43 +0000 Subject: [PATCH 4/6] fix(preview): preserve tab routing during favicon lookup --- apps/server/src/mcp/PreviewAutomationBroker.test.ts | 7 +++++++ apps/server/src/mcp/PreviewAutomationBroker.ts | 3 +++ apps/server/src/mcp/toolkits/preview/handlers.ts | 1 + .../src/orchestration/ActivityPayloadProjection.test.ts | 7 ++++++- apps/server/src/orchestration/ActivityPayloadProjection.ts | 3 ++- 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/apps/server/src/mcp/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/PreviewAutomationBroker.test.ts index 42f849f5edf3..6f75b8a22200 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.test.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.test.ts @@ -166,6 +166,13 @@ it.effect("does not let an older response replace a newer explicit tab target", .pipe(Effect.forkScoped); yield* Fiber.join(newer); yield* Fiber.join(older); + yield* broker.invoke({ + scope, + operation: "status", + input: {}, + tabId: olderTabId, + updateCurrentTab: false, + }); yield* broker.invoke({ scope, operation: "snapshot", input: {} }); expect(routedRequests.at(-1)?.tabId).toBe(newerTabId); diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index d8f17973c218..e7b69baa9820 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -44,6 +44,8 @@ export interface PreviewAutomationInvokeInput { readonly input: unknown; readonly tabId?: PreviewTabId; readonly timeoutMs?: number; + /** Background metadata reads must not change the agent's current tab. */ + readonly updateCurrentTab?: boolean; } export class PreviewAutomationBroker extends Context.Service< @@ -575,6 +577,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { }); }); const result = yield* awaitResponse().pipe(Effect.ensuring(removePending)); + if (input.updateCurrentTab === false) return result; const responseTabId = readResultTabId(result); const resultTabId = responseTabId === undefined ? input.tabId : responseTabId; if (resultTabId === undefined) return result; diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 5954a5a361ec..86c670a1edfd 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -80,6 +80,7 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( operation: "status", input: {}, timeoutMs: 500, + updateCurrentTab: false, ...(statusTabId === undefined ? {} : { tabId: statusTabId }), }) .pipe(Effect.catch(() => Effect.succeed(null))); diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts index ad2141c74f87..3047fdfebd5b 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts @@ -264,7 +264,12 @@ describe("projectActivityPayload", () => { { tool: "t3-code_preview_status", state: { output: '{"url":"https://example.com/"}' } }, { toolName: "mcp__t3_code__preview_snapshot", - result: { content: [{ type: "text", text: '{"url":"https://example.com/"}' }] }, + result: { + content: [ + { type: "text", text: '{"url":"https://example.com/"}' }, + { type: "text", text: "Snapshot text was bounded. Omitted: accessibilityTree." }, + ], + }, }, { toolName: "mcp__t3-code__preview_click", diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.ts b/apps/server/src/orchestration/ActivityPayloadProjection.ts index 64b0f2e5f2f5..f11d2890fc86 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.ts @@ -4,6 +4,7 @@ import type { OrchestrationThreadDetailSnapshot, } from "@t3tools/contracts"; import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; +import { extractJsonObject } from "@t3tools/shared/schemaJson"; function asRecord(value: unknown): Record | null { return value !== null && typeof value === "object" && !Array.isArray(value) @@ -269,7 +270,7 @@ function projectPreviewToolMetadata(data: Record, status: unkno const text = extractMcpResultText(result); if (!text || text.length > 2 * 1024 * 1024) return {}; try { - page = asRecord(JSON.parse(text)); + page = asRecord(JSON.parse(extractJsonObject(text))); } catch { return {}; } From 408c1fd5989852bb3300cdf31a105a29a98e2272 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Fri, 11 Sep 2026 02:53:21 +0000 Subject: [PATCH 5/6] fix(preview): unwrap provider result envelopes for favicons --- .../ActivityPayloadProjection.test.ts | 9 ++++++ .../ActivityPayloadProjection.ts | 28 +++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts index baad95a6f698..e6468ff8f789 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.test.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.test.ts @@ -279,6 +279,15 @@ describe("projectActivityPayload", () => { toolName: "mcp__t3_code__preview_snapshot", result: { content: '{"url":"https://example.com/"}\n{"accessibilityTree":"truncated' }, }, + ...[false, true].map((truncated) => ({ + toolName: "mcp__t3_code__preview_snapshot", + result: { + content: JSON.stringify({ + content: [{ type: "text", text: '{"url":"https://example.com/"}' }], + structuredContent: { url: "https://example.com/", visibleText: "page" }, + }).slice(0, truncated ? -5 : undefined), + }, + })), ...[ "type", "press", diff --git a/apps/server/src/orchestration/ActivityPayloadProjection.ts b/apps/server/src/orchestration/ActivityPayloadProjection.ts index 19a01a5840f6..0525aae7b72b 100644 --- a/apps/server/src/orchestration/ActivityPayloadProjection.ts +++ b/apps/server/src/orchestration/ActivityPayloadProjection.ts @@ -266,15 +266,31 @@ function projectPreviewToolMetadata(data: Record, status: unkno ) return {}; - let page = asRecord(record?.structuredContent); - if (!page) { - const text = extractMcpResultText(result); - if (!text) return {}; + let page = record; + let output: unknown = result; + for (let depth = 0; depth < 3; depth += 1) { + if (page?.isError === true || page?.is_error === true) return {}; + const structured = asRecord(page?.structuredContent); + if (structured) { + page = structured; + break; + } + const text = extractMcpResultText(output)?.slice(0, 2 * 1024 * 1024); + if (!text) break; try { - page = asRecord(JSON.parse(extractJsonObject(text.slice(0, 2 * 1024 * 1024)))); + page = asRecord(JSON.parse(extractJsonObject(text))); } catch { - return {}; + // A truncated MCP envelope can still contain a complete first text block. + const firstBlock = /^\s*\{\s*"content"\s*:\s*\[\s*/.exec(text); + if (!firstBlock) return {}; + try { + const block = asRecord(JSON.parse(extractJsonObject(text.slice(firstBlock[0].length)))); + page = block?.type === "text" ? { content: [block] } : null; + } catch { + return {}; + } } + output = page; } const rawUrl = asTrimmedString( asRecord(page?.toolIcon)?.pageUrl ?? From 75128f2cdec8568bf8e7f99cb38273ba20292523 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Fri, 11 Sep 2026 02:58:30 +0000 Subject: [PATCH 6/6] fix(preview): pin favicon lookup to the action target --- .../src/mcp/PreviewAutomationBroker.test.ts | 109 ++++++++++-------- .../server/src/mcp/PreviewAutomationBroker.ts | 3 + .../src/mcp/toolkits/preview/handlers.ts | 6 +- 3 files changed, 69 insertions(+), 49 deletions(-) diff --git a/apps/server/src/mcp/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/PreviewAutomationBroker.test.ts index 6f75b8a22200..27557d43b701 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.test.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.test.ts @@ -127,57 +127,70 @@ it.effect("targets multiple tabs explicitly while retaining a default tab", () = ), ); -it.effect("does not let an older response replace a newer explicit tab target", () => - Effect.scoped( - Effect.gen(function* () { - const broker = yield* makeBroker; - const olderTabId = PreviewTabId.make("tab-older-request"); - const newerTabId = PreviewTabId.make("tab-newer-request"); - const releaseOlderResponse = yield* Deferred.make(); - const routedRequests: RoutedRequest[] = []; - const requests = requestsFrom(yield* broker.connect(makeHost())); - yield* Stream.runForEach(requests, (request) => { - routedRequests.push(request); - const response = Effect.gen(function* () { - if (request.tabId === olderTabId) { - yield* Deferred.await(releaseOlderResponse); - } - yield* broker.respond({ - clientId: "client-1", - connectionId: request.connectionId, - requestId: request.requestId, - ok: true, - result: { url: "http://localhost:3200" }, +it.effect.each([true, false])( + "keeps an older target stable while a newer explicit tab responds (implicit: %s)", + (implicit) => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* makeBroker; + const olderTabId = PreviewTabId.make("tab-older-request"); + const newerTabId = PreviewTabId.make("tab-newer-request"); + const releaseOlderResponse = yield* Deferred.make(); + const routedRequests: RoutedRequest[] = []; + const requests = requestsFrom(yield* broker.connect(makeHost())); + yield* Stream.runForEach(requests, (request) => { + routedRequests.push(request); + const response = Effect.gen(function* () { + if (request.tabId === olderTabId && request.operation === "snapshot") { + yield* Deferred.await(releaseOlderResponse); + } + yield* broker.respond({ + clientId: "client-1", + connectionId: request.connectionId, + requestId: request.requestId, + ok: true, + result: { url: "http://localhost:3200" }, + }); + if (request.tabId === newerTabId) { + yield* Deferred.succeed(releaseOlderResponse, undefined); + } }); - if (request.tabId === newerTabId) { - yield* Deferred.succeed(releaseOlderResponse, undefined); - } + return response.pipe(Effect.forkScoped, Effect.asVoid); + }).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + + yield* broker.invoke({ scope, operation: "status", input: {}, tabId: olderTabId }); + let capturedTabId: PreviewTabId | undefined; + const older = yield* broker + .invoke({ + scope, + operation: "snapshot", + input: {}, + ...(implicit ? {} : { tabId: olderTabId }), + onTargetTab: (tabId) => { + capturedTabId = tabId; + }, + }) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + const newer = yield* broker + .invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId }) + .pipe(Effect.forkScoped); + yield* Fiber.join(newer); + yield* Fiber.join(older); + yield* broker.invoke({ + scope, + operation: "status", + input: {}, + tabId: olderTabId, + updateCurrentTab: false, }); - return response.pipe(Effect.forkScoped, Effect.asVoid); - }).pipe(Effect.forkScoped); - yield* Effect.yieldNow; - - const older = yield* broker - .invoke({ scope, operation: "snapshot", input: {}, tabId: olderTabId }) - .pipe(Effect.forkScoped); - yield* Effect.yieldNow; - const newer = yield* broker - .invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId }) - .pipe(Effect.forkScoped); - yield* Fiber.join(newer); - yield* Fiber.join(older); - yield* broker.invoke({ - scope, - operation: "status", - input: {}, - tabId: olderTabId, - updateCurrentTab: false, - }); - yield* broker.invoke({ scope, operation: "snapshot", input: {} }); + yield* broker.invoke({ scope, operation: "snapshot", input: {} }); - expect(routedRequests.at(-1)?.tabId).toBe(newerTabId); - }), - ), + expect(routedRequests.at(-1)?.tabId).toBe(newerTabId); + expect(capturedTabId).toBe(olderTabId); + }), + ), ); it.effect("tracks the tab returned by a targeted recording stop", () => diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index e7b69baa9820..8d92059bde8a 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -46,6 +46,8 @@ export interface PreviewAutomationInvokeInput { readonly timeoutMs?: number; /** Background metadata reads must not change the agent's current tab. */ readonly updateCurrentTab?: boolean; + /** Capture the routed tab before another request changes the current assignment. */ + readonly onTargetTab?: (tabId: PreviewTabId | undefined) => void; } export class PreviewAutomationBroker extends Context.Service< @@ -543,6 +545,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { }); } const { connection, requestId, requestContext, requestSequence } = route; + input.onTargetTab?.(requestContext.tabId); const removePending = SynchronizedRef.update(state, (next) => { if (!next.pending.has(requestId)) return next; const pending = new Map(next.pending); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 86c670a1edfd..caa4cbd157cf 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -62,7 +62,11 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( > { const scope = yield* McpInvocationContext.requireMcpCapability("preview"); const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + let targetTabId = tabId; const result = yield* broker.invoke({ + onTargetTab: (resolvedTabId) => { + targetTabId = resolvedTabId; + }, scope, operation, input, @@ -73,7 +77,7 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( const statusTabId = (operation !== "evaluate" && typeof result === "object" && result !== null ? (result as { tabId?: PreviewTabId }).tabId - : undefined) ?? tabId; + : undefined) ?? targetTabId; const page = yield* broker .invoke({ scope,