Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 92 additions & 2 deletions apps/server/src/mcp/toolkits/preview/handlers.test.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,107 @@
import { describe, expect, it } from "@effect/vitest";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { ThreadId } from "@t3tools/contracts";
import {
EnvironmentId,
PreviewTabId,
ProviderInstanceId,
ThreadId,
type PreviewAutomationRequest,
} from "@t3tools/contracts";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import * as Stream from "effect/Stream";

import {
createPendingAttachmentId,
parseThreadSegmentFromAttachmentId,
} from "../../../attachmentStore.ts";
import * as ServerConfig from "../../../config.ts";
import { claimPreviewRecording } from "./handlers.ts";
import * as McpInvocationContext from "../../McpInvocationContext.ts";
import * as McpToolAccess from "../../McpToolAccess.ts";
import { liveThreadsLayer } from "../../McpToolAccess.testkit.ts";
import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";
import { claimPreviewRecording, layerStandard } from "./handlers.ts";
import { PreviewStandardToolkit } from "./tools.ts";

it.effect("preview_open navigates the opened tab to a dev server in one tool call", () =>
Effect.scoped(
Effect.gen(function* () {
const scope = yield* McpInvocationContext.McpInvocationContext;
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
const connected = yield* Deferred.make<void>();
const tabId = PreviewTabId.make("opened-tab");
const target = {
kind: "environment-port" as const,
port: 3000,
path: "/settings?tab=account",
};
const requests: PreviewAutomationRequest[] = [];
const status = {
available: true,
visible: false,
tabId,
url: "http://localhost:3000/settings?tab=account",
title: "Settings",
loading: false,
};
const events = yield* broker.connect({
clientId: "client-1",
environmentId: scope.environmentId,
});
yield* Stream.runForEach(events, (event) => {
if (event.type === "connected") return Deferred.succeed(connected, undefined);
requests.push(event.request);
return broker.respond({
clientId: "client-1",
connectionId: event.connectionId,
requestId: event.request.requestId,
ok: true,
result: event.request.operation === "open" ? { ...status, url: "about:blank" } : status,
});
}).pipe(Effect.forkScoped);
yield* Deferred.await(connected);
const toolkit = yield* PreviewStandardToolkit.pipe(
Effect.provide(McpToolAccess.HandlersLayer.layer(layerStandard)),
);
const results = yield* toolkit
.handle("preview_open", {
url: "http://localhost:3000",
target,
open: false,
reuseExistingTab: false,
})
.pipe(Stream.unwrap, Stream.runCollect);
expect(results.at(-1)?.result).toEqual(status);
expect(requests.map(({ operation, input }) => ({ operation, input }))).toEqual([
{ operation: "open", input: { open: false, show: false, reuseExistingTab: false } },
{ operation: "navigate", input: { target } },
]);
expect(requests[1]?.tabId).toBe(tabId);
expect(requests[1]?.tabIdExplicit).toBe(true);
}).pipe(
Effect.provideService(McpInvocationContext.McpInvocationContext, {
environmentId: EnvironmentId.make("environment-1"),
requestNamespace: "provider-session-1",
thread: {
threadId: ThreadId.make("thread-1"),
providerSessionId: "provider-session-1",
providerInstanceId: ProviderInstanceId.make("codex"),
},
client: undefined,
capabilities: new Set(["preview"] as const),
issuedAt: 1,
}),
Effect.provide(
Layer.mergeAll(PreviewAutomationBroker.layer, liveThreadsLayer).pipe(
Layer.provide(NodeServices.layer),
),
),
),
),
);

describe("claimPreviewRecording", () => {
it.effect("overlapping and repeated claims return the same retained recording", () =>
Expand Down
17 changes: 15 additions & 2 deletions apps/server/src/mcp/toolkits/preview/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,21 @@ const handlers = {
preview_status: McpToolAccess.readsAsCaller((input) =>
invokeTargeted<PreviewAutomationStatus>("status", input ?? {}),
),
preview_open: McpToolAccess.actsAsCaller((input) =>
invokeTargeted<PreviewAutomationStatus>("open", normalizePreviewOpenInput(input)),
// Models often pass a dev server as both `url` and `target`; the target knows the environment.
preview_open: McpToolAccess.actsAsCaller(({ target, url, ...input }) =>
Effect.gen(function* () {
const opened = yield* invokeTargeted<PreviewAutomationStatus>(
"open",
normalizePreviewOpenInput(
target === undefined && url !== undefined ? { ...input, url } : input,
),
);
if (target === undefined || opened.tabId === null) return opened;
return yield* invokeTargeted<PreviewAutomationStatus>("navigate", {
target,
tabId: opened.tabId,
});
}),
),
preview_navigate: McpToolAccess.actsAsCaller((input) =>
invokeTargeted<PreviewAutomationStatus>("navigate", input, input.timeoutMs),
Expand Down
15 changes: 15 additions & 0 deletions apps/server/src/mcp/toolkits/preview/tools.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { expect, it } from "@effect/vitest";
import * as Schema from "effect/Schema";
import { Tool } from "effect/ai";

import { PreviewToolkit } from "./tools.ts";
Expand Down Expand Up @@ -36,3 +37,17 @@ it("exports provider-compatible object schemas", () => {
).toBeDefined();
}
});

it("preview_open accepts a target while preserving open input constraints", () => {
const decode = Schema.decodeUnknownSync(PreviewToolkit.tools.preview_open.parametersSchema);
const target = { kind: "environment-port", port: 3000 };
expect(decode({ target })).toEqual({ target });
expect(decode({})).toEqual({});
expect(decode({ url: "https://example.com" })).toEqual({ url: "https://example.com" });
expect(decode({ url: "http://localhost:3000", target })).toEqual({
url: "http://localhost:3000",
target,
});
expect(() => decode({ target: { ...target, port: 0 } })).toThrow();
expect(() => decode({ target, tabId: "tab-1", reuseExistingTab: false })).toThrow();
});
11 changes: 9 additions & 2 deletions apps/server/src/mcp/toolkits/preview/tools.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {
BrowserNavigationTarget,
OrchestratorMcpFailure,
ToolActivityIcon,
PreviewAutomationActInput,
Expand Down Expand Up @@ -62,11 +63,17 @@ const PreviewStatusTool = Tool.make("preview_status", {
.annotate(Tool.Destructive, false)
.annotate(Tool.Idempotent, true);

// `url` already covers websites, so open only adds the dev-server target.
const EnvironmentPortTarget = BrowserNavigationTarget.members[1];

const PreviewOpenTool = browserTool(
Tool.make("preview_open", {
description:
"Open a preview tab in this thread and show it. `open=false` keeps it in the background. Reuses your current tab unless `tabId` or `reuseExistingTab=false` says otherwise. Parallel subagents in one session must each use `reuseExistingTab=false`, then their own returned `tabId`.",
parameters: PreviewAutomationOpenInput,
"Open a preview tab in this thread and show it. For a dev server, pass `target` instead of `url`. `open=false` keeps it in the background. Reuses your current tab unless `tabId` or `reuseExistingTab=false` says otherwise. Parallel subagents in one session must each use `reuseExistingTab=false`, then their own returned `tabId`.",
parameters: PreviewAutomationOpenInput.mapFields(
(fields) => ({ ...fields, target: Schema.optional(EnvironmentPortTarget) }),
{ unsafePreserveChecks: true },
),
success: PreviewAutomationStatus,
failure: PreviewToolFailure,
dependencies,
Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/provider/agentContextBudget.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"claudeSystemAppendBytes": 5153,
"codexAdditionalContextBytes": 6883,
"mcpToolListJsonBytes": 47074
"mcpToolListJsonBytes": 47538
}
1 change: 1 addition & 0 deletions evals/mcp-tool-use/cases/visual-preview.ndjson
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,4 @@
{"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
{"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
{"id": "requested-after-video-for-edit", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "agent refuses requested browser recording of its own edit", "experience": "A user asked for an after video of a UI fix. The agent declined because the injected instructions prohibited preview for its own edits.", "rule": "Use preview when the user asks for browser work, including recording an edit.", "setup": {"role": "top", "context": "The UI fix is complete and a development server is already running at http://localhost:3000. Browser verification and recording are authorized.", "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_recording_start": {"recording": true, "tabId": "tab_1", "startedAt": "2026-01-01T00:00:00.000Z"}, "preview_recording_stop": {"id": "recording_1", "tabId": "tab_1", "path": "/tmp/after.webm", "mimeType": "video/webm", "sizeBytes": 1024, "createdAt": "2026-01-01T00:00:01.000Z"}}}, "prompt": "Use the T3 browser to verify the UI fix you just made and record an after video.", "expect": {"calls": [{"tool": "preview_recording_start"}, {"tool": "preview_recording_stop"}], "order": ["preview_recording_start", "preview_recording_stop"]}, "source": {"report": "A user requested an after video of a UI fix; the agent declined because preview was prohibited for its own edits."}}
{"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}}
16 changes: 14 additions & 2 deletions evals/mcp-tool-use/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -362,12 +362,15 @@ export const normalizeToolName = (name: string) => {
? bare.slice(T3_PREFIX.length)
: (CODEX_NAMES[bare] ?? bare.slice(bare.lastIndexOf("_") + 1));
};
export const normalizeToolInput = (name: string, input: unknown) =>
normalizeToolName(name) === "Bash" && isRecord(input)
export const normalizeToolInput = (name: string, input: unknown) => {
const decode = t3InputDecoders.get(normalizeToolName(name));
if (decode !== undefined) return Option.getOrElse(decode(input), () => input);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return normalizeToolName(name) === "Bash" && isRecord(input)
? { ...input, command: input.command ?? input.cmd }
: typeof input === "string" && normalizeToolName(name) === "Edit"
? { patch: input }
: input;
};

// McpHttpServer advertises each tool's JSON schema, and Claude Code passes the MCP name,
// description, and schema through unchanged.
Expand All @@ -384,6 +387,15 @@ const t3SourceTools: ReadonlyArray<Tool.Any> = [
DeviceToolkit,
HtmlToolkit,
].flatMap((toolkit): ReadonlyArray<Tool.Any> => Object.values(toolkit.tools));
// McpHttpServer decodes arguments with each tool's JSON codec, which drops undeclared keys.
const t3InputDecoders = new Map(
t3SourceTools.map((tool) => [
tool.name,
Schema.decodeUnknownOption(
Schema.toCodecJson(tool.parametersSchema as Schema.Codec<unknown, unknown>),
),
]),
);
const t3Tools: ReadonlyArray<ApiTool> = t3SourceTools.map((tool) => ({
name: `${T3_PREFIX}${tool.name}`,
description: Tool.getDescription(tool) ?? "",
Expand Down
Loading