Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 83 additions & 2 deletions apps/server/src/mcp/toolkits/preview/handlers.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
import { describe, expect, it } from "vite-plus/test";
import { describe, expect, it } from "@effect/vitest";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { ThreadId } from "@t3tools/contracts";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";

import { normalizePreviewOpenInput } from "./handlers.ts";
import {
createPendingAttachmentId,
parseThreadSegmentFromAttachmentId,
} from "../../../attachmentStore.ts";
import * as ServerConfig from "../../../config.ts";
import { claimPreviewRecording, normalizePreviewOpenInput } from "./handlers.ts";

describe("normalizePreviewOpenInput", () => {
it("leaves an unstated visibility for the client preference to decide", () => {
Expand Down Expand Up @@ -30,3 +41,73 @@ describe("normalizePreviewOpenInput", () => {
});
});
});

describe("claimPreviewRecording", () => {
it.effect.each([6, 5])(
"claims only a complete uploaded recording (reported bytes: %s)",
(sizeBytes) =>
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const fileSystem = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const uploadedAttachmentId = createPendingAttachmentId(".webm");
const pendingPath = path.join(config.attachmentsDir, `${uploadedAttachmentId}.webm`);
yield* fileSystem.makeDirectory(config.attachmentsDir, { recursive: true });
yield* fileSystem.writeFileString(pendingPath, "video!");
const response = {
id: "desktop-recording",
tabId: "tab-1",
path: "/desktop/recording.webm",
mimeType: "video/webm",
sizeBytes,
createdAt: "2026-09-07T00:00:00.000Z",
uploadedAttachmentId,
};
const result = yield* claimPreviewRecording(ThreadId.make("thread-1"), response).pipe(
Effect.result,
);
if (sizeBytes === 6) {
expect(result._tag).toBe("Success");
if (result._tag !== "Success") return;
expect(result.success.path).not.toBe(response.path);
expect(parseThreadSegmentFromAttachmentId(result.success.id)).toBe("thread-1");
expect(yield* fileSystem.readFileString(result.success.path)).toBe("video!");
expect(yield* fileSystem.exists(pendingPath)).toBe(false);
} else {
expect(result._tag).toBe("Failure");
if (result._tag !== "Failure") return;
expect(result.failure.reason).toBe("size-mismatch");
expect(yield* fileSystem.exists(pendingPath)).toBe(true);
}
}).pipe(
Effect.provide(
ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-recording-" }).pipe(
Layer.provideMerge(NodeServices.layer),
),
),
),
);

it.effect("reports an older desktop without returning its inaccessible path", () =>
Effect.gen(function* () {
const result = yield* claimPreviewRecording(ThreadId.make("thread-1"), {
id: "desktop-recording",
tabId: "tab-1",
path: "/desktop/recording.webm",
mimeType: "video/webm",
sizeBytes: 6,
createdAt: "2026-09-07T00:00:00.000Z",
}).pipe(Effect.result);
expect(result._tag).toBe("Failure");
if (result._tag !== "Failure") return;
expect(result.failure.reason).toBe("desktop-update-required");
expect(result.failure.message).toContain("Update the desktop app");
}).pipe(
Effect.provide(
ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-recording-" }).pipe(
Layer.provideMerge(NodeServices.layer),
),
),
),
);
});
105 changes: 95 additions & 10 deletions apps/server/src/mcp/toolkits/preview/handlers.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,24 @@
import * as Effect from "effect/Effect";
import type {
PreviewAutomationOperation,
PreviewAutomationOpenInput,
import * as FileSystem from "effect/FileSystem";
import * as Schema from "effect/Schema";
import {
PROVIDER_SEND_TURN_MAX_FILE_BYTES,
PREVIEW_RECORDING_STOP_TIMEOUT_MS,
PreviewAutomationRecordingTransferError,
PreviewAutomationRecordingArtifact,
PreviewAutomationRecordingStatus,
PreviewAutomationResizeResult,
PreviewAutomationSetColorSchemeResult,
PreviewAutomationSnapshot,
PreviewAutomationStatus,
PreviewTabId,
type ThreadId,
type PreviewAutomationOperation,
type PreviewAutomationOpenInput,
type PreviewAutomationRecordingStatus,
type PreviewAutomationResizeResult,
type PreviewAutomationSetColorSchemeResult,
type PreviewAutomationSnapshot,
type PreviewAutomationStatus,
type PreviewTabId,
} from "@t3tools/contracts";

import { planAttachmentClaim } from "../../../attachmentStore.ts";
import * as ServerConfig from "../../../config.ts";
import * as McpInvocationContext from "../../McpInvocationContext.ts";
import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";
import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from "./tools.ts";
Expand Down Expand Up @@ -67,6 +75,75 @@ const invokeTargeted = <A>(
return invoke<A>(operation, operationInput, timeoutMs, tabId);
};

const UploadedRecordingArtifact = Schema.Struct({
...PreviewAutomationRecordingArtifact.fields,
uploadedAttachmentId: Schema.optional(Schema.String),
});
const decodeUploadedRecordingArtifact = Schema.decodeUnknownEffect(UploadedRecordingArtifact);
const isRecordingTransferError = Schema.is(PreviewAutomationRecordingTransferError);

export const claimPreviewRecording = Effect.fn("PreviewToolkit.claimRecording")(function* (
threadId: ThreadId,
response: unknown,
) {
const artifact = yield* decodeUploadedRecordingArtifact(response).pipe(
Effect.mapError(
(cause) =>
new PreviewAutomationRecordingTransferError({
threadId,
reason: "invalid-metadata",
cause,
}),
),
);
if (!artifact.uploadedAttachmentId) {
return yield* new PreviewAutomationRecordingTransferError({
threadId,
reason: "desktop-update-required",
});
}
const config = yield* ServerConfig.ServerConfig;
const claim = planAttachmentClaim({
attachmentsDir: config.attachmentsDir,
threadId,
attachmentId: artifact.uploadedAttachmentId,
});
if (!claim.ok) {
return yield* new PreviewAutomationRecordingTransferError({
threadId,
reason: "invalid-upload",
});
}
const fileSystem = yield* FileSystem.FileSystem;
yield* Effect.gen(function* () {
const stat = yield* fileSystem.stat(claim.currentPath);
if (
stat.type !== "File" ||
Number(stat.size) !== artifact.sizeBytes ||
artifact.sizeBytes <= 0 ||
artifact.sizeBytes > PROVIDER_SEND_TURN_MAX_FILE_BYTES
) {
return yield* new PreviewAutomationRecordingTransferError({
threadId,
reason: "size-mismatch",
});
}
yield* fileSystem.rename(claim.currentPath, claim.finalPath);
Comment thread
maria-rcks marked this conversation as resolved.
Outdated
}).pipe(
Effect.mapError((cause) =>
isRecordingTransferError(cause)
? cause
: new PreviewAutomationRecordingTransferError({
threadId,
reason: "retain-failed",
cause,
}),
),
);
const { uploadedAttachmentId: _uploadedAttachmentId, ...recording } = artifact;
return { ...recording, id: claim.finalId, path: claim.finalPath };
});

const handlers = {
preview_status: (input) => invokeTargeted<PreviewAutomationStatus>("status", input ?? {}),
preview_open: (input) =>
Expand Down Expand Up @@ -94,7 +171,15 @@ const handlers = {
preview_recording_start: (input) =>
invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),
preview_recording_stop: (input) =>
invokeTargeted<PreviewAutomationRecordingArtifact>("recordingStop", input ?? {}),
Effect.gen(function* () {
const scope = yield* McpInvocationContext.requireMcpCapability("preview");
const response = yield* invokeTargeted<unknown>(
Comment thread
maria-rcks marked this conversation as resolved.
"recordingStop",
{ ...input, transferToEnvironment: true },
PREVIEW_RECORDING_STOP_TIMEOUT_MS,
);
return yield* claimPreviewRecording(scope.threadId, response);
}),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];

const { preview_snapshot, ...standardHandlers } = handlers;
Expand Down
6 changes: 4 additions & 2 deletions apps/server/src/mcp/toolkits/preview/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ import {
PreviewAutomationWaitForInput,
} from "@t3tools/contracts";
import * as Schema from "effect/Schema";
import * as FileSystem from "effect/FileSystem";
import { Tool, Toolkit } from "effect/unstable/ai";

import * as McpInvocationContext from "../../McpInvocationContext.ts";
import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";
import * as ServerConfig from "../../../config.ts";

const dependencies = [
McpInvocationContext.McpInvocationContext,
Expand Down Expand Up @@ -207,11 +209,11 @@ export const PreviewRecordingStartTool = safeBrowserTool(
export const PreviewRecordingStopTool = safeBrowserTool(
Tool.make("preview_recording_stop", {
description:
"Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and save it as a local evidence artifact.",
"Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and transfer the compressed recording once (up to 50 MiB) to an evidence file readable in this agent's environment. Returns its environment-local path after transfer succeeds.",
parameters: PreviewAutomationTabTargetInput,
success: PreviewAutomationRecordingArtifact,
failure: PreviewAutomationError,
dependencies,
dependencies: [...dependencies, FileSystem.FileSystem, ServerConfig.ServerConfig],
}).annotate(Tool.Title, "Stop browser recording"),
);

Expand Down
43 changes: 43 additions & 0 deletions apps/web/src/browser/browserRecording.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ import {
readActiveBrowserRecordingTargets,
startBrowserRecording,
stopBrowserRecording,
stopBrowserRecordingForUpload,
} from "./browserRecording";
import { useBrowserSurfaceStore } from "./browserSurfaceStore";
import { previewRuntimeTabId } from "./previewRuntimeTabId";
Expand Down Expand Up @@ -180,6 +181,48 @@ describe("browser recording", () => {
await stopBrowserRecording("automation-recording-tab");
});

it("saves locally and releases capture before transferring the encoded recording once", async () => {
const stopTrack = vi.fn();
getDisplayMedia.mockResolvedValue({
getVideoTracks: () => [],
getTracks: () => [{ stop: stopTrack }],
});
await startBrowserRecording("transfer-tab");
let finishUpload!: () => void;
const uploaded = new Promise<void>((resolve) => {
finishUpload = resolve;
});
const transfer = vi.fn(async (artifact, blob: Blob) => {
expect(save).toHaveBeenCalledOnce();
expect(stopTrack).toHaveBeenCalled();
expect(artifact.path).toBe("/tmp/recording-test.webm");
expect(blob.type).toBe("video/webm;codecs=vp9");
await uploaded;
return "uploaded-recording";
});
const localStop = stopBrowserRecording("transfer-tab");
const firstStop = stopBrowserRecordingForUpload("transfer-tab", transfer);
const secondStop = stopBrowserRecordingForUpload("transfer-tab", transfer);
finishUpload();
expect(await firstStop).toEqual(await secondStop);
expect((await firstStop)?.uploadedAttachmentId).toBe("uploaded-recording");
expect((await localStop)?.path).toBe("/tmp/recording-test.webm");
expect(transfer).toHaveBeenCalledOnce();
});

it("keeps the saved desktop file and releases the recording when transfer fails", async () => {
await startBrowserRecording("failed-transfer-tab");
await expect(
stopBrowserRecordingForUpload("failed-transfer-tab", async () => {
throw new Error("Connection interrupted");
}),
).rejects.toThrow("Connection interrupted");
expect(save).toHaveBeenCalledOnce();
expect(readActiveBrowserRecordingTabIds().has("failed-transfer-tab")).toBe(false);
await startBrowserRecording("failed-transfer-tab");
await stopBrowserRecording("failed-transfer-tab");
});

it("paints and holds a hidden browser surface for the recording lifetime", async () => {
startScreencast.mockImplementationOnce(async (tabId: string) => {
expect(animationFrameCount).toBe(2);
Expand Down
16 changes: 16 additions & 0 deletions apps/web/src/browser/browserRecording.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ interface ActiveRecording {
releaseSurfaceActivity: (() => void) | null;
stream: MediaStream | null;
recorder: MediaRecorder | null;
savedBlob?: Blob;
uploadPromise?: Promise<string>;
lifecycle: BrowserRecordingLifecycle;
}

Expand Down Expand Up @@ -708,6 +710,7 @@ const finalizeBrowserRecording = async (
mimeType,
new Uint8Array(await blob.arrayBuffer()),
);
recording.savedBlob = blob;
result = { _tag: "Success", artifact };
} catch (cause) {
throw new BrowserRecordingOperationError({
Expand Down Expand Up @@ -814,3 +817,16 @@ export function stopBrowserRecording(
recording.lifecycle = { phase: "stopping", stopPromise };
return stopPromise;
}

/** Joins local stops and shares one upload among concurrent automation requests. */
export async function stopBrowserRecordingForUpload(
tabId: string,
upload: (artifact: DesktopPreviewRecordingArtifact, blob: Blob) => Promise<string>,
): Promise<(DesktopPreviewRecordingArtifact & { uploadedAttachmentId: string }) | null> {
const recording = activeRecordings.get(tabId);
if (!recording) return null;
const artifact = await stopBrowserRecording(tabId);
if (!artifact || !recording.savedBlob) return null;
recording.uploadPromise ??= upload(artifact, recording.savedBlob);
return { ...artifact, uploadedAttachmentId: await recording.uploadPromise };
}
Loading
Loading