Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 2 additions & 10 deletions apps/web/src/components/preview/PreviewView.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ const mocks = vi.hoisted(() => ({
openPictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
closePictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
pickElement: vi.fn(),
capturePreviewAnnotationScreenshot: vi.fn(),
addPreviewAnnotation: vi.fn(),
addImage: vi.fn(),
toggleAnnotation: null as (() => void) | null,
Expand Down Expand Up @@ -90,10 +89,6 @@ vi.mock("~/composerDraftStore", () => ({
}),
}));

vi.mock("~/lib/previewAnnotation", () => ({
capturePreviewAnnotationScreenshot: mocks.capturePreviewAnnotationScreenshot,
}));

vi.mock("~/localApi", () => ({
ensureLocalApi: vi.fn(),
}));
Expand Down Expand Up @@ -342,8 +337,6 @@ describe("PreviewView navigation", () => {
mocks.openPictureInPicture.mockClear();
mocks.closePictureInPicture.mockClear();
mocks.pickElement.mockReset();
mocks.capturePreviewAnnotationScreenshot.mockReset();
mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "none" });
mocks.addPreviewAnnotation.mockClear();
vi.mocked(toastManager.add).mockClear();
mocks.addImage.mockClear();
Expand Down Expand Up @@ -585,7 +578,7 @@ describe("PreviewView navigation", () => {
expect(toastManager.add).toHaveBeenCalledTimes(1);
});

it("still sends when the picked element's crop cannot be captured", async () => {
it("still sends annotation text when the picked element's crop is malformed", async () => {
const annotation = {
id: "annotation-2",
pageUrl: "https://example.com/dashboard",
Expand All @@ -596,7 +589,7 @@ describe("PreviewView navigation", () => {
strokes: [],
styleChanges: [],
screenshot: {
dataUrl: "data:image/png;base64,c2NyZWVuc2hvdA==",
dataUrl: "data:image/png;base64,%%%",
width: 10,
height: 10,
cropRect: { x: 0, y: 0, width: 10, height: 10 },
Expand All @@ -605,7 +598,6 @@ describe("PreviewView navigation", () => {
};
const onSendAnnotation = vi.fn();
mocks.pickElement.mockResolvedValue({ annotation, submission: "send" });
mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "failed" });

renderToStaticMarkup(
<PreviewView
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/components/preview/PreviewView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,7 @@ export function PreviewView({
// instead of holding the composer for an attachment that never lands.
// The stored copy drops the screenshot on failure, otherwise the prompt
// would tell the agent a crop is attached when none was sent.
const capture = await capturePreviewAnnotationScreenshot(picked);
const capture = capturePreviewAnnotationScreenshot(picked);
// Main reports a crop that failed or timed out on its side; the local
// conversion can fail too. Either way the user should hear about it.
const cropDropped = screenshotFailed || capture.status === "failed";
Expand Down
74 changes: 28 additions & 46 deletions apps/web/src/lib/previewAnnotation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,66 +9,48 @@ const annotation: PreviewAnnotationPayload = {
pageTitle: "Example",
comment: "Make these cards feel related.",
elements: [],
regions: [{ id: "region_1", rect: { x: 10, y: 20, width: 100, height: 80 } }],
strokes: [
{
id: "stroke_1",
color: "#7c3aed",
width: 4,
points: [
{ x: 10, y: 10 },
{ x: 20, y: 20 },
],
bounds: { x: 6, y: 6, width: 18, height: 18 },
},
],
styleChanges: [
{
targetId: "element_1",
selector: ".card",
property: "border-radius",
previousValue: "4px",
value: "16px",
},
],
regions: [],
strokes: [],
styleChanges: [],
screenshot: {
dataUrl: "data:image/png;base64,AA==",
width: 100,
height: 80,
cropRect: { x: 10, y: 20, width: 100, height: 80 },
dataUrl:
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII=",
width: 1,
height: 1,
cropRect: { x: 0, y: 0, width: 1, height: 1 },
},
createdAt: "2026-06-11T00:00:00.000Z",
};

describe("preview annotation capture", () => {
afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
});

it("returns the crop when the fetch resolves", async () => {
vi.stubGlobal("fetch", async () => new Response(new Blob(["png"], { type: "image/png" })));
const capture = await capturePreviewAnnotationScreenshot(annotation);
it("preserves the PNG bytes, MIME type, and filename without fetching", async () => {
const fetch = vi.fn().mockRejectedValue(new TypeError("Blocked by connect-src"));
vi.stubGlobal("fetch", fetch);
const capture = capturePreviewAnnotationScreenshot(annotation);
expect(capture.status).toBe("captured");
if (capture.status !== "captured") throw new Error("Expected a screenshot file");
expect(capture.file.name).toBe("preview-annotation-annotation_1.png");
expect(capture.file.type).toBe("image/png");
expect(Buffer.from(await capture.file.arrayBuffer()).toString("hex")).toBe(
"89504e470d0a1a0a0000000d4948445200000001000000010804000000b51c0c020000000b4944415478da63fcff1f0003030200efa2a75b0000000049454e44ae426082",
);
expect(fetch).not.toHaveBeenCalled();
});

it("reports none when the annotation carries no crop", async () => {
const capture = await capturePreviewAnnotationScreenshot({ ...annotation, screenshot: null });
it("reports none when the annotation carries no crop", () => {
const capture = capturePreviewAnnotationScreenshot({ ...annotation, screenshot: null });
expect(capture).toEqual({ status: "none" });
});

it("fails instead of hanging when the crop never arrives", async () => {
vi.useFakeTimers();
vi.stubGlobal("fetch", () => new Promise<Response>(() => {}));
const capturePromise = capturePreviewAnnotationScreenshot(annotation, 1_000);
await vi.advanceTimersByTimeAsync(1_000);
expect(await capturePromise).toEqual({ status: "failed" });
});

it("fails when the crop fetch throws", async () => {
vi.stubGlobal("fetch", async () => {
throw new Error("data url unreadable");
});
expect(await capturePreviewAnnotationScreenshot(annotation)).toEqual({ status: "failed" });
});
it.each(["data:image/jpeg;base64,AA==", "data:image/png;base64,", "data:image/png;base64,%%%"])(
"reports a malformed screenshot as failed: %s",
(dataUrl) => {
const picked = { ...annotation, screenshot: { ...annotation.screenshot!, dataUrl } };
expect(capturePreviewAnnotationScreenshot(picked)).toEqual({ status: "failed" });
},
);
});
45 changes: 13 additions & 32 deletions apps/web/src/lib/previewAnnotation.ts
Original file line number Diff line number Diff line change
@@ -1,49 +1,30 @@
import type { PreviewAnnotationPayload } from "@t3tools/contracts";

async function previewAnnotationScreenshotFile(
annotation: PreviewAnnotationPayload,
): Promise<File | null> {
if (!annotation.screenshot) return null;
const response = await fetch(annotation.screenshot.dataUrl);
const blob = await response.blob();
return new File([blob], `preview-annotation-${annotation.id}.png`, {
type: blob.type || "image/png",
});
}

/** Upper bound on turning a picked element's crop into a composer attachment. */
const PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS = 5_000;
import { dataUrlToFile } from "./imageCompression";

export type PreviewAnnotationCapture =
/** The crop is ready to attach. */
| { readonly status: "captured"; readonly file: File }
/** The pick carried no crop, which is normal for comment-only annotations. */
| { readonly status: "none" }
/** The crop stalled or threw. Send the annotation without it. */
/** The crop could not be decoded. Send the annotation without it. */
| { readonly status: "failed" };

/**
* Bounded wrapper around `previewAnnotationScreenshotFile`. The picker holds the
* composer while this runs, so it must always settle: a stalled crop resolves as
* `failed` instead of leaving the caller waiting.
*/
export async function capturePreviewAnnotationScreenshot(
const PNG_DATA_URL_PREFIX = "data:image/png;base64,";

/** Decode Electron's PNG crop locally; fetching a data URL violates desktop connect-src. */
export function capturePreviewAnnotationScreenshot(
annotation: PreviewAnnotationPayload,
timeoutMs: number = PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS,
): Promise<PreviewAnnotationCapture> {
): PreviewAnnotationCapture {
if (!annotation.screenshot) return { status: "none" };
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const file = await Promise.race([
previewAnnotationScreenshotFile(annotation),
new Promise<null>((resolve) => {
timer = setTimeout(() => resolve(null), timeoutMs);
}),
]);
return file ? { status: "captured", file } : { status: "failed" };
const { dataUrl } = annotation.screenshot;
if (!dataUrl.startsWith(PNG_DATA_URL_PREFIX)) {
Comment thread
juliusmarminge marked this conversation as resolved.
return { status: "failed" };
}
const file = dataUrlToFile(dataUrl, `preview-annotation-${annotation.id}.png`, "image/png");
Comment thread
coderabbitai[bot] marked this conversation as resolved.
return file.size > 0 ? { status: "captured", file } : { status: "failed" };
} catch {
return { status: "failed" };
} finally {
clearTimeout(timer);
}
}
Loading