From 480fc4f768948ffcd2fe002ad26a01077f32d1b6 Mon Sep 17 00:00:00 2001 From: bird <6666242+bird@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:36:16 -0400 Subject: [PATCH] feat(web): paste images directly into dashboard Chat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard Chat tab is an xterm mirror of a TUI running inside the gateway (often a remote Docker container). Ctrl+V / `/paste` route to the server-side clipboard, which has no access to the browser's clipboard, so pasting a screenshot always fails with "No image found in clipboard" (issue #24860). Handle it fully client-side: intercept the DOM `paste` event on the terminal host, and when the clipboard holds an image, upload the bytes the browser already has to the gateway's managed-files root via the existing /api/files/upload-stream endpoint, then drive the TUI's existing `/image ` command over the PTY. No new RPC, no server changes. Text paste is untouched — the handler only fires when an image is present and otherwise lets the event through. Adds web/src/lib/chatImagePaste.ts (clipboard parsing + upload helper) with unit tests for the clipboard-extraction logic. --- web/src/lib/chatImagePaste.test.ts | 66 +++++++++++++++++++ web/src/lib/chatImagePaste.ts | 100 +++++++++++++++++++++++++++++ web/src/pages/ChatPage.tsx | 46 +++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 web/src/lib/chatImagePaste.test.ts create mode 100644 web/src/lib/chatImagePaste.ts diff --git a/web/src/lib/chatImagePaste.test.ts b/web/src/lib/chatImagePaste.test.ts new file mode 100644 index 0000000000000..1d9d5cb3e000a --- /dev/null +++ b/web/src/lib/chatImagePaste.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; + +import { firstImageFromClipboard } from "./chatImagePaste"; + +// Minimal DataTransfer stand-ins. jsdom's DataTransfer doesn't let us seed +// items/files, so we hand-roll the shape firstImageFromClipboard reads. +function makeItem(kind: string, type: string, file: File | null) { + return { kind, type, getAsFile: () => file } as unknown as DataTransferItem; +} + +function makeData(opts: { + items?: DataTransferItem[]; + files?: File[]; +}): DataTransfer { + const items = opts.items ?? []; + const files = opts.files ?? []; + const itemList: Record = { length: items.length }; + items.forEach((it, i) => { + itemList[i] = it; + }); + const fileList: Record = { length: files.length }; + files.forEach((f, i) => { + fileList[i] = f; + }); + return { + items: itemList, + files: fileList, + } as unknown as DataTransfer; +} + +const png = new File([new Uint8Array([1, 2, 3])], "x.png", { + type: "image/png", +}); + +describe("firstImageFromClipboard", () => { + it("returns null for null clipboard data", () => { + expect(firstImageFromClipboard(null)).toBeNull(); + }); + + it("finds an image via items[].getAsFile()", () => { + const data = makeData({ items: [makeItem("file", "image/png", png)] }); + expect(firstImageFromClipboard(data)).toBe(png); + }); + + it("ignores non-file and non-image items", () => { + const data = makeData({ + items: [ + makeItem("string", "text/plain", null), + makeItem("file", "application/pdf", new File([], "a.pdf")), + ], + }); + expect(firstImageFromClipboard(data)).toBeNull(); + }); + + it("falls back to files[] when items are absent (Safari/Firefox)", () => { + const data = makeData({ files: [png] }); + expect(firstImageFromClipboard(data)).toBe(png); + }); + + it("returns null when nothing image-like is present", () => { + const data = makeData({ + files: [new File([], "notes.txt", { type: "text/plain" })], + }); + expect(firstImageFromClipboard(data)).toBeNull(); + }); +}); diff --git a/web/src/lib/chatImagePaste.ts b/web/src/lib/chatImagePaste.ts new file mode 100644 index 0000000000000..5a553ccf13bad --- /dev/null +++ b/web/src/lib/chatImagePaste.ts @@ -0,0 +1,100 @@ +import { api } from "@/lib/api"; + +// Clipboard image MIME → file extension. Mirrors the set the TUI's /image +// attach path and the gateway's image sniffer accept. +const IMAGE_MIME_EXT: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/gif": "gif", + "image/webp": "webp", + "image/bmp": "bmp", +}; + +// Upload subdirectory under the dashboard managed-files root (the gateway's +// HERMES_HOME, e.g. /opt/data). Kept shallow + predictable so users can find +// and prune pasted images. +const PASTE_UPLOAD_DIR = "uploads/paste"; + +// Anthropic caps a single vision image at ~25 MB; reject earlier client-side +// with a clear message rather than round-tripping a doomed upload. +const MAX_IMAGE_BYTES = 25 * 1024 * 1024; + +export interface ImagePasteResult { + /** Absolute container path the gateway wrote the image to. */ + path: string; + /** Byte size of the uploaded image. */ + bytes: number; + /** File extension chosen for the upload (no dot). */ + ext: string; +} + +/** Pull the first image blob out of a DataTransfer, or null if none present. */ +export function firstImageFromClipboard( + data: DataTransfer | null, +): File | null { + if (!data) return null; + const items = data.items; + if (items) { + for (let i = 0; i < items.length; i++) { + const it = items[i]; + if (it.kind === "file" && it.type.startsWith("image/")) { + const f = it.getAsFile(); + if (f) return f; + } + } + } + // Safari/Firefox sometimes expose files but not items for pasted images. + const files = data.files; + if (files) { + for (let i = 0; i < files.length; i++) { + if (files[i].type.startsWith("image/")) return files[i]; + } + } + return null; +} + +function extForBlob(blob: Blob): string { + return IMAGE_MIME_EXT[blob.type] || "png"; +} + +function timestampName(ext: string): string { + const d = new Date(); + const p = (n: number) => String(n).padStart(2, "0"); + const stamp = + `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}` + + `_${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}` + + `_${p(d.getMilliseconds() % 1000)}`; + return `paste_${stamp}.${ext}`; +} + +/** + * Upload a pasted image blob to the gateway's managed-files root and return + * the absolute container path. + * + * The dashboard Chat tab is an xterm mirror of a TUI running INSIDE the + * gateway (often a remote Docker container). The container has no access to + * the browser's clipboard, so the server-side `clipboard.paste` path can + * never see a pasted image. Instead we upload the bytes the browser already + * holds, then hand the resulting container path to the TUI's `/image` + * command (see attachPastedImage in ChatPage). + */ +export async function uploadPastedImage(blob: Blob): Promise { + if (blob.size === 0) throw new Error("clipboard image is empty"); + if (blob.size > MAX_IMAGE_BYTES) { + const mb = Math.round(MAX_IMAGE_BYTES / (1024 * 1024)); + throw new Error(`image too large (max ${mb} MB)`); + } + const ext = extForBlob(blob); + const name = timestampName(ext); + const relPath = `${PASTE_UPLOAD_DIR}/${name}`; + const file = + blob instanceof File + ? new File([blob], name, { type: blob.type }) + : new File([blob], name, { type: blob.type }); + const res = await api.uploadFile(relPath, file, true); + // The server resolves + returns the absolute container path; prefer the + // entry path, fall back to the top-level path field. + const absPath = res.entry?.path || res.path; + if (!absPath) throw new Error("upload succeeded but no path was returned"); + return { path: absPath, bytes: blob.size, ext }; +} diff --git a/web/src/pages/ChatPage.tsx b/web/src/pages/ChatPage.tsx index bb623cf2c6c1e..99d91ba809ba6 100644 --- a/web/src/pages/ChatPage.tsx +++ b/web/src/pages/ChatPage.tsx @@ -36,6 +36,10 @@ import { usePageHeader } from "@/contexts/usePageHeader"; import { useI18n } from "@/i18n"; import { api } from "@/lib/api"; import { normalizeSessionTitle } from "@/lib/chat-title"; +import { + firstImageFromClipboard, + uploadPastedImage, +} from "@/lib/chatImagePaste"; import { PluginSlot } from "@/plugins"; import { useTheme } from "@/themes"; import { useProfileScope } from "@/contexts/useProfileScope"; @@ -522,6 +526,47 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { return false; }); + // ── Image paste ────────────────────────────────────────────────────── + // Bare Ctrl+V / Ctrl+Shift+V text paste is handled by the key handler + // above (and by the browser's own paste into the focused terminal). Image + // paste can't go through the server-side clipboard: the TUI runs inside + // the gateway (often a remote Docker container) with no access to the + // browser clipboard. So we intercept the DOM `paste` event here, upload + // the image bytes the browser holds to the gateway's managed-files root, + // then drive the TUI's existing `/image ` command over the PTY. + const onHostPaste = (ev: ClipboardEvent) => { + const blob = firstImageFromClipboard(ev.clipboardData); + if (!blob) return; // no image → let text paste proceed normally + ev.preventDefault(); + ev.stopPropagation(); + void (async () => { + try { + const { path } = await uploadPastedImage(blob); + const ws = wsRef.current; + if (!ws || ws.readyState !== WebSocket.OPEN) { + term.write( + "\r\n\x1b[33m[image paste] chat not connected — try again]\x1b[0m\r\n", + ); + return; + } + // Mirror handleCopyLast's burst-then-Return timing so Ink's stdin + // tokenizer sees the slash command as keypresses, then submits. + ws.send(`/image ${path}`); + setTimeout(() => { + const s = wsRef.current; + if (s && s.readyState === WebSocket.OPEN) s.send("\r"); + }, 100); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.warn("[dashboard clipboard] image paste failed:", msg); + term.write( + `\r\n\x1b[31m[image paste failed: ${msg}]\x1b[0m\r\n`, + ); + } + })(); + }; + host.addEventListener("paste", onHostPaste); + const unicode11 = new Unicode11Addon(); term.loadAddon(unicode11); term.unicode.activeVersion = "11"; @@ -829,6 +874,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { syncMetricsRef.current = null; onDataDisposable?.dispose(); onResizeDisposable?.dispose(); + host.removeEventListener("paste", onHostPaste); if (metricsDebounce) clearTimeout(metricsDebounce); window.removeEventListener("resize", scheduleSyncTerminalMetrics); window.visualViewport?.removeEventListener(