diff --git a/docs/design/session-media-references.md b/docs/design/session-media-references.md new file mode 100644 index 00000000000..3f30aa405e2 --- /dev/null +++ b/docs/design/session-media-references.md @@ -0,0 +1,46 @@ +# Session media references + +## Problem + +Image prompts currently repeat base64 data in request JSON, pending queues, +SSE events, and the replay ring. Mid-turn batches amplify this because several +images can be emitted in one event. + +## Design + +The daemon stores uploaded image bytes in a session-owned temporary +directory and returns a small media reference: + +```ts +{ + type: 'image'; + mediaId: string; + mimeType: string; + size: number; +} +``` + +Prompt and mid-turn APIs accept these references. Queues, reconciliation +snapshots, SSE events, and persisted user-message metadata retain only +references. Immediately before an ACP prompt or mid-turn drain crosses into the +child, the bridge resolves references to the protocol's inline base64 content +blocks. + +The TypeScript session client hydrates references in replay/live events and +queue snapshots through the authenticated media download route. Existing UI +reducers and renderers therefore continue receiving their current inline image +shape without carrying base64 through the daemon event bus. + +## Ownership and limits + +- Media is scoped to the resolved daemon session and protected by the + existing session client authorization. +- Each object is limited to 8 MiB, each live session to 100 MiB and 256 objects, + and the daemon retains at most 512 MiB across sessions. +- Objects remain available across client detach and reload for up to three + hours; explicit close, kill, and daemon shutdown remove them immediately. +- References from another or unavailable session fail instead of falling back to a + primary runtime. + +Legacy inline media remains accepted and echoed unchanged for older clients. +Media-reference-capable clients avoid placing those bytes in the replay ring. diff --git a/integration-tests/cli/qwen-serve-routes.test.ts b/integration-tests/cli/qwen-serve-routes.test.ts index be731880170..31563c00a9e 100644 --- a/integration-tests/cli/qwen-serve-routes.test.ts +++ b/integration-tests/cli/qwen-serve-routes.test.ts @@ -314,6 +314,7 @@ describe('qwen serve — capabilities envelope', () => { 'session_source_metadata', 'session_side_task', 'session_prompt', + 'session_media', 'session_mid_turn_message_mutation', 'session_mid_turn_message_query', 'session_cancel', diff --git a/packages/acp-bridge/package.json b/packages/acp-bridge/package.json index a53474d9f01..23e2306e9c4 100644 --- a/packages/acp-bridge/package.json +++ b/packages/acp-bridge/package.json @@ -55,6 +55,10 @@ "types": "./dist/sessionArtifacts.d.ts", "import": "./dist/sessionArtifacts.js" }, + "./sessionMedia": { + "types": "./dist/sessionMedia.d.ts", + "import": "./dist/sessionMedia.js" + }, "./daemonEventTypes": { "types": "./dist/daemonEventTypes.d.ts", "import": "./dist/daemonEventTypes.js" diff --git a/packages/acp-bridge/src/bridge.test.ts b/packages/acp-bridge/src/bridge.test.ts index c80befa46fe..a52ccc2719f 100644 --- a/packages/acp-bridge/src/bridge.test.ts +++ b/packages/acp-bridge/src/bridge.test.ts @@ -110,6 +110,10 @@ import { SESS_A, } from './internal/testUtils.js'; import { SessionArtifactAuthorizationError } from './sessionArtifacts.js'; +import { + SESSION_MEDIA_MAX_TOTAL_BYTES, + SessionMediaStore, +} from './sessionMedia.js'; import { REQUESTED_SESSION_ID_META_KEY, MID_TURN_QUEUE_DRAIN_METHOD, @@ -13266,6 +13270,283 @@ describe('createAcpSessionBridge', () => { await bridge.shutdown(); }); + it('keeps media references on the event bus and resolves bytes for ACP', async () => { + const prompts: PromptRequest[] = []; + const factory: ChannelFactory = async () => + makeChannel({ + promptImpl: (req) => { + prompts.push(req); + return { stopReason: 'end_turn' }; + }, + }).channel; + const bridge = makeBridge({ channelFactory: factory }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.from([1, 2, 3]), + 'image/png', + { clientId: session.clientId }, + ); + const abort = new AbortController(); + const iter = bridge.subscribeEvents(session.sessionId, { + signal: abort.signal, + }); + const echoed = (async () => { + for await (const event of iter) { + if (event.type !== 'session_update') continue; + const content = (event.data as { update?: { content?: unknown } }) + .update?.content; + if ((content as { type?: string } | undefined)?.type === 'image') { + return content; + } + } + return undefined; + })(); + + await bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [reference] }, + undefined, + { clientId: session.clientId }, + ); + + expect(prompts[0]?.prompt).toEqual([ + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ]); + expect(prompts[0]?._meta?.['qwen.daemon.mediaReferences']).toEqual([ + reference, + ]); + expect(await echoed).toEqual(reference); + abort.abort(); + await bridge.shutdown(); + }); + + it('keeps inline media bytes in echoes for legacy clients', async () => { + const bridge = makeBridge({ + channelFactory: async () => + makeChannel({ promptImpl: () => ({ stopReason: 'end_turn' }) }) + .channel, + }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const abort = new AbortController(); + const iter = bridge.subscribeEvents(session.sessionId, { + signal: abort.signal, + }); + const echoed = (async () => { + for await (const event of iter) { + if (event.type !== 'session_update') continue; + const content = (event.data as { update?: { content?: unknown } }) + .update?.content; + if ((content as { type?: string } | undefined)?.type === 'image') { + return content; + } + } + return undefined; + })(); + const image = { + type: 'image' as const, + data: 'AQID', + mimeType: 'image/png', + }; + + await bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [image] }, + undefined, + { clientId: session.clientId }, + ); + + expect(await echoed).toEqual(image); + abort.abort(); + await bridge.shutdown(); + }); + + it('retains uploaded media across last-client detach and session load', async () => { + const bridge = makeBridge({ + channelFactory: async () => + makeChannel({ + loadSessionImpl: () => ({}), + }).channel, + }); + const session = await bridge.spawnOrAttach({ + workspaceCwd: WS_A, + sessionScope: 'thread', + }); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.from([1, 2, 3]), + 'image/png', + { clientId: session.clientId }, + ); + + await bridge.detachClient(session.sessionId, session.clientId); + expect(bridge.sessionCount).toBe(0); + + const restored = await bridge.loadSession({ + sessionId: session.sessionId, + workspaceCwd: WS_A, + }); + expect( + await bridge.readSessionMedia(restored.sessionId, reference.mediaId, { + clientId: restored.clientId, + }), + ).toEqual({ data: Buffer.from([1, 2, 3]), mimeType: 'image/png' }); + + await bridge.shutdown(); + }); + + it('retains uploaded media across idle timeout and session load', async () => { + const bridge = makeBridge({ + channelFactory: async () => + makeChannel({ loadSessionImpl: () => ({}) }).channel, + }); + const session = await bridge.spawnOrAttach({ + workspaceCwd: WS_A, + sessionScope: 'thread', + }); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.from([1, 2, 3]), + 'image/png', + { clientId: session.clientId }, + ); + + await bridge.closeSession(session.sessionId, undefined, { + reason: 'idle_timeout', + }); + const restored = await bridge.loadSession({ + sessionId: session.sessionId, + workspaceCwd: WS_A, + }); + + expect( + await bridge.readSessionMedia(restored.sessionId, reference.mediaId, { + clientId: restored.clientId, + }), + ).toEqual({ data: Buffer.from([1, 2, 3]), mimeType: 'image/png' }); + await bridge.shutdown(); + }); + + it('reaps retained media after a detached session exceeds the media TTL', async () => { + vi.useFakeTimers(); + try { + const bridge = makeBridge({ + channelFactory: async () => + makeChannel({ loadSessionImpl: () => ({}) }).channel, + sessionReapIntervalMs: 60 * 60_000, + sessionIdleTimeoutMs: 0, + }); + const session = await bridge.spawnOrAttach({ + workspaceCwd: WS_A, + sessionScope: 'thread', + }); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.from([1, 2, 3]), + 'image/png', + { clientId: session.clientId }, + ); + + await bridge.detachClient(session.sessionId, session.clientId); + await vi.advanceTimersByTimeAsync(4 * 60 * 60_000); + + const restored = await bridge.loadSession({ + sessionId: session.sessionId, + workspaceCwd: WS_A, + }); + expect( + await bridge.readSessionMedia(restored.sessionId, reference.mediaId, { + clientId: restored.clientId, + }), + ).toBeUndefined(); + + await bridge.shutdown(); + } finally { + vi.useRealTimers(); + } + }); + + it('still sweeps retained media when the session reaper is disabled', async () => { + // sessionReapIntervalMs: 0 turns off idle-session reaping, but the + // retained-media TTL sweep must keep running — it is the only runtime + // release path for detach-retained media. + vi.useFakeTimers(); + try { + const bridge = makeBridge({ + channelFactory: async () => + makeChannel({ loadSessionImpl: () => ({}) }).channel, + sessionReapIntervalMs: 0, + sessionIdleTimeoutMs: 0, + }); + const session = await bridge.spawnOrAttach({ + workspaceCwd: WS_A, + sessionScope: 'thread', + }); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.from([1, 2, 3]), + 'image/png', + { clientId: session.clientId }, + ); + + await bridge.detachClient(session.sessionId, session.clientId); + await vi.advanceTimersByTimeAsync(4 * 60 * 60_000); + + const restored = await bridge.loadSession({ + sessionId: session.sessionId, + workspaceCwd: WS_A, + }); + expect( + await bridge.readSessionMedia(restored.sessionId, reference.mediaId, { + clientId: restored.clientId, + }), + ).toBeUndefined(); + + await bridge.shutdown(); + } finally { + vi.useRealTimers(); + } + }); + + it('releases uploaded media when killSession falls back to a force kill', async () => { + // A wedged child never answers the kill's cooperative close-notify, + // so the close request times out and kill takes the force-kill + // fallback; the media must still be removed immediately instead of + // degrading to the crash-path detach retention. (A structured + // RequestError answer is the separate definitive-refusal case, where + // kill spares the channel and retries later.) + const close = vi.spyOn(SessionMediaStore.prototype, 'close'); + const handle = makeChannel({ + extMethodImpl: (method) => { + if (method === SERVE_CONTROL_EXT_METHODS.sessionClose) { + return new Promise(() => {}); + } + return {}; + }, + }); + const bridge = makeBridge({ + channelFactory: async () => handle.channel, + initializeTimeoutMs: 20, + }); + try { + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.from([1, 2, 3]), + 'image/png', + { clientId: session.clientId }, + ); + + await expect(bridge.killSession(session.sessionId)).resolves.toBe(true); + + expect(close).toHaveBeenCalledOnce(); + expect(bridge.sessionCount).toBe(0); + } finally { + close.mockRestore(); + await bridge.shutdown(); + } + }); + it('broadcasts prompt_cancelled with originator attribution on cancelSession', async () => { // Cross-client sync: a cancel must surface as a first-class event // so peer subscribers don't have to infer it from the absence of @@ -15102,6 +15383,66 @@ describe('createAcpSessionBridge', () => { await bridge.shutdown(); }); + it('removes uploaded media owned by a deleted queued prompt', async () => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const bridge = makeBridge({ + channelFactory: async () => + makeChannel({ + promptImpl: async (req) => { + if ((req.prompt[0] as { text?: string }).text === 'blocker') { + await gate; + } + return { stopReason: 'end_turn' }; + }, + }).channel, + }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const running = bridge.sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'blocker' }], + }, + undefined, + { promptId: 'media-delete-blocker' }, + ); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1, 2, 3), + 'image/png', + { clientId: session.clientId }, + ); + const queued = bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [reference] }, + undefined, + { clientId: session.clientId, promptId: 'media-delete-queued' }, + ); + void queued.catch(() => {}); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toHaveLength(2), + ); + + expect( + bridge.removePendingPrompt(session.sessionId, 'media-delete-queued', { + clientId: session.clientId, + }), + ).toEqual({ removed: true }); + expect( + await bridge.readSessionMedia(session.sessionId, reference.mediaId, { + clientId: session.clientId, + }), + ).toBeUndefined(); + + release(); + await running; + await expect(queued).rejects.toBeDefined(); + await bridge.shutdown(); + }); + it('skips an accepted queued prompt when its caller aborts before it starts', async () => { const events: BridgeEvent[] = []; let resolveFirst: (() => void) | undefined; @@ -17537,19 +17878,113 @@ describe('createAcpSessionBridge', () => { await bridge.shutdown(); }); - it('exit fired on planned shutdown does NOT trigger the unexpected-cleanup path', async () => { + it('reaps retained media after an unexpectedly closed session expires', async () => { + vi.useFakeTimers(); + const close = vi.spyOn(SessionMediaStore.prototype, 'close'); + const handle = makeChannel(); + const bridge = makeBridge({ + channelFactory: async () => handle.channel, + sessionReapIntervalMs: 1_000, + sessionIdleTimeoutMs: 0, + }); + try { + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1), + 'image/png', + { clientId: session.clientId }, + ); + + handle.crash(); + await vi.waitFor(() => expect(bridge.sessionCount).toBe(0)); + await vi.advanceTimersByTimeAsync(2 * 60 * 60 * 1_000); + expect(close).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60 * 60 * 1_000 + 1_000); + + expect(close).toHaveBeenCalledOnce(); + } finally { + await bridge.shutdown(); + close.mockRestore(); + vi.useRealTimers(); + } + }); + + it('does not reap retained media while a restore of the session is in flight', async () => { + // A restore registers inFlightRestores synchronously but lands in byId + // only after the async channel-spawn/loadSession gap. A sweep tick + // inside that gap must not release the media being restored, or the + // restored session non-deterministically loses every attachment. + vi.useFakeTimers(); const handles: ChannelHandle[] = []; + const load = deferred(); const factory: ChannelFactory = async () => { - const h = makeChannel(); + const h = makeChannel({ loadSessionImpl: () => load.promise }); handles.push(h); return h.channel; }; - const bridge = makeBridge({ channelFactory: factory }); - const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); - - // No subscribers; planned shutdown removes the entry first, THEN - // calls channel.kill() which resolves channel.exited. The cleanup - // .then() handler runs but sees byId.get(sessionId) === undefined + const bridge = makeBridge({ + channelFactory: factory, + sessionReapIntervalMs: 1_000, + sessionIdleTimeoutMs: 0, + sessionRestoreTimeoutMs: 4 * 60 * 60 * 1_000, + }); + try { + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1), + 'image/png', + { clientId: session.clientId }, + ); + + // Crash path: entry removed from byId, media retained with a + // detachedAt stamp. + handles[0]!.crash(); + await vi.waitFor(() => expect(bridge.sessionCount).toBe(0)); + + // Start the restore and hold it open inside the loadSession gap. + const restore = bridge.loadSession({ + sessionId: session.sessionId, + workspaceCwd: WS_A, + }); + await vi.waitFor(() => { + expect(handles).toHaveLength(2); + expect(handles[1]!.agent.loadSessionCalls).toHaveLength(1); + }); + + // Sweep ticks straddling the TTL boundary land inside the gap. + await vi.advanceTimersByTimeAsync(3 * 60 * 60 * 1_000 + 61_000); + + load.resolve({}); + await restore; + + // The retained media survived the restore. + const media = await bridge.readSessionMedia( + session.sessionId, + reference.mediaId, + ); + expect(media).toBeDefined(); + expect([...(media?.data ?? [])]).toEqual([1]); + } finally { + await bridge.shutdown(); + vi.useRealTimers(); + } + }); + + it('exit fired on planned shutdown does NOT trigger the unexpected-cleanup path', async () => { + const handles: ChannelHandle[] = []; + const factory: ChannelFactory = async () => { + const h = makeChannel(); + handles.push(h); + return h.channel; + }; + const bridge = makeBridge({ channelFactory: factory }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + + // No subscribers; planned shutdown removes the entry first, THEN + // calls channel.kill() which resolves channel.exited. The cleanup + // .then() handler runs but sees byId.get(sessionId) === undefined // (already removed), so it no-ops and doesn't double-publish. await bridge.shutdown(); @@ -23424,6 +23859,45 @@ describe('createAcpSessionBridge', () => { await bridge.shutdown(); }); + it('continues closing the session when media cleanup fails', async () => { + const handle = makeChannel(); + const bridge = makeBridge({ + channelFactory: async () => handle.channel, + channelIdleTimeoutMs: 0, + }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1), + 'image/png', + { clientId: session.clientId }, + ); + const closeMedia = vi + .spyOn(SessionMediaStore.prototype, 'close') + .mockRejectedValueOnce(new Error('cleanup failed')); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + try { + await expect(bridge.closeSession(session.sessionId)).resolves.toBe( + undefined, + ); + + expect(bridge.sessionCount).toBe(0); + expect(handle.agent.cancelCalls).toEqual([ + { sessionId: session.sessionId }, + ]); + expect(stderr).toHaveBeenCalledWith( + expect.stringContaining('failed to release media for closed session'), + ); + } finally { + closeMedia.mockRestore(); + stderr.mockRestore(); + await bridge.shutdown(); + } + }); + it('preserves bridge state when required agent close fails so retry can flush', async () => { let failClose = true; const handle = makeChannel({ @@ -26250,7 +26724,7 @@ describe('session idle reaper', () => { } }); - it('is disabled when sessionIdleTimeoutMs is 0', async () => { + it('does not reap active sessions when sessionIdleTimeoutMs is 0', async () => { vi.useFakeTimers(); try { const handle = makeChannel(); @@ -27207,6 +27681,105 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa await bridge.shutdown(); }); + it('removes uploaded media owned by a deleted mid-turn message', async () => { + const { factory, release } = hangingPromptFactory(); + const bridge = makeBridge({ channelFactory: factory }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const prompt = bridge.sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'go' }], + }, + undefined, + { clientId: session.clientId }, + ); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1, 2, 3), + 'image/png', + { clientId: session.clientId }, + ); + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'remove image', + { clientId: session.clientId }, + 'media-delete-mid-turn', + { content: [reference] }, + ), + ).toEqual({ accepted: true, messageId: 'media-delete-mid-turn' }); + + expect( + bridge.removeMidTurnMessage(session.sessionId, 'media-delete-mid-turn', { + clientId: session.clientId, + }), + ).toEqual({ removed: true }); + expect( + await bridge.readSessionMedia(session.sessionId, reference.mediaId, { + clientId: session.clientId, + }), + ).toBeUndefined(); + + release(); + await prompt; + await bridge.shutdown(); + }); + + it('acks a same-id retry after the queued media is removed', async () => { + const { factory, release } = hangingPromptFactory(); + const bridge = makeBridge({ channelFactory: factory }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const promptPromise = bridge + .sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'go' }], + }, + undefined, + { clientId: session.clientId }, + ) + .catch(() => {}); + await new Promise((r) => setTimeout(r, 10)); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1, 2, 3), + 'image/png', + { clientId: session.clientId }, + ); + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'look at this', + { clientId: session.clientId }, + 'retry-after-media-gone', + { content: [reference] }, + ), + ).toEqual({ accepted: true, messageId: 'retry-after-media-gone' }); + expect( + bridge.removeMidTurnMessage(session.sessionId, 'retry-after-media-gone', { + clientId: session.clientId, + }), + ).toEqual({ removed: true }); + + // The removal settled the id; a same-id retry must hit the settled ring + // and ack, not throw session_media_gone (410). + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'look at this', + { clientId: session.clientId }, + 'retry-after-media-gone', + { content: [reference] }, + ), + ).toEqual({ accepted: true, messageId: 'retry-after-media-gone' }); + + release(); + await promptPromise; + await bridge.shutdown(); + }); + it('does not recreate a deleted stable id when its admission is retried', async () => { const { factory, release } = hangingPromptFactory(); const bridge = makeBridge({ channelFactory: factory }); @@ -27380,6 +27953,18 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa ); expect(drained).toEqual({ messages: ['m1', 'm2'], + items: [ + { + messageId: expect.any(String), + displayText: 'm1', + content: [{ type: 'text', text: 'm1' }], + }, + { + messageId: expect.any(String), + displayText: 'm2', + content: [{ type: 'text', text: 'm2' }], + }, + ], hasQueuedPrompt: false, }); // Spliced out, so the next batch's drain is empty. @@ -27387,99 +27972,808 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa await handle.agentConnection.extMethod('craft/drainMidTurnQueue', { sessionId: session.sessionId, }), - ).toEqual({ messages: [], hasQueuedPrompt: false }); + ).toEqual({ messages: [], items: [], hasQueuedPrompt: false }); release?.(); await prompt; await bridge.shutdown(); }); - it('promotes every undrained message at settle', async () => { - const releases: Array<() => void> = []; + it('drains media blocks as structured items alongside the text', async () => { + let release: (() => void) | undefined; const handle = makeChannel({ promptImpl: async () => { await new Promise((r) => { - releases.push(r); + release = r; }); return { stopReason: 'end_turn' }; }, }); const bridge = makeBridge({ channelFactory: async () => handle.channel }); const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); - const send = (text: string) => - bridge - .sendPrompt( - session.sessionId, - { sessionId: session.sessionId, prompt: [{ type: 'text', text }] }, - undefined, - { clientId: session.clientId }, - ) - .catch(() => {}); - const t1 = send('t1'); + const prompt = bridge + .sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'go' }], + }, + undefined, + { clientId: session.clientId }, + ) + .catch(() => {}); await new Promise((r) => setTimeout(r, 10)); - const admission = bridge.enqueueMidTurnMessage( - session.sessionId, - 'leftover', - ); - expect(admission).toEqual({ - accepted: true, - messageId: expect.any(String), - }); - releases[0]!(); - await t1; - await vi.waitFor(() => expect(releases).toHaveLength(2)); - expect(bridge.getPendingPrompts(session.sessionId)).toEqual([ - expect.objectContaining({ - promptId: admission.messageId, - text: 'leftover', + const image = { + type: 'image', + data: 'aW1n', + mimeType: 'image/png', + } as const; + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'see this', + { clientId: session.clientId }, + 'media-1', + { content: [image, { type: 'text', text: 'sneaky' }] }, + ), + ).toEqual({ accepted: true, messageId: 'media-1' }); + // Empty text is admitted when media blocks are present. + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + ' ', + { clientId: session.clientId }, + 'media-2', + { content: [image] }, + ), + ).toEqual({ accepted: true, messageId: 'media-2' }); + // No text AND no media stays rejected. + expect( + bridge.enqueueMidTurnMessage(session.sessionId, '', undefined, 'nope', { + content: [], }), - ]); - releases[1]!(); - await vi.waitFor(() => - expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ).toEqual({ accepted: false }); + // Same-id retry with the SAME payload is idempotent... + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'see this', + { clientId: session.clientId }, + 'media-1', + { content: [image, { type: 'text', text: 'ignored' }] }, + ), + ).toEqual({ accepted: true, messageId: 'media-1' }); + // ...but a same-id retry with DIFFERENT media is rejected, so the + // original attachments can never be silently swapped. + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'see this', + { clientId: session.clientId }, + 'media-1', + { content: [{ ...image, data: 'T1RIRVI=' }] }, + ), + ).toEqual({ accepted: false }); + + const drained = await handle.agentConnection.extMethod( + 'craft/drainMidTurnQueue', + { sessionId: session.sessionId }, ); + expect(drained).toEqual({ + messages: ['see this', ''], + items: [ + { + messageId: 'media-1', + displayText: 'see this', + // Text blocks from `content` are dropped — the text rides `message`. + content: [{ type: 'text', text: 'see this' }, image], + }, + { messageId: 'media-2', displayText: '', content: [image] }, + ], + hasQueuedPrompt: false, + }); + + release?.(); + await prompt; await bridge.shutdown(); }); - it('promotes messages after their client detaches without reapplying the cap', async () => { - const prompts: string[] = []; + it('promotes a media message into the prompt FIFO with its blocks', async () => { + const prompts: unknown[] = []; const releases: Array<() => void> = []; const handle = makeChannel({ promptImpl: async (req) => { - prompts.push( - (req.prompt[0] as { text?: string } | undefined)?.text ?? '', - ); - await new Promise((resolve) => { - releases.push(resolve); + prompts.push(req.prompt); + await new Promise((r) => { + releases.push(r); }); return { stopReason: 'end_turn' }; }, }); - const bridge = makeBridge({ - channelFactory: async () => handle.channel, - maxPendingPromptsPerSession: 1, - }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); - const first = bridge.sendPrompt( - session.sessionId, - { - sessionId: session.sessionId, - prompt: [{ type: 'text', text: 'first' }], - }, - undefined, - { clientId: session.clientId, promptId: 'first-id' }, - ); - await vi.waitFor(() => expect(prompts).toEqual(['first'])); + const image = { + type: 'image', + data: 'aW1n', + mimeType: 'image/png', + } as const; + // Idle session: the message is promoted straight into the FIFO. expect( bridge.enqueueMidTurnMessage( session.sessionId, - 'follow up', - { clientId: session.clientId }, - 'webui-follow-up', + 'with media', + undefined, + 'media-promo-1', + { content: [image] }, ), - ).toEqual({ accepted: true, messageId: 'webui-follow-up' }); + ).toEqual({ accepted: true, messageId: 'media-promo-1' }); + await vi.waitFor(() => expect(prompts).toHaveLength(1)); + expect(prompts[0]).toEqual([{ type: 'text', text: 'with media' }, image]); + releases[0]!(); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + + // Image-only promotion keeps the prompt media-only. + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + '', + undefined, + 'media-promo-2', + { content: [image] }, + ), + ).toEqual({ accepted: true, messageId: 'media-promo-2' }); + await vi.waitFor(() => expect(prompts).toHaveLength(2)); + expect(prompts[1]).toEqual([image]); + releases[1]!(); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + await bridge.shutdown(); + }); + + it('ignores legacy audio when matching a promoted mid-turn retry', async () => { + const done = deferred(); + const handle = makeChannel({ + promptImpl: async () => { + await done.promise; + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const prompt = bridge + .sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [ + { type: 'text', text: 'listen' }, + { type: 'audio', data: 'YXVkaW8=', mimeType: 'audio/wav' }, + ], + }, + undefined, + { promptId: 'promoted-audio' }, + ) + .catch(() => {}); + await vi.waitFor(() => expect(handle.agent.promptCalls).toHaveLength(1)); + + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'listen', + undefined, + 'promoted-audio', + ), + ).toEqual({ accepted: true, messageId: 'promoted-audio' }); + + done.resolve(); + await prompt; + await bridge.shutdown(); + }); + + it('promotes an undrained media message at settle with its blocks', async () => { + const prompts: unknown[] = []; + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push(req.prompt); + await new Promise((r) => { + releases.push(r); + }); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const t1 = bridge + .sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 't1' }], + }, + undefined, + { clientId: session.clientId }, + ) + .catch(() => {}); + await new Promise((r) => setTimeout(r, 10)); + + const image = { + type: 'image', + data: 'aW1n', + mimeType: 'image/png', + } as const; + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'leftover media', + undefined, + 'media-leftover', + { content: [image] }, + ), + ).toEqual({ accepted: true, messageId: 'media-leftover' }); + + // The turn settles without draining: the message promotes into the FIFO + // and must keep its media blocks. + releases[0]!(); + await t1; + await vi.waitFor(() => expect(prompts).toHaveLength(2)); + expect(prompts[1]).toEqual([ + { type: 'text', text: 'leftover media' }, + image, + ]); + releases[1]!(); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + await bridge.shutdown(); + }); + + it('degrades expired media at settle without skipping later messages', async () => { + const prompts: unknown[] = []; + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push(req.prompt); + await new Promise((resolve) => releases.push(resolve)); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const first = bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [{ type: 'text', text: 't1' }] }, + undefined, + { clientId: session.clientId }, + ); + await vi.waitFor(() => expect(prompts).toHaveLength(1)); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1), + 'image/png', + { clientId: session.clientId }, + ); + bridge.enqueueMidTurnMessage( + session.sessionId, + 'expired attachment', + { clientId: session.clientId }, + 'expired-media', + { content: [reference] }, + ); + bridge.enqueueMidTurnMessage( + session.sessionId, + 'still runs', + { clientId: session.clientId }, + 'normal-message', + ); + await bridge.removeSessionMedia(session.sessionId, reference.mediaId, { + clientId: session.clientId, + }); + + releases[0]!(); + await first; + await vi.waitFor(() => expect(prompts).toHaveLength(2)); + expect(prompts[1]).toEqual([ + { + type: 'text', + text: 'expired attachment\n[Attached media is no longer available]', + }, + ]); + releases[1]!(); + await vi.waitFor(() => expect(prompts).toHaveLength(3)); + expect(prompts[2]).toEqual([{ type: 'text', text: 'still runs' }]); + releases[2]!(); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + await bridge.shutdown(); + }); + + it('keeps resolvable siblings when one reference is removed before settle', async () => { + // The settle-time degrade must drop only the removed reference: the + // sibling the store still holds reaches the model instead of being + // swallowed by a wholesale marker. + const prompts: unknown[] = []; + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push(req.prompt); + await new Promise((resolve) => releases.push(resolve)); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const first = bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [{ type: 'text', text: 't1' }] }, + undefined, + { clientId: session.clientId }, + ); + await vi.waitFor(() => expect(prompts).toHaveLength(1)); + const kept = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1, 2), + 'image/png', + { clientId: session.clientId }, + ); + const removed = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(3, 4), + 'image/png', + { clientId: session.clientId }, + ); + bridge.enqueueMidTurnMessage( + session.sessionId, + 'two images', + { clientId: session.clientId }, + 'mixed-media', + { content: [removed, kept] }, + ); + await bridge.removeSessionMedia(session.sessionId, removed.mediaId, { + clientId: session.clientId, + }); + + releases[0]!(); + await first; + await vi.waitFor(() => expect(prompts).toHaveLength(2)); + expect(prompts[1]).toEqual([ + { + type: 'text', + text: 'two images\n[Attached media is no longer available]', + }, + { type: 'image', data: 'AQI=', mimeType: 'image/png' }, + ]); + releases[1]!(); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + await bridge.shutdown(); + }); + + it('keeps a resolvable sibling when one reference dies between admission and dispatch', async () => { + const prompts: unknown[] = []; + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push(req.prompt); + await new Promise((resolve) => releases.push(resolve)); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const first = bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [{ type: 'text', text: 'p1' }] }, + undefined, + { clientId: session.clientId, promptId: 'prompt-p1' }, + ); + await vi.waitFor(() => expect(prompts).toHaveLength(1)); + const kept = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1, 2), + 'image/png', + { clientId: session.clientId }, + ); + const removed = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(3, 4), + 'image/png', + { clientId: session.clientId }, + ); + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'm1', + { clientId: session.clientId }, + 'mid-m1', + ), + ).toEqual({ accepted: true, messageId: 'mid-m1' }); + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'm2', + { clientId: session.clientId }, + 'mid-m2', + { content: [removed, kept] }, + ), + ).toEqual({ accepted: true, messageId: 'mid-m2' }); + + releases[0]!(); + await first; + // m1 promoted to running; m2 admitted queued behind it. + await vi.waitFor(() => expect(prompts).toHaveLength(2)); + + await bridge.removeSessionMedia(session.sessionId, removed.mediaId, { + clientId: session.clientId, + }); + + releases[1]!(); + await vi.waitFor(() => expect(prompts).toHaveLength(3)); + // The dispatch-time degrade drops only the dead reference; the stored + // sibling still reaches the model. + expect(prompts[2]).toEqual([ + { + type: 'text', + text: 'm2\n[Attached media is no longer available]', + }, + { type: 'image', data: 'AQI=', mimeType: 'image/png' }, + ]); + releases[2]!(); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + await bridge.shutdown(); + }); + + it('rejects queued inline media past the session byte budget', async () => { + // Inline base64 never passes through the media store, so admission must + // budget the aggregate queued inline bytes per session instead of + // letting a full queue pin hundreds of MiB. + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async () => { + await new Promise((resolve) => releases.push(resolve)); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const busy = bridge + .sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 't1' }], + }, + undefined, + { clientId: session.clientId }, + ) + .catch(() => {}); + await new Promise((r) => setTimeout(r, 10)); + + const half = 'x'.repeat(SESSION_MEDIA_MAX_TOTAL_BYTES / 2 + 1024); + const inline = { + type: 'image', + data: half, + mimeType: 'image/png', + } as const; + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'first half', + { clientId: session.clientId }, + 'inline-1', + { content: [inline] }, + ), + ).toEqual({ accepted: true, messageId: 'inline-1' }); + // The second payload pushes the queued inline total past the budget. + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'second half', + { clientId: session.clientId }, + 'inline-2', + { content: [inline] }, + ), + ).toEqual({ accepted: false }); + // A small inline payload stays admissible under the budget. + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'small', + { clientId: session.clientId }, + 'inline-3', + { + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }, + ), + ).toEqual({ accepted: true, messageId: 'inline-3' }); + + releases[0]!(); + await busy; + await bridge.shutdown(); + }); + + it('degrades media removed between admission and dispatch in place, keeping FIFO order and one terminal', async () => { + const prompts: unknown[] = []; + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push(req.prompt); + await new Promise((resolve) => releases.push(resolve)); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const events: BridgeEvent[] = []; + const sub = (async () => { + for await (const ev of bridge.subscribeEvents(session.sessionId)) { + events.push(ev); + } + })(); + sub.catch(() => {}); + + const first = bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [{ type: 'text', text: 'p1' }] }, + undefined, + { clientId: session.clientId, promptId: 'prompt-p1' }, + ); + await vi.waitFor(() => expect(prompts).toHaveLength(1)); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1), + 'image/png', + { clientId: session.clientId }, + ); + // Two undrained messages: at settle the first promotes to running and + // this one is admitted QUEUED behind it — the window in which its media + // can disappear after admission but before dispatch. + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'm1', + { clientId: session.clientId }, + 'mid-m1', + ), + ).toEqual({ accepted: true, messageId: 'mid-m1' }); + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'm2', + { clientId: session.clientId }, + 'mid-m2', + { content: [reference] }, + ), + ).toEqual({ accepted: true, messageId: 'mid-m2' }); + + releases[0]!(); + await first; + await vi.waitFor(() => expect(prompts).toHaveLength(2)); + + await bridge.removeSessionMedia(session.sessionId, reference.mediaId, { + clientId: session.clientId, + }); + // An ordinary prompt admitted after the media message must stay behind it. + const p3 = bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [{ type: 'text', text: 'p3' }] }, + undefined, + { clientId: session.clientId, promptId: 'prompt-p3' }, + ); + + releases[1]!(); + // The degraded prompt keeps its FIFO position instead of re-admitting at + // the tail behind p3. + await vi.waitFor(() => expect(prompts).toHaveLength(3)); + expect(prompts[2]).toEqual([ + { + type: 'text', + text: 'm2\n[Attached media is no longer available]', + }, + ]); + releases[2]!(); + await vi.waitFor(() => expect(prompts).toHaveLength(4)); + expect(prompts[3]).toEqual([{ type: 'text', text: 'p3' }]); + releases[3]!(); + await p3; + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + + // One terminal per promptId, and the degraded turn completes normally — + // a re-admitted fallback would publish turn_error then a second terminal. + const terminalsFor = (promptId: string) => + events.filter( + (e) => + (e.type === 'turn_complete' || e.type === 'turn_error') && + e.promptId === promptId, + ); + await vi.waitFor(() => { + for (const promptId of ['prompt-p1', 'mid-m1', 'mid-m2', 'prompt-p3']) { + const terms = terminalsFor(promptId); + expect(terms).toHaveLength(1); + expect(terms[0]?.type).toBe('turn_complete'); + } + // No second admission: exactly one started event for the degraded turn. + expect( + events.filter( + (e) => e.type === 'pending_prompt_started' && e.promptId === 'mid-m2', + ), + ).toHaveLength(1); + }); + + await bridge.shutdown(); + }); + + it('degrades in place on a fully detached session instead of racing the deferred close', async () => { + const prompts: unknown[] = []; + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push(req.prompt); + await new Promise((resolve) => releases.push(resolve)); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const first = bridge.sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [{ type: 'text', text: 'p1' }] }, + undefined, + { clientId: session.clientId, promptId: 'prompt-p1' }, + ); + await vi.waitFor(() => expect(prompts).toHaveLength(1)); + const reference = await bridge.storeSessionMedia( + session.sessionId, + Uint8Array.of(1), + 'image/png', + { clientId: session.clientId }, + ); + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'm1', + { clientId: session.clientId }, + 'detached-m1', + ), + ).toEqual({ accepted: true, messageId: 'detached-m1' }); + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'm2', + { clientId: session.clientId }, + 'detached-m2', + { content: [reference] }, + ), + ).toEqual({ accepted: true, messageId: 'detached-m2' }); + + // Fully detached from here: no attached clients and no subscribers. + await bridge.detachClient(session.sessionId, session.clientId); + + releases[0]!(); + await first; + // m1 promoted to running, m2 admitted queued behind it. + await vi.waitFor(() => expect(prompts).toHaveLength(2)); + + // Media disappears between admission and dispatch (async arm). + await bridge.removeSessionMedia(session.sessionId, reference.mediaId); + + releases[1]!(); + // The degraded prompt must still reach the child — a re-admitted + // fallback loses the race with the deferred close-on-prompt-complete + // on a detached session and is swallowed by `session_closing`. + await vi.waitFor(() => expect(prompts).toHaveLength(3)); + expect(prompts[2]).toEqual([ + { + type: 'text', + text: 'm2\n[Attached media is no longer available]', + }, + ]); + releases[2]!(); + await vi.waitFor(() => + expect(() => bridge.getSessionSummary(session.sessionId)).toThrow( + SessionNotFoundError, + ), + ); + await bridge.shutdown(); + }); + + it('promotes every undrained message at settle', async () => { + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async () => { + await new Promise((r) => { + releases.push(r); + }); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const send = (text: string) => + bridge + .sendPrompt( + session.sessionId, + { sessionId: session.sessionId, prompt: [{ type: 'text', text }] }, + undefined, + { clientId: session.clientId }, + ) + .catch(() => {}); + const t1 = send('t1'); + await new Promise((r) => setTimeout(r, 10)); + const admission = bridge.enqueueMidTurnMessage( + session.sessionId, + 'leftover', + ); + expect(admission).toEqual({ + accepted: true, + messageId: expect.any(String), + }); + releases[0]!(); + await t1; + + await vi.waitFor(() => expect(releases).toHaveLength(2)); + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([ + expect.objectContaining({ + promptId: admission.messageId, + text: 'leftover', + }), + ]); + releases[1]!(); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + await bridge.shutdown(); + }); + + it('promotes messages after their client detaches without reapplying the cap', async () => { + const prompts: string[] = []; + const releases: Array<() => void> = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push( + (req.prompt[0] as { text?: string } | undefined)?.text ?? '', + ); + await new Promise((resolve) => { + releases.push(resolve); + }); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ + channelFactory: async () => handle.channel, + maxPendingPromptsPerSession: 1, + }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const first = bridge.sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'first' }], + }, + undefined, + { clientId: session.clientId, promptId: 'first-id' }, + ); + await vi.waitFor(() => expect(prompts).toEqual(['first'])); + + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'follow up', + { clientId: session.clientId }, + 'webui-follow-up', + ), + ).toEqual({ accepted: true, messageId: 'webui-follow-up' }); expect( bridge.enqueueMidTurnMessage( session.sessionId, @@ -27653,7 +28947,7 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa await handle.agentConnection.extMethod('craft/drainMidTurnQueue', { sessionId: session.sessionId, }), - ).toEqual({ messages: [], hasQueuedPrompt: true }); + ).toEqual({ messages: [], items: [], hasQueuedPrompt: true }); expect( bridge.enqueueMidTurnMessage(session.sessionId, 'x', { clientId: session.clientId, @@ -27670,7 +28964,17 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa await handle.agentConnection.extMethod('craft/drainMidTurnQueue', { sessionId: session.sessionId, }), - ).toEqual({ messages: ['x'], hasQueuedPrompt: false }); + ).toEqual({ + messages: ['x'], + items: [ + { + messageId: expect.any(String), + displayText: 'x', + content: [{ type: 'text', text: 'x' }], + }, + ], + hasQueuedPrompt: false, + }); expect((await injected).promptId).toBe('prompt-2'); releases[1]!(); @@ -27829,7 +29133,17 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa 'craft/drainMidTurnQueue', { sessionId: session.sessionId }, ); - expect(drained).toEqual({ messages: ['hi'], hasQueuedPrompt: false }); + expect(drained).toEqual({ + messages: ['hi'], + items: [ + { + messageId: admission.messageId, + displayText: 'hi', + content: [{ type: 'text', text: 'hi' }], + }, + ], + hasQueuedPrompt: false, + }); const it = iter[Symbol.asyncIterator](); const next = await it.next(); @@ -27983,6 +29297,13 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa ); expect(drained).toEqual({ messages: ['hello'], + items: [ + { + messageId: expect.any(String), + displayText: 'hello', + content: [{ type: 'text', text: 'hello' }], + }, + ], hasQueuedPrompt: false, }); @@ -28057,6 +29378,124 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa await bridge.shutdown(); }); + it('getMidTurnMessages returns media blocks so a refresh keeps images', async () => { + let release: (() => void) | undefined; + const handle = makeChannel({ + promptImpl: async () => { + await new Promise((r) => { + release = r; + }); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const prompt = bridge + .sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'go' }], + }, + undefined, + { clientId: session.clientId }, + ) + .catch(() => {}); + await new Promise((r) => setTimeout(r, 10)); + + const image = { + type: 'image', + data: 'aW1n', + mimeType: 'image/png', + } as const; + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + 'with media', + { clientId: session.clientId }, + 'media-snap', + { content: [image] }, + ), + ).toEqual({ accepted: true, messageId: 'media-snap' }); + + expect( + bridge.getMidTurnMessages(session.sessionId, { + clientId: session.clientId, + }), + ).toEqual({ + messages: [ + { + messageId: 'media-snap', + text: 'with media', + content: [image], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + + release?.(); + await prompt; + await bridge.shutdown(); + }); + + it('getPendingPrompts returns media blocks so a refresh keeps images', async () => { + let release: (() => void) | undefined; + const handle = makeChannel({ + promptImpl: async () => { + await new Promise((r) => { + release = r; + }); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + + const image = { + type: 'image', + data: 'aW1n', + mimeType: 'image/png', + } as const; + + // Start first prompt that will hang + bridge.sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'go' }], + }, + undefined, + { clientId: session.clientId }, + ); + + // Send second prompt with media - it should be queued + bridge.sendPrompt( + session.sessionId, + { + sessionId: session.sessionId, + prompt: [{ type: 'text', text: 'with media' }, image], + }, + undefined, + { clientId: session.clientId, promptId: 'media-prompt' }, + ); + + // Wait for both prompts to be in the queue + await new Promise((r) => setTimeout(r, 20)); + const pending = bridge.getPendingPrompts(session.sessionId); + expect(pending).toHaveLength(2); + + const mediaEntry = pending.find((p) => p.promptId === 'media-prompt'); + expect(mediaEntry).toMatchObject({ + promptId: 'media-prompt', + text: 'with media', + content: [image], + }); + + release?.(); + await bridge.shutdown(); + }); + it('getMidTurnMessages moves drained stable ids into the settled ring', async () => { // The reconciliation contract: after a drain, a client that missed the // SSE echo (or refreshed the page) sees its messageId under @@ -28171,6 +29610,50 @@ describe('createAcpSessionBridge — mid-turn message queue (enqueueMidTurnMessa await bridge.shutdown(); }); + it('answers an image-only retry from the promoted ring without running it twice', async () => { + const prompts: unknown[] = []; + const handle = makeChannel({ + promptImpl: async (req) => { + prompts.push(req.prompt); + return { stopReason: 'end_turn' }; + }, + }); + const bridge = makeBridge({ channelFactory: async () => handle.channel }); + const session = await bridge.spawnOrAttach({ workspaceCwd: WS_A }); + const image = { + type: 'image' as const, + data: 'AQID', + mimeType: 'image/png', + }; + + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + '', + { clientId: session.clientId }, + 'stable-image-promoted', + { content: [image] }, + ), + ).toEqual({ accepted: true, messageId: 'stable-image-promoted' }); + await vi.waitFor(() => expect(prompts).toHaveLength(1)); + await vi.waitFor(() => + expect(bridge.getPendingPrompts(session.sessionId)).toEqual([]), + ); + + expect( + bridge.enqueueMidTurnMessage( + session.sessionId, + '', + { clientId: session.clientId }, + 'stable-image-promoted', + { content: [image] }, + ), + ).toEqual({ accepted: true, messageId: 'stable-image-promoted' }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(prompts).toHaveLength(1); + await bridge.shutdown(); + }); + it('promotes multiple undrained messages at settle in FIFO order', async () => { const prompts: string[] = []; const releases: Array<() => void> = []; diff --git a/packages/acp-bridge/src/bridge.ts b/packages/acp-bridge/src/bridge.ts index 1b33025cf71..a91a60ce744 100644 --- a/packages/acp-bridge/src/bridge.ts +++ b/packages/acp-bridge/src/bridge.ts @@ -15,7 +15,9 @@ import { import type { CancelNotification, Client, + ContentBlock, PromptRequest, + PromptResponse, SetSessionConfigOptionRequest, SetSessionConfigOptionResponse, SetSessionModelRequest, @@ -134,6 +136,7 @@ import { CHANNEL_STARTUP_PROFILE_META_KEY, CHANNEL_STARTUP_PROFILE_VERSION, DAEMON_CHANNEL_DELIVERY_META_KEY, + DAEMON_MEDIA_REFERENCES_META_KEY, DAEMON_MODEL_PROMPT_META_KEY, DAEMON_PROMPT_DISPLAY_TEXT_META_KEY, LOAD_REPLAY_BULK_MODE, @@ -179,9 +182,18 @@ import type { BridgeSessionTranscriptPageRequest, BridgeGenerationStreamEvent, BridgeWorkspaceGenerationStreamEvent, + BridgePromptContentBlock, + BridgePromptRequest, RuntimeMcpServerAddResult, RuntimeMcpServerRemoveResult, } from './bridgeTypes.js'; +import { + isSessionMediaReference, + SESSION_MEDIA_MAX_TOTAL_BYTES, + SessionMediaReferenceError, + SessionMediaStore, + withMediaDegradationMarker, +} from './sessionMedia.js'; import type { BridgeFreshSessionAdmissionContext, BridgeFreshSessionReservation, @@ -938,6 +950,8 @@ interface SessionEntry { events: EventBus; /** Per-session structured artifact registry. */ artifacts: SessionArtifactStore; + /** Session-owned temporary media referenced by prompts and SSE events. */ + media: SessionMediaStore; /** Sticky in-memory health state for the session's transcript recorder. */ recordingDegraded: boolean; /** Set synchronously while agent-owned state and its writer lease close. */ @@ -1394,7 +1408,7 @@ function pickUserInputEchoMeta(meta: unknown): Record { */ function echoPromptToSessionBus( entry: SessionEntry, - req: PromptRequest, + req: BridgePromptRequest, promptId: string, originatorClientId: string | undefined, displayText: string | undefined, @@ -1959,25 +1973,45 @@ function hasControlCharacter(value: string): boolean { * placeholder for image-only prompts. */ function extractPromptText( - prompt: ReadonlyArray>, + prompt: readonly BridgePromptContentBlock[], ): string { if (!Array.isArray(prompt)) return ''; let hasImage = false; for (const block of prompt) { - if (block['type'] === 'image') { + const record = block as unknown as Record; + if (record['type'] === 'image') { hasImage = true; } if ( - block['type'] === 'text' && - typeof block['text'] === 'string' && - block['text'].length > 0 + record['type'] === 'text' && + typeof record['text'] === 'string' && + record['text'].length > 0 ) { - return block['text']; + return record['text']; } } return hasImage ? '[image]' : ''; } +/** + * Extract inline media content blocks from a prompt for storage in the + * pending-prompt queue. Image references use the session media store; legacy + * audio blocks remain inline. This lets refreshed clients restore the payload. + */ +function extractMediaBlocks( + prompt: readonly BridgePromptContentBlock[], +): BridgePromptContentBlock[] | undefined { + if (!Array.isArray(prompt)) return undefined; + const media: BridgePromptContentBlock[] = []; + for (const block of prompt) { + if (!block || typeof block !== 'object') continue; + if (block.type === 'image' || block.type === 'audio') { + media.push(block); + } + } + return media.length > 0 ? media : undefined; +} + const DEFAULT_INIT_TIMEOUT_MS = 10_000; const PERSIST_TIMEOUT_MS = 5_000; // Bounded retries for the sub-session `parentSessionId` transcript write on the @@ -2019,6 +2053,21 @@ const MAX_SHELL_OUTPUT_FOR_HISTORY = 10_000; // a `BridgeOptions` knob the same way `maxPendingPromptsPerSession` (the // analogous bound `/prompt` enforces, default 5) is wired. const MAX_MID_TURN_QUEUE_DEPTH = 20; +// Inline base64 blocks never pass through the media store, so the store's +// caps never see them: bound the aggregate queued inline bytes instead, or a +// full queue of body-limit-sized messages pins hundreds of MiB per session +// and re-embeds them into every reconciliation snapshot. +const inlineMediaBlockBytes = ( + blocks: readonly BridgePromptContentBlock[], +): number => { + let total = 0; + for (const block of blocks) { + if (block.type === 'image' && 'data' in block) { + total += block.data.length; + } + } + return total; +}; const DEFAULT_MAX_SESSIONS = 32; // Keep in sync with CLI serve/server.ts and SDK DaemonClient.ts. const DEFAULT_MAX_PENDING_PROMPTS_PER_SESSION = 5; @@ -2051,6 +2100,8 @@ const DEFAULT_PERMISSION_TIMEOUT_MS = 5 * 60 * 1000; const DEFAULT_MAX_PENDING_PER_SESSION = 64; const DEFAULT_SESSION_REAP_INTERVAL_MS = 60_000; const DEFAULT_SESSION_IDLE_TIMEOUT_MS = 30 * 60_000; +const MAX_RETAINED_SESSION_MEDIA_BYTES = 512 * 1024 * 1024; +const RETAINED_SESSION_MEDIA_TTL_MS = 3 * 60 * 60_000; export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { let liveScreenContextCaptureHandler: @@ -2336,6 +2387,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { DEFAULT_SESSION_IDLE_TIMEOUT_MS, ); let sessionReaper: ReturnType | undefined; + let mediaSweeper: ReturnType | undefined; // Tracks the most recent "activity" event for idle-detection by // external schedulers. Updated on prompt start/end and session @@ -2961,9 +3013,44 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { } } + function sweepRetainedMedia(): void { + const now = Date.now(); + for (const [id] of retainedMedia) { + // A restore registers `inFlightRestores` synchronously but lands in + // `byId` only after its async channel-spawn/loadSession gap; sweeping + // inside that gap would delete the media mid-restore. + if (byId.has(id) || inFlightRestores.has(id)) continue; + const detachedAt = retainedMediaDetachedAt.get(id); + if ( + detachedAt === undefined || + now - detachedAt < RETAINED_SESSION_MEDIA_TTL_MS + ) { + continue; + } + void releaseSessionMedia(id).catch((error) => { + writeStderrLine( + `qwen serve: failed to release retained media for ${JSON.stringify(id)}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + } + } + function startSessionReaper(): void { - if (sessionReapIntervalMs <= 0 || sessionIdleTimeoutMs <= 0) { - writeStderrLine('qwen serve: session reaper disabled'); + if (sessionReapIntervalMs <= 0) { + // Idle-session reaping is off, but the retained-media TTL sweep must + // still run: it is the only runtime release path for detach-retained + // media, and detach-driven auto-close keeps producing it. + writeStderrLine( + 'qwen serve: session reaper disabled; retained media sweep runs ' + + `every ${DEFAULT_SESSION_REAP_INTERVAL_MS}ms`, + ); + mediaSweeper = setInterval(() => { + if (shuttingDown) return; + sweepRetainedMedia(); + }, DEFAULT_SESSION_REAP_INTERVAL_MS); + mediaSweeper.unref(); return; } writeStderrLine( @@ -2975,6 +3062,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { if (shuttingDown) return; const now = Date.now(); for (const [id, entry] of byId) { + if (sessionIdleTimeoutMs <= 0) break; // Shared guards first (`pendingPromptCount` rather than `promptActive`, // so queued prompts and the FIFO hand-off gap between two prompts also // block the reap), then the reaper's own TTL policy on top. @@ -3002,6 +3090,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { closeReason: 'idle_timeout', }); } + sweepRetainedMedia(); }, sessionReapIntervalMs); sessionReaper.unref(); } @@ -3011,6 +3100,10 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { clearInterval(sessionReaper); sessionReaper = undefined; } + if (mediaSweeper !== undefined) { + clearInterval(mediaSweeper); + mediaSweeper = undefined; + } } // BkUyD: superset of `channelInfo` covering channels @@ -3033,6 +3126,23 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { // daemon. Cleared in the `finally` of the creator. let inFlightChannelSpawn: Promise | undefined; const byId = new Map(); + const retainedMedia = new Map(); + const retainedMediaDetachedAt = new Map(); + let mediaPutQueue: Promise = Promise.resolve(); + const retainSessionMedia = ( + sessionId: string, + store = retainedMedia.get(sessionId) ?? new SessionMediaStore(), + ): SessionMediaStore => { + retainedMedia.set(sessionId, store); + return store; + }; + const releaseSessionMedia = async (sessionId: string): Promise => { + const store = retainedMedia.get(sessionId); + if (!store) return; + retainedMedia.delete(sessionId); + retainedMediaDetachedAt.delete(sessionId); + await store.close(); + }; const forwardRunningPromptCancel = async ( entry: SessionEntry, pending: PendingPromptEntry, @@ -3769,6 +3879,9 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { touchActivity(); } byId.delete(sid); + if (retainedMedia.has(sid)) { + retainedMediaDetachedAt.set(sid, Date.now()); + } telemetry.metrics?.sessionLifecycle('die'); emitSessionLifecycle({ type: 'removed', @@ -5386,6 +5499,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { workspaceCwd, persistence: createSessionArtifactPersistence(ci.connection, sessionId), }), + media: retainSessionMedia(sessionId), recordingDegraded: false, closing: false, cwdChangeQueue: Promise.resolve(), @@ -6768,6 +6882,9 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { const restoreEntry = byId.get(req.sessionId); if (restoreEntry?.events === restoreEvents) { byId.delete(req.sessionId); + if (retainedMedia.has(req.sessionId)) { + retainedMediaDetachedAt.set(req.sessionId, Date.now()); + } ci?.sessionIds.delete(req.sessionId); emitSessionLifecycle({ type: 'removed', @@ -6982,6 +7099,15 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { // `session_closed` is terminal. Close the bus before ACP cancel so any // late cancellation frames from the agent are intentionally dropped. entry.events.close(); + if (reason === 'last_client_detached' || reason === 'idle_timeout') { + retainedMediaDetachedAt.set(sessionId, Date.now()); + } else { + await releaseSessionMedia(sessionId).catch((error) => { + writeStderrLine( + `qwen serve: failed to release media for closed session ${JSON.stringify(sessionId)}: ${error instanceof Error ? error.message : String(error)}`, + ); + }); + } if (!agentSessionClosed) { try { await telemetry.withSpan( @@ -7014,36 +7140,98 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { } }; + const removeQueuedMedia = ( + entry: SessionEntry, + content: readonly BridgePromptContentBlock[] | undefined, + ) => { + for (const block of content ?? []) { + if (!isSessionMediaReference(block)) continue; + void entry.media.remove(block.mediaId).catch((error) => { + writeStderrLine( + `[session-media] session=${JSON.stringify(entry.sessionId)} failed to remove queued media ${JSON.stringify(block.mediaId)}: ${JSON.stringify(error instanceof Error ? error.message : String(error))}`, + ); + }); + } + }; + const promoteMidTurnMessage = ( entry: SessionEntry, messageId: string, text: string, originatorClientId?: string, + content?: readonly BridgePromptContentBlock[], ) => { - void bridgeApi - .sendPrompt( + // Drop references that are already gone BEFORE admission: the admission + // check throws on the first dead reference, and the fallback would then + // replace the ENTIRE prompt with the marker, discarding the siblings the + // store still holds. + const resolvableBlocks: BridgePromptContentBlock[] = []; + let degraded = 0; + for (const block of content ?? []) { + try { + entry.media.assertReference(block); + resolvableBlocks.push(block); + } catch (error) { + if (!(error instanceof SessionMediaReferenceError)) throw error; + degraded += 1; + } + } + let prompt: BridgePromptContentBlock[] = [ + ...(text ? [{ type: 'text', text } as ContentBlock] : []), + ...resolvableBlocks, + ]; + if (degraded > 0) { + prompt = withMediaDegradationMarker(prompt); + } + const context = { + promptId: messageId, + promotedMidTurn: { originatorClientId }, + onPromptAdmitted: () => { + rememberMidTurnId(entry.promotedMidTurnMessageIds, messageId); + }, + }; + const sendFallback = () => + bridgeApi.sendPrompt( entry.sessionId, { sessionId: entry.sessionId, - prompt: [{ type: 'text', text }], + prompt: withMediaDegradationMarker( + text ? [{ type: 'text', text } as ContentBlock] : [], + ), }, undefined, + context, + ); + let result: Promise; + try { + result = bridgeApi.sendPrompt( + entry.sessionId, { - promptId: messageId, - promotedMidTurn: { originatorClientId }, - // Record the id only once sendPrompt owns its FIFO slot: an - // admission failure must not land the id in the reconciliation - // ring, or retries would be acked for a message that never runs. - onPromptAdmitted: () => { - rememberMidTurnId(entry.promotedMidTurnMessageIds, messageId); - }, + sessionId: entry.sessionId, + prompt, }, - ) - .catch((error: unknown) => { + undefined, + context, + ); + } catch (error) { + try { + if (!(error instanceof SessionMediaReferenceError)) throw error; + result = sendFallback(); + } catch (fallbackError) { writeStderrLine( - `[mid-turn] session=${JSON.stringify(entry.sessionId)} failed to run promoted message ${JSON.stringify(messageId)}: ${JSON.stringify(error instanceof Error ? error.message : String(error))}`, + `[mid-turn] session=${JSON.stringify(entry.sessionId)} failed to run promoted message ${JSON.stringify(messageId)}: ${JSON.stringify(fallbackError instanceof Error ? fallbackError.message : String(fallbackError))}`, ); - }); + return; + } + } + // SessionMediaReferenceError can no longer reject this result + // asynchronously: admission-time reference checks throw synchronously + // (handled above) and dispatch degrades in place. + void result.catch((error: unknown) => { + writeStderrLine( + `[mid-turn] session=${JSON.stringify(entry.sessionId)} failed to run promoted message ${JSON.stringify(messageId)}: ${JSON.stringify(error instanceof Error ? error.message : String(error))}`, + ); + }); }; const bridgeApi: AcpSessionBridge = { @@ -7579,11 +7767,17 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { ), ); } + if (!Array.isArray(req.prompt)) { + return Promise.reject( + RequestError.invalidParams(undefined, 'Prompt must be an array'), + ); + } const promotedMidTurn = context?.promotedMidTurn; const isPromotedMidTurn = promotedMidTurn !== undefined; const originatorClientId = promotedMidTurn ? promotedMidTurn.originatorClientId : resolveTrustedClientId(entry, context?.clientId); + entry.media.assertReferences(req.prompt); const modelPrompt = context?.modelPrompt; if ( modelPrompt !== undefined && @@ -7662,6 +7856,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { ...(originatorClientId !== undefined ? { originatorClientId } : {}), ...(isPromotedMidTurn ? { promotedMidTurn: true } : {}), text: pendingText, + content: extractMediaBlocks(req.prompt), abortController: pendingAbort, state: isQueued ? 'queued' : 'running', }; @@ -7831,10 +8026,40 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { : {}), }, async () => { + // Degrade in place, never by re-admission: a fallback + // re-admitted at the FIFO tail would double the terminal + // for this promptId, transpose the turn behind later + // prompts, and race the deferred close-on-prompt-complete + // on a detached session. + let dispatchBlocks = req.prompt; + let resolvedPrompt: ContentBlock[]; + try { + resolvedPrompt = + await entry.media.resolveContent(dispatchBlocks); + } catch (error) { + if ( + !isPromotedMidTurn || + !(error instanceof SessionMediaReferenceError) + ) { + throw error; + } + // Degrade per block: one dead reference drops itself and + // keeps its resolvable siblings instead of replacing the + // whole prompt with the marker. + const perBlock = + await entry.media.resolveContentDegrading(dispatchBlocks); + dispatchBlocks = perBlock.retainedBlocks; + // The batch resolve threw on a dead reference, so the + // marker always applies here. + resolvedPrompt = withMediaDegradationMarker( + perBlock.resolvedBlocks, + ); + } const normalized: PromptRequest = telemetry.injectPromptContext( { ...req, sessionId, + prompt: resolvedPrompt, }, ); assertLivePromptEntry(sessionId, entry); @@ -7870,6 +8095,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { delete meta[DAEMON_CHANNEL_DELIVERY_META_KEY]; delete meta[DAEMON_PROMPT_DISPLAY_TEXT_META_KEY]; delete meta[DAEMON_MODEL_PROMPT_META_KEY]; + delete meta[DAEMON_MEDIA_REFERENCES_META_KEY]; // Channel classification is authenticated channel-worker // metadata; the daemon prompt route validates the worker // authorization and re-arms it through the trusted @@ -7892,6 +8118,12 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { if (modelPrompt !== undefined) { meta[DAEMON_MODEL_PROMPT_META_KEY] = modelPrompt; } + const mediaReferences = dispatchBlocks.filter( + isSessionMediaReference, + ); + if (mediaReferences.length > 0) { + meta[DAEMON_MEDIA_REFERENCES_META_KEY] = mediaReferences; + } if (context?.channelPrompt === true) { meta[CHANNEL_PROMPT_META_KEY] = true; } @@ -7943,7 +8175,10 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { if (!isRetry && !isContinue) { echoPromptToSessionBus( entry, - promptRequest, + { + ...promptRequest, + prompt: dispatchBlocks, + }, pendingEntry.promptId, originatorClientId, channelDisplayText, @@ -8186,6 +8421,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { message.messageId, message.text, message.originatorClientId, + message.content, ); } // DAEMON-005: deferred close-on-prompt-complete. Lives here (not @@ -10024,6 +10260,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { .map((p) => ({ promptId: p.promptId, text: p.text, + ...(p.content ? { content: p.content } : {}), queuedAt: p.queuedAt, state: p.state, ...(p.originatorClientId !== undefined @@ -10032,6 +10269,49 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { })); }, + async storeSessionMedia(sessionId, data, mimeType, context) { + const entry = byId.get(sessionId); + if (!entry) throw new SessionNotFoundError(sessionId); + resolveTrustedClientId(entry, context?.clientId); + const operation = mediaPutQueue.then(async () => { + if (byId.get(sessionId) !== entry) { + throw new SessionNotFoundError(sessionId); + } + const retainedBytes = [...new Set(retainedMedia.values())].reduce( + (total, store) => total + store.sizeBytes, + 0, + ); + if ( + retainedBytes + data.byteLength > + MAX_RETAINED_SESSION_MEDIA_BYTES + ) { + throw new RangeError( + `Session media exceeds the ${MAX_RETAINED_SESSION_MEDIA_BYTES}-byte daemon limit`, + ); + } + return await entry.media.put(data, mimeType); + }); + mediaPutQueue = operation.then( + () => undefined, + () => undefined, + ); + return await operation; + }, + + async readSessionMedia(sessionId, mediaId, context) { + const entry = byId.get(sessionId); + if (!entry) throw new SessionNotFoundError(sessionId); + resolveTrustedClientId(entry, context?.clientId); + return await entry.media.read(mediaId); + }, + + async removeSessionMedia(sessionId, mediaId, context) { + const entry = byId.get(sessionId); + if (!entry) throw new SessionNotFoundError(sessionId); + resolveTrustedClientId(entry, context?.clientId); + return await entry.media.remove(mediaId); + }, + removePendingPrompt(sessionId, promptId, context) { const entry = byId.get(sessionId); if (!entry) throw new SessionNotFoundError(sessionId); @@ -10058,6 +10338,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { // A queued prompt never dispatches once aborted — safe to drop // from the list immediately. entry.pendingPromptList.splice(idx, 1); + removeQueuedMedia(entry, target.content); } else { // A RUNNING prompt must stay on the list (hidden from // `getPendingPrompts` via the `removed` flag) until it settles @@ -10115,7 +10396,13 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { context?.clientId, ); const trimmed = message.trim(); - if (trimmed.length === 0) { + // Media blocks travel with the message through drain and promotion; + // anything else a caller sneaks into `content` (text/resource blocks) + // is dropped here so the drain never duplicates the message text. + const mediaBlocks = (options?.content ?? []).filter( + (block): block is BridgePromptContentBlock => block.type === 'image', + ); + if (trimmed.length === 0 && mediaBlocks.length === 0) { writeStderrLine( `[mid-turn] session=${entry.sessionId} rejected: empty`, ); @@ -10126,11 +10413,17 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { (queued) => queued.messageId === requestedMessageId, ); if (existing) { - if (existing.text === trimmed) { + // A retry under the same id is only idempotent when the WHOLE + // payload matches — accepting different media under an existing id + // would silently keep the original attachments. + const sameMedia = + JSON.stringify(existing.content ?? []) === + JSON.stringify(mediaBlocks); + if (existing.text === trimmed && sameMedia) { return { accepted: true, messageId: requestedMessageId }; } writeStderrLine( - `[mid-turn] session=${JSON.stringify(entry.sessionId)} rejected id ${JSON.stringify(requestedMessageId)}: text mismatch`, + `[mid-turn] session=${JSON.stringify(entry.sessionId)} rejected id ${JSON.stringify(requestedMessageId)}: text or content mismatch`, ); return { accepted: false }; } @@ -10138,11 +10431,20 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { (pending) => pending.promptId === requestedMessageId, ); if (promoted) { - if (promoted.text === trimmed) { + const promotedImages = (promoted.content ?? []).filter( + (block) => block.type === 'image', + ); + const sameMedia = + JSON.stringify(promotedImages) === JSON.stringify(mediaBlocks); + const promotedText = + promoted.text === '[image]' && trimmed.length === 0 + ? '' + : promoted.text; + if (promotedText === trimmed && sameMedia) { return { accepted: true, messageId: requestedMessageId }; } writeStderrLine( - `[mid-turn] session=${JSON.stringify(entry.sessionId)} rejected promoted id ${JSON.stringify(requestedMessageId)}: text mismatch`, + `[mid-turn] session=${JSON.stringify(entry.sessionId)} rejected promoted id ${JSON.stringify(requestedMessageId)}: text or content mismatch`, ); return { accepted: false }; } @@ -10162,6 +10464,25 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { ); return { accepted: false }; } + // Validate only genuinely new admissions, AFTER the retry-ack rings: + // a same-id retry whose media was already removed (delete racing an + // in-flight POST, or a refresh re-enqueueing from the snapshot) must + // settle idempotently instead of failing with session_media_gone. + entry.media.assertReferences(mediaBlocks); + const inlineBytes = inlineMediaBlockBytes(mediaBlocks); + if (inlineBytes > 0) { + const queuedInlineBytes = entry.midTurnMessageQueue.reduce( + (total, queued) => + total + inlineMediaBlockBytes(queued.content ?? []), + 0, + ); + if (queuedInlineBytes + inlineBytes > SESSION_MEDIA_MAX_TOTAL_BYTES) { + writeStderrLine( + `[mid-turn] session=${entry.sessionId} rejected: queued inline media exceeds the ${SESSION_MEDIA_MAX_TOTAL_BYTES}-byte session budget`, + ); + return { accepted: false }; + } + } const messageId = requestedMessageId ?? randomUUID(); // If the turn settled while the POST was in flight, start it through the // normal prompt path. A client-supplied id keeps retries idempotent. @@ -10175,7 +10496,13 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { ); return { accepted: false }; } - promoteMidTurnMessage(entry, messageId, trimmed, originatorClientId); + promoteMidTurnMessage( + entry, + messageId, + trimmed, + originatorClientId, + mediaBlocks.length > 0 ? mediaBlocks : undefined, + ); return { accepted: true, messageId }; } // Bound the drain queue. Rejected requests remain unowned. @@ -10188,6 +10515,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { const queuedMessage: MidTurnQueueEntry = { messageId, text: trimmed, + ...(mediaBlocks.length > 0 ? { content: mediaBlocks } : {}), originatorClientId, ...(options?.queueOnly ? { @@ -10255,6 +10583,7 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { const [removed] = entry.midTurnMessageQueue.splice(index, 1); rememberMidTurnId(entry.settledMidTurnMessageIds, messageId); if (removed) { + removeQueuedMedia(entry, removed.content); try { entry.events.publish({ type: 'pending_prompt_completed', @@ -10316,6 +10645,10 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { .map((message) => ({ messageId: message.messageId, text: message.text, + // Carry the media blocks so a refreshed client can rebuild the + // queued row with its attachments (the snapshot is the only + // recovery source once the client's in-memory copy is gone). + ...(message.content ? { content: message.content } : {}), })), settledMessageIds: [...entry.settledMidTurnMessageIds], promotedMessageIds: [...entry.promotedMidTurnMessageIds], @@ -11174,6 +11507,11 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { closingChannel, `force kill closing session ${JSON.stringify(sessionId)}`, ); + await releaseSessionMedia(sessionId).catch((releaseError) => { + writeStderrLine( + `qwen serve: failed to release media for killed session ${JSON.stringify(sessionId)}: ${releaseError instanceof Error ? releaseError.message : String(releaseError)}`, + ); + }); return true; } entry.closing = true; @@ -11210,6 +11548,15 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { ci, `force kill session ${JSON.stringify(sessionId)}`, ); + // Kill removes media immediately (design: "explicit close, kill, and + // daemon shutdown remove them immediately"); without this the + // channel-exit handler's crash-path retention would keep it for the + // detach TTL. + await releaseSessionMedia(sessionId).catch((releaseError) => { + writeStderrLine( + `qwen serve: failed to release media for killed session ${JSON.stringify(sessionId)}: ${releaseError instanceof Error ? releaseError.message : String(releaseError)}`, + ); + }); return true; } entry.closing = false; @@ -11272,6 +11619,11 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { /* bus already closed */ } entry.events.close(); + await releaseSessionMedia(sessionId).catch((error) => { + writeStderrLine( + `qwen serve: failed to release media for killed session ${JSON.stringify(sessionId)}: ${error instanceof Error ? error.message : String(error)}`, + ); + }); // Only kill the channel when no other sessions remain AND no // restore is in flight. // `pendingRestoreIds` covers in-flight `session/load` and @@ -11468,6 +11820,8 @@ export function createAcpSessionBridge(opts: BridgeOptions): AcpSessionBridge { : Promise.resolve(); const teardownResults = await Promise.allSettled([ ...channels.map((ci) => ci.channel.kill()), + ...[...retainedMedia.values()].map((store) => store.close()), + mediaPutQueue, ...inFlightSessionAwaits, ...inFlightRestoreAwaits, inFlightChannelAwait, diff --git a/packages/acp-bridge/src/bridgeClient.test.ts b/packages/acp-bridge/src/bridgeClient.test.ts index 17a672eaa44..453eb6a59b3 100644 --- a/packages/acp-bridge/src/bridgeClient.test.ts +++ b/packages/acp-bridge/src/bridgeClient.test.ts @@ -70,6 +70,10 @@ import type { ClientMcpMessageSender } from './bridgeOptions.js'; import { CancelSentinelCollisionError } from './bridgeErrors.js'; import { CANCEL_VOTE_SENTINEL } from './permissionMediator.js'; import { SessionArtifactStore } from './sessionArtifacts.js'; +import { + SESSION_MEDIA_MAX_ITEM_BYTES, + SessionMediaStore, +} from './sessionMedia.js'; /** * Minimal-stub constructor for a `BridgeClient` whose only purpose is @@ -3115,6 +3119,7 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = events: { publish: ReturnType }; activePromptId?: string; promptActive?: boolean; + media?: SessionMediaStore; } | undefined, ownsSession?: (sessionId: string) => boolean, @@ -3122,6 +3127,7 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = const resolvedEntry = entry ? { ...entry, + media: entry.media ?? new SessionMediaStore(), pendingPromptList: entry.pendingPromptList ?? [], settledMidTurnMessageIds: entry.settledMidTurnMessageIds ?? [], } @@ -3183,6 +3189,28 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = 'anonymous ordinary', 'second', ], + items: [ + { + messageId: 'mid-1', + displayText: 'first', + content: [{ type: 'text', text: 'first' }], + }, + { + messageId: 'internal', + displayText: '', + content: [{ type: 'text', text: '' }], + }, + { + messageId: 'anonymous-ui', + displayText: 'anonymous ordinary', + content: [{ type: 'text', text: 'anonymous ordinary' }], + }, + { + messageId: 'mid-2', + displayText: 'second', + content: [{ type: 'text', text: 'second' }], + }, + ], hasQueuedPrompt: false, }); // Queue emptied so the same messages can't be re-injected on the next batch. @@ -3204,6 +3232,8 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = sessionId: 'sess:drain', messages: ['first', 'anonymous ordinary', 'second'], messageIds: ['mid-1', 'anonymous-ui', 'mid-2'], + // Echo frame carries content blocks per message (empty here — text-only). + items: [{}, {}, {}], }, }); // The session-wide frame omits internal anonymous steering while the child @@ -3211,6 +3241,393 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = expect(publish.mock.calls[0][0].originatorClientId).toBeUndefined(); }); + it('echo frame carries media content blocks for image-bearing messages', async () => { + const publish = vi.fn().mockReturnValue(true); + const image = { + type: 'image', + data: 'base64data', + mimeType: 'image/png', + } as const; + const entry = { + sessionId: 'sess:media', + activePromptId: 'prompt-media', + midTurnMessageQueue: [ + { + messageId: 'mid-text', + text: 'plain', + }, + { + messageId: 'mid-image', + text: 'look at this', + content: [image], + }, + ], + settledMidTurnMessageIds: [], + events: { publish }, + }; + const client = makeClientWithEntry('sess:media', entry); + + await client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:media', + }); + + expect(publish).toHaveBeenCalledTimes(1); + expect(publish.mock.calls[0][0]).toMatchObject({ + type: 'mid_turn_message_injected', + data: { + sessionId: 'sess:media', + messages: ['plain', 'look at this'], + messageIds: ['mid-text', 'mid-image'], + items: [{}, { content: [image] }], + }, + }); + }); + + it('resolves media references for the child and preserves their replay metadata', async () => { + const publish = vi.fn().mockReturnValue(true); + const media = new SessionMediaStore(); + try { + const reference = await media.put(Uint8Array.of(1, 2, 3), 'image/png'); + const entry = { + sessionId: 'sess:media-reference', + midTurnMessageQueue: [ + { + messageId: 'mid-reference', + text: 'look', + content: [reference], + }, + ], + settledMidTurnMessageIds: [] as string[], + events: { publish }, + media, + }; + const client = makeClientWithEntry('sess:media-reference', entry); + + await expect( + client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:media-reference', + }), + ).resolves.toMatchObject({ + items: [ + { + content: [ + { type: 'text', text: 'look' }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + mediaReferences: [reference], + }, + ], + }); + } finally { + await media.close(); + } + }); + + it('degrades a mediaId reused across drained messages after its first use', async () => { + const publish = vi.fn().mockReturnValue(true); + const media = new SessionMediaStore(); + try { + const reference = await media.put(Uint8Array.of(1, 2, 3), 'image/png'); + const read = vi.spyOn(media, 'read'); + const entry = { + sessionId: 'sess:shared-media', + midTurnMessageQueue: [ + { messageId: 'mid-a', text: 'a', content: [reference] }, + { messageId: 'mid-b', text: 'b', content: [reference] }, + { messageId: 'mid-c', text: 'c', content: [reference] }, + { messageId: 'mid-d', text: 'd', content: [reference] }, + ], + settledMidTurnMessageIds: [] as string[], + events: { publish }, + media, + }; + const client = makeClientWithEntry('sess:shared-media', entry); + + await expect( + client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:shared-media', + }), + ).resolves.toMatchObject({ + items: [ + { + content: [ + { type: 'text', text: 'a' }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + }, + { + content: [ + { + type: 'text', + text: 'b\n[Attached media is no longer available]', + }, + ], + }, + { + content: [ + { + type: 'text', + text: 'c\n[Attached media is no longer available]', + }, + ], + }, + { + content: [ + { + type: 'text', + text: 'd\n[Attached media is no longer available]', + }, + ], + }, + ], + }); + // Cross-message reuse is unsupported: only the first occurrence is + // serialized, so one stored blob cannot amplify the drain response. + expect(read).toHaveBeenCalledTimes(1); + } finally { + await media.close(); + } + }); + it('claims drained ids before media resolution yields', async () => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const media = { + resolveContent: vi.fn(async (content: MidTurnQueueEntry['content']) => { + await gate; + return content ?? []; + }), + } as unknown as SessionMediaStore; + const entry = { + sessionId: 'sess:slow-media', + midTurnMessageQueue: [ + { + messageId: 'mid-slow', + text: 'look', + content: [ + { type: 'image' as const, data: 'AQID', mimeType: 'image/png' }, + ], + }, + ], + settledMidTurnMessageIds: [] as string[], + events: { publish: vi.fn().mockReturnValue(true) }, + media, + }; + const client = makeClientWithEntry('sess:slow-media', entry); + + const drain = client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:slow-media', + }); + await Promise.resolve(); + + expect(entry.midTurnMessageQueue).toEqual([]); + expect(entry.settledMidTurnMessageIds).toEqual(['mid-slow']); + + release(); + await expect(drain).resolves.toMatchObject({ + messages: ['look'], + items: [{ messageId: 'mid-slow' }], + }); + }); + + it('degrades an expired media item without blocking later messages', async () => { + const publish = vi.fn().mockReturnValue(true); + const queued = { + messageId: 'mid-expired', + text: 'look at this', + content: [ + { + type: 'image' as const, + mediaId: 'expired', + mimeType: 'image/png', + size: 3, + }, + ], + }; + const entry = { + sessionId: 'sess:expired', + midTurnMessageQueue: [ + queued, + { messageId: 'mid-ok', text: 'keep going' }, + ], + settledMidTurnMessageIds: [] as string[], + events: { publish }, + }; + const client = makeClientWithEntry('sess:expired', entry); + + await expect( + client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:expired', + }), + ).resolves.toMatchObject({ + messages: ['look at this', 'keep going'], + items: [ + { + messageId: 'mid-expired', + content: [ + { + type: 'text', + text: 'look at this\n[Attached media is no longer available]', + }, + ], + }, + { + messageId: 'mid-ok', + content: [{ type: 'text', text: 'keep going' }], + }, + ], + }); + + expect(entry.midTurnMessageQueue).toEqual([]); + expect(entry.settledMidTurnMessageIds).toEqual(['mid-expired', 'mid-ok']); + expect(publish).toHaveBeenCalledOnce(); + }); + + it('drains every valid media reference even when their total exceeds 16 MiB', async () => { + const media = new SessionMediaStore(); + try { + const large = new Uint8Array(SESSION_MEDIA_MAX_ITEM_BYTES); + const refs = [ + await media.put(large, 'image/png'), + await media.put(large, 'image/png'), + await media.put(Uint8Array.of(1), 'image/png'), + ]; + const entry = { + sessionId: 'sess:large-drain', + midTurnMessageQueue: [ + { messageId: 'mid-large', text: 'all images', content: refs }, + ], + settledMidTurnMessageIds: [] as string[], + events: { publish: vi.fn().mockReturnValue(true) }, + media, + }; + const client = makeClientWithEntry('sess:large-drain', entry); + + const result = (await client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:large-drain', + })) as { + items: Array<{ + content: Array>; + mediaReferences?: unknown[]; + }>; + }; + + expect( + result.items[0]?.content.filter((block) => block['type'] === 'image'), + ).toHaveLength(3); + expect(result.items[0]?.mediaReferences).toEqual(refs); + } finally { + await media.close(); + } + }); + + it('requeues drained messages when media resolution fails with a non-media error', async () => { + // A transient fs error (fd exhaustion) must not silently degrade the + // media of every queued message: the store still holds the bytes, so the + // drain surfaces the error and hands the messages back for the next one. + const publish = vi.fn().mockReturnValue(true); + const media = new SessionMediaStore(); + const readFile = vi + .spyOn(fsp, 'readFile') + .mockRejectedValueOnce( + Object.assign(new Error('too many open files'), { code: 'EMFILE' }), + ); + try { + const reference = await media.put(Uint8Array.of(1, 2, 3), 'image/png'); + const entry = { + sessionId: 'sess:emfile', + midTurnMessageQueue: [ + { messageId: 'mid-a', text: 'a', content: [reference] }, + { messageId: 'mid-b', text: 'b' }, + ], + settledMidTurnMessageIds: [] as string[], + events: { publish }, + media, + }; + const client = makeClientWithEntry('sess:emfile', entry); + + await expect( + client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:emfile', + }), + ).rejects.toThrow('too many open files'); + // Requeued for the next drain; the settled ring no longer claims the + // ids, and nothing was echoed to the browser. + expect(entry.midTurnMessageQueue.map((item) => item.messageId)).toEqual([ + 'mid-a', + 'mid-b', + ]); + expect(entry.settledMidTurnMessageIds).toEqual([]); + expect(publish).not.toHaveBeenCalled(); + + // The retry drain re-reads the still-stored bytes and delivers both. + await expect( + client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:emfile', + }), + ).resolves.toMatchObject({ + items: [ + { + messageId: 'mid-a', + content: [ + { type: 'text', text: 'a' }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + }, + { messageId: 'mid-b', content: [{ type: 'text', text: 'b' }] }, + ], + }); + } finally { + readFile.mockRestore(); + await media.close(); + } + }); + + it('keeps a resolvable sibling when one reference is gone at drain', async () => { + // One dead reference must drop only itself, not the whole message's + // media: the sibling the store still holds reaches the child. + const publish = vi.fn().mockReturnValue(true); + const media = new SessionMediaStore(); + try { + const live = await media.put(Uint8Array.of(1, 2, 3), 'image/png'); + const gone = await media.put(Uint8Array.of(4, 5), 'image/png'); + await media.remove(gone.mediaId); + const entry = { + sessionId: 'sess:mixed', + midTurnMessageQueue: [ + { messageId: 'mid-mixed', text: 'mixed', content: [gone, live] }, + ], + settledMidTurnMessageIds: [] as string[], + events: { publish }, + media, + }; + const client = makeClientWithEntry('sess:mixed', entry); + + await expect( + client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:mixed', + }), + ).resolves.toMatchObject({ + items: [ + { + messageId: 'mid-mixed', + content: [ + { + type: 'text', + text: 'mixed\n[Attached media is no longer available]', + }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + mediaReferences: [live], + }, + ], + }); + } finally { + await media.close(); + } + }); + it('does not publish an injected frame for anonymous steering', async () => { const publish = vi.fn().mockReturnValue(true); const entry = { @@ -3233,6 +3650,13 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = }), ).resolves.toEqual({ messages: [''], + items: [ + { + messageId: 'internal', + displayText: '', + content: [{ type: 'text', text: '' }], + }, + ], hasQueuedPrompt: false, }); expect(entry.settledMidTurnMessageIds).toEqual(['internal']); @@ -3272,6 +3696,23 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = }); expect(result).toEqual({ messages: ['a', 'b', 'c'], + items: [ + { + messageId: 'mid-a', + displayText: 'a', + content: [{ type: 'text', text: 'a' }], + }, + { + messageId: 'mid-b', + displayText: 'b', + content: [{ type: 'text', text: 'b' }], + }, + { + messageId: 'mid-c', + displayText: 'c', + content: [{ type: 'text', text: 'c' }], + }, + ], hasQueuedPrompt: false, }); expect(entry.midTurnMessageQueue).toEqual([]); @@ -3320,6 +3761,13 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = // (a) the child still receives the message despite the dropped echo. expect(result).toEqual({ messages: ['still-delivered'], + items: [ + { + messageId: 'mid-delivered', + displayText: 'still-delivered', + content: [{ type: 'text', text: 'still-delivered' }], + }, + ], hasQueuedPrompt: false, }); expect(entry.midTurnMessageQueue).toEqual([]); @@ -3348,7 +3796,11 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = sessionId: 'sess:empty', }); - expect(result).toEqual({ messages: [], hasQueuedPrompt: false }); + expect(result).toEqual({ + messages: [], + items: [], + hasQueuedPrompt: false, + }); expect(publish).not.toHaveBeenCalled(); }); @@ -3394,7 +3846,7 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = const result = await client.extMethod('craft/drainMidTurnQueue', { sessionId: 'sess:absent', }); - expect(result).toEqual({ messages: [], hasQueuedPrompt: false }); + expect(result).toEqual({ messages: [], items: [], hasQueuedPrompt: false }); }); it('short-circuits to an empty drain when no sessionId is supplied', async () => { @@ -3414,7 +3866,35 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = Infinity, ); const result = await client.extMethod('craft/drainMidTurnQueue', {}); - expect(result).toEqual({ messages: [], hasQueuedPrompt: false }); + expect(result).toEqual({ messages: [], items: [], hasQueuedPrompt: false }); + }); + + it('does not drain a session not owned by this channel', async () => { + const publish = vi.fn().mockReturnValue(true); + const entry = { + sessionId: 'sess:other-channel', + midTurnMessageQueue: [{ messageId: 'mid-1', text: 'private message' }], + settledMidTurnMessageIds: [] as string[], + events: { publish }, + }; + const client = makeClientWithEntry( + 'sess:other-channel', + entry, + () => false, + ); + + await expect( + client.extMethod('craft/drainMidTurnQueue', { + sessionId: 'sess:other-channel', + }), + ).resolves.toEqual({ + messages: [], + items: [], + hasQueuedPrompt: false, + }); + expect(entry.midTurnMessageQueue).toHaveLength(1); + expect(entry.settledMidTurnMessageIds).toEqual([]); + expect(publish).not.toHaveBeenCalled(); }); it('reports only complete, non-aborted queued prompts', async () => { @@ -3446,14 +3926,14 @@ describe('BridgeClient — mid-turn queue drain (craft/drainMidTurnQueue)', () = client.extMethod('craft/drainMidTurnQueue', { sessionId: 'sess:queued', }), - ).resolves.toEqual({ messages: [], hasQueuedPrompt: true }); + ).resolves.toEqual({ messages: [], items: [], hasQueuedPrompt: true }); queued.abortController.abort(); await expect( client.extMethod('craft/drainMidTurnQueue', { sessionId: 'sess:queued', }), - ).resolves.toEqual({ messages: [], hasQueuedPrompt: false }); + ).resolves.toEqual({ messages: [], items: [], hasQueuedPrompt: false }); }); it('claims only for the live running owner and reports queued competition', async () => { diff --git a/packages/acp-bridge/src/bridgeClient.ts b/packages/acp-bridge/src/bridgeClient.ts index 492b0a30b3c..75be6d97029 100644 --- a/packages/acp-bridge/src/bridgeClient.ts +++ b/packages/acp-bridge/src/bridgeClient.ts @@ -9,6 +9,7 @@ import { promises as fs } from 'node:fs'; import * as path from 'node:path'; import type { Client, + ContentBlock, ReadTextFileRequest, ReadTextFileResponse, RequestPermissionRequest, @@ -83,6 +84,13 @@ import type { SessionArtifactInput, SessionArtifactStore, } from './sessionArtifacts.js'; +import { + isSessionMediaReference, + SessionMediaReferenceError, + withMediaDegradationMarker, + type SessionMediaReference, + type SessionMediaStore, +} from './sessionMedia.js'; /** * Validate a channel-wide active-work snapshot off the wire. @@ -607,6 +615,7 @@ export interface BridgeClientSessionEntry { effectiveCwd: string; events: EventBus; artifacts: SessionArtifactStore; + media: SessionMediaStore; recordingDegraded: boolean; pendingPermissionIds: Set; /** Pollable pending human interactions, keyed by permission request id. */ @@ -1179,8 +1188,8 @@ export class BridgeClient implements Client { * between tool batches to pull any messages the browser queued mid-turn. We splice the per-session * queue, return them to the child as the response, and — when non-empty — * publish a `mid_turn_message_injected` SSE frame so the browser can move - * those messages out of its pending queue (a dedupe signal, not a transcript - * render). Unknown methods reject with ACP `methodNotFound` (-32601), matching + * those messages out of its pending queue and render the immediate echo. + * Unknown methods reject with ACP `methodNotFound` (-32601), matching * the SDK's * default for an unimplemented client surface; the child's drain caller * treats that as "drain unsupported" and stops asking. @@ -1229,15 +1238,16 @@ export class BridgeClient implements Client { // The drain always carries a sessionId; without one we can't route it on a // multi-session channel (and `resolveEntry(undefined)` would throw there), // so answer with an empty drain rather than poisoning the turn. - if (!sessionId) return { messages: [], hasQueuedPrompt: false }; + if (!sessionId) return { messages: [], items: [], hasQueuedPrompt: false }; + if (!this.ownsSession(sessionId)) { + return { messages: [], items: [], hasQueuedPrompt: false }; + } const entry = this.resolveEntry(sessionId); - if (!entry) return { messages: [], hasQueuedPrompt: false }; + if (!entry) return { messages: [], items: [], hasQueuedPrompt: false }; const drained = entry.midTurnMessageQueue.splice(0); if (drained.length > 0) { - // Record the handoff so clients that lost their bookkeeping (page - // refresh) or missed the echo frame below can reconcile via - // `getMidTurnMessages` instead of resending an already-injected - // message as the next turn. Ring is bounded; oldest ids evicted. + // Claim the ids before media I/O yields so retries and removals cannot + // observe a drained message as neither queued nor settled. for (const item of drained) { entry.settledMidTurnMessageIds.push(item.messageId); } @@ -1252,9 +1262,89 @@ export class BridgeClient implements Client { ); } } + // Shared across every message in this drain: one stored mediaId that + // several queued messages reference is read and base64-encoded once + // instead of once per message. + const mediaMemo = new Map>(); + const serializedMediaIds = new Set(); + const items: Array<{ + messageId: string; + displayText: string; + content: ContentBlock[]; + mediaReferences?: SessionMediaReference[]; + }> = []; + try { + for (const item of drained) { + let degraded = 0; + const planned = (item.content ?? []).filter((block) => { + if (!isSessionMediaReference(block)) return true; + if (serializedMediaIds.has(block.mediaId)) { + degraded += 1; + return false; + } + serializedMediaIds.add(block.mediaId); + return true; + }); + let resolvedBlocks: ContentBlock[]; + let mediaReferences: SessionMediaReference[]; + try { + resolvedBlocks = await entry.media.resolveContent(planned, mediaMemo); + mediaReferences = planned.filter(isSessionMediaReference); + } catch (error) { + // Only a gone/invalid reference degrades — per block, so one dead + // reference drops itself and keeps its siblings. Any other error + // (fd exhaustion, I/O failure) propagates instead of silently + // destroying the media of every message sharing the mediaId. + if (!(error instanceof SessionMediaReferenceError)) throw error; + writeStderrLine( + `[mid-turn] session=${JSON.stringify(entry.sessionId)} degraded media for message ${JSON.stringify(item.messageId)}: ${JSON.stringify(error instanceof Error ? error.message : String(error))}`, + ); + const perBlock = await entry.media.resolveContentDegrading( + planned, + mediaMemo, + ); + resolvedBlocks = perBlock.resolvedBlocks; + mediaReferences = perBlock.retainedBlocks.filter( + isSessionMediaReference, + ); + degraded += perBlock.degraded; + } + let content: ContentBlock[] = [ + ...(item.text ? [{ type: 'text' as const, text: item.text }] : []), + ...resolvedBlocks, + ]; + if (degraded > 0) content = withMediaDegradationMarker(content); + items.push({ + messageId: item.messageId, + displayText: item.text, + content, + ...(mediaReferences.length > 0 ? { mediaReferences } : {}), + }); + } + } catch (error) { + // Non-media resolution failure after the splice + settle above: the + // store still holds the bytes, so hand the messages back to the queue + // for the next drain instead of losing them, and take their ids back + // out of the settled ring so a same-id retry is not acked as already + // delivered. + const requeued = new Set(drained.map((queued) => queued.messageId)); + const ring = entry.settledMidTurnMessageIds; + const kept = ring.filter((id) => !requeued.has(id)); + ring.splice(0, ring.length, ...kept); + entry.midTurnMessageQueue.unshift(...drained); + writeStderrLine( + `[mid-turn] session=${JSON.stringify(entry.sessionId)} drain failed, requeued ${drained.length} message(s): ${JSON.stringify(error instanceof Error ? error.message : String(error))}`, + ); + throw error; + } // Queue-only entries are private coordinator steering, not UI transcript. const echoed = drained.filter((item) => !item.queueOnly); const messages = drained.map((item) => item.text); + // Structured twin of `messages` carrying any queued media blocks. The ACP + // child prefers `items` when present and falls back to `messages`, so + // text-only entries surface as a single text block and old children keep + // working unchanged. References travel beside the resolved content so the + // child can persist replay-safe metadata rather than inline bytes. const hasQueuedPrompt = entry.pendingPromptList.some( (prompt) => prompt.state === 'queued' && !prompt.abortController.signal.aborted, @@ -1272,6 +1362,15 @@ export class BridgeClient implements Client { sessionId: entry.sessionId, messages: echoed.map((item) => item.text), messageIds: echoed.map((item) => item.messageId), + // Carry the structured `items` twin (content blocks per message) so + // the browser-side echo renderer can show attached images alongside + // the message text. Older consumers that don't read this field keep + // working unchanged. + items: echoed.map((item) => ({ + ...(item.content && item.content.length > 0 + ? { content: item.content } + : {}), + })), }, }); writeStderrLine( @@ -1280,7 +1379,7 @@ export class BridgeClient implements Client { : `[mid-turn] session=${entry.sessionId} drained=${messages.length} echoed=${echoed.length} echo frame dropped (bus closed); reconciliation required`, ); } - return { messages, hasQueuedPrompt }; + return { messages, items, hasQueuedPrompt }; } private async handleExternalToolGuardPrepare( diff --git a/packages/acp-bridge/src/bridgeTypes.ts b/packages/acp-bridge/src/bridgeTypes.ts index ba9b631df8e..c7af69b97d3 100644 --- a/packages/acp-bridge/src/bridgeTypes.ts +++ b/packages/acp-bridge/src/bridgeTypes.ts @@ -11,6 +11,7 @@ import type { } from '@qwen-code/qwen-code-core'; import type { CancelNotification, + ContentBlock, LoadSessionResponse, PromptRequest, PromptResponse, @@ -34,6 +35,7 @@ import type { SessionArtifactMutationResult, SessionArtifactsEnvelope, } from './sessionArtifacts.js'; +import type { SessionMediaReference } from './sessionMedia.js'; import type { ServeSessionContextStatus, ServeSessionHooksStatus, @@ -56,6 +58,12 @@ export interface RewindSnapshotInfo { diffStats: { filesChanged: number; insertions: number; deletions: number }; } +export type BridgePromptContentBlock = ContentBlock | SessionMediaReference; + +export type BridgePromptRequest = Omit & { + prompt: BridgePromptContentBlock[]; +}; + export interface RewindRequest { promptId: string; rewindFiles?: boolean; @@ -755,6 +763,7 @@ export interface BridgeClientRequestContext { } export const DAEMON_MODEL_PROMPT_META_KEY = 'qwen.daemon.modelPrompt'; +export const DAEMON_MEDIA_REFERENCES_META_KEY = 'qwen.daemon.mediaReferences'; export const MAX_TRUSTED_MODEL_PROMPT_CHARS = 64 * 1024; export function isValidTrustedModelPrompt(value: unknown): value is string { @@ -880,6 +889,12 @@ export type ClientMcpOverWsRuntimeConfig = Record & { export interface MidTurnQueueEntry { messageId: string; text: string; + /** + * Image content blocks attached to the message. The drain + * combines them with `text` into structured `items` for the ACP child; + * the promotion path sends them alongside the text block. + */ + content?: BridgePromptContentBlock[]; originatorClientId?: string; queueOnly?: boolean; onSettledWithoutDrain?: () => void; @@ -891,7 +906,7 @@ export interface MidTurnQueueEntry { * running turn and messages promoted into the normal pending-prompt FIFO. */ export interface BridgeMidTurnMessagesSnapshot { - messages: Array>; + messages: Array>; settledMessageIds: string[]; promotedMessageIds: string[]; } @@ -909,6 +924,12 @@ export interface PendingPromptEntry { originatorClientId?: string; promotedMidTurn?: true; text: string; + /** + * Image content blocks attached to this prompt. Used by + * `getPendingPrompts` so a refreshed client can restore the full payload + * (text + images) instead of just the text. + */ + content?: BridgePromptContentBlock[]; abortController: AbortController; state: 'queued' | 'running'; /** @@ -942,6 +963,12 @@ export interface PendingPromptEntry { export interface PendingPromptSummary { promptId: string; text: string; + /** + * Image content blocks attached to this prompt, so a + * refreshed client can restore the full payload (text + images) instead + * of just the text. + */ + content?: BridgePromptContentBlock[]; queuedAt: number; state: 'queued' | 'running'; originatorClientId?: string; @@ -1222,7 +1249,7 @@ export interface AcpSessionBridge { */ sendPrompt( sessionId: string, - req: PromptRequest, + req: BridgePromptRequest, signal?: AbortSignal, context?: BridgeClientRequestContext, ): Promise; @@ -1729,6 +1756,8 @@ export interface AcpSessionBridge { * With `options.queueOnly` an idle session rejects instead of promoting. If * a busy session settles before draining the message, * `onSettledWithoutDrain` lets the caller drive the next turn itself. + * `options.content` carries image blocks with the message; + * an empty `message` is admitted when media blocks are present. */ enqueueMidTurnMessage( sessionId: string, @@ -1738,9 +1767,29 @@ export interface AcpSessionBridge { options?: { queueOnly?: boolean; onSettledWithoutDrain?: () => void; + content?: readonly BridgePromptContentBlock[]; }, ): { accepted: boolean; messageId?: string }; + storeSessionMedia( + sessionId: string, + data: Uint8Array, + mimeType: string, + context?: BridgeClientRequestContext, + ): Promise; + + readSessionMedia( + sessionId: string, + mediaId: string, + context?: BridgeClientRequestContext, + ): Promise<{ data: Buffer; mimeType: string } | undefined>; + + removeSessionMedia( + sessionId: string, + mediaId: string, + context?: BridgeClientRequestContext, + ): Promise; + /** Remove a queued or promoted mid-turn message. */ removeMidTurnMessage( sessionId: string, diff --git a/packages/acp-bridge/src/index.ts b/packages/acp-bridge/src/index.ts index da304418892..4145dddbf3f 100644 --- a/packages/acp-bridge/src/index.ts +++ b/packages/acp-bridge/src/index.ts @@ -13,6 +13,7 @@ export * from './workspacePaths.js'; export * from './status.js'; export * from './bridgeErrors.js'; export * from './sessionArtifacts.js'; +export * from './sessionMedia.js'; export * from './bridgeTypes.js'; export * from './session-source.js'; export * from './bridgeOptions.js'; diff --git a/packages/acp-bridge/src/sessionMedia.test.ts b/packages/acp-bridge/src/sessionMedia.test.ts new file mode 100644 index 00000000000..abb3904e186 --- /dev/null +++ b/packages/acp-bridge/src/sessionMedia.test.ts @@ -0,0 +1,396 @@ +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { promises as fs } from 'node:fs'; +import type { ContentBlock } from '@agentclientprotocol/sdk'; +import { describe, expect, it, vi } from 'vitest'; +import { + SESSION_MEDIA_UNAVAILABLE_TEXT, + SESSION_MEDIA_MAX_ITEM_BYTES, + SESSION_MEDIA_MAX_ITEMS, + SESSION_MEDIA_MAX_TOTAL_BYTES, + SessionMediaStore, + withMediaDegradationMarker, +} from './sessionMedia.js'; + +describe('SessionMediaStore', () => { + it('does not append the media degradation marker twice', () => { + const once = withMediaDegradationMarker([ + { type: 'text', text: 'look at this' }, + ]); + + expect(withMediaDegradationMarker(once)).toEqual([ + { + type: 'text', + text: `look at this\n${SESSION_MEDIA_UNAVAILABLE_TEXT}`, + }, + ]); + }); + + it('stores bytes by reference and resolves them only at dispatch', async () => { + const store = new SessionMediaStore(); + try { + const reference = await store.put( + Uint8Array.from([1, 2, 3]), + 'image/png', + ); + + expect(reference).toMatchObject({ + type: 'image', + mimeType: 'image/png', + size: 3, + }); + expect(await store.resolveContent([reference])).toEqual([ + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ]); + expect(await store.read(reference.mediaId)).toEqual({ + data: Buffer.from([1, 2, 3]), + mimeType: 'image/png', + }); + } finally { + await store.close(); + } + }); + + it('resolves duplicate references with a single read', async () => { + // Duplicate references to one stored item must not multiply the disk + // reads and base64 encodes at dispatch — that amplification let one + // small request pin gigabytes of heap. + const store = new SessionMediaStore(); + const readFile = vi.spyOn(fs, 'readFile'); + try { + const reference = await store.put( + Uint8Array.from([1, 2, 3]), + 'image/png', + ); + readFile.mockClear(); + + const resolved = await store.resolveContent([ + reference, + { ...reference }, + { ...reference }, + ]); + + expect(resolved).toEqual([ + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ]); + expect(readFile).toHaveBeenCalledTimes(1); + } finally { + readFile.mockRestore(); + await store.close(); + } + }); + + it('shares reads across resolveContent calls via a caller-supplied memo', async () => { + const store = new SessionMediaStore(); + const readFile = vi.spyOn(fs, 'readFile'); + try { + const reference = await store.put( + Uint8Array.from([1, 2, 3]), + 'image/png', + ); + readFile.mockClear(); + + const memo = new Map>(); + const block = { + type: 'image', + data: 'AQID', + mimeType: 'image/png', + }; + expect(await store.resolveContent([reference], memo)).toEqual([block]); + expect(await store.resolveContent([reference], memo)).toEqual([block]); + expect(readFile).toHaveBeenCalledTimes(1); + + // Omitting the memo keeps the per-call default: a fresh map, so the + // blob is read again. + expect(await store.resolveContent([reference])).toEqual([block]); + expect(readFile).toHaveBeenCalledTimes(2); + } finally { + readFile.mockRestore(); + await store.close(); + } + }); + it('keeps media for the lifetime of the store', async () => { + const store = new SessionMediaStore(); + try { + const reference = await store.put(Uint8Array.of(1), 'image/png'); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2100-01-01T00:00:00Z')); + + expect(await store.read(reference.mediaId)).toBeDefined(); + } finally { + vi.useRealTimers(); + await store.close(); + } + }); + + it('rejects non-image uploads', async () => { + const store = new SessionMediaStore(); + try { + await expect(store.put(Uint8Array.of(1), 'audio/wav')).rejects.toThrow( + 'Session media must be image/*', + ); + } finally { + await store.close(); + } + }); + + it('rejects empty and oversized uploads', async () => { + const store = new SessionMediaStore(); + try { + await expect(store.put(new Uint8Array(), 'image/png')).rejects.toThrow( + /between 1 and/, + ); + await expect( + store.put( + new Uint8Array(SESSION_MEDIA_MAX_ITEM_BYTES + 1), + 'image/png', + ), + ).rejects.toThrow(/between 1 and/); + } finally { + await store.close(); + } + }); + + it('retries directory creation after a transient failure', async () => { + const mkdir = vi + .spyOn(fs, 'mkdtemp') + .mockRejectedValueOnce( + Object.assign(new Error('full'), { code: 'ENOSPC' }), + ); + const store = new SessionMediaStore(); + try { + await expect(store.put(Uint8Array.of(1), 'image/png')).rejects.toThrow( + 'full', + ); + mkdir.mockRestore(); + await expect( + store.put(Uint8Array.of(1), 'image/png'), + ).resolves.toMatchObject({ size: 1 }); + } finally { + mkdir.mockRestore(); + await store.close(); + } + }); + + it('removes a partial file after writing fails', async () => { + const write = vi + .spyOn(fs, 'writeFile') + .mockRejectedValueOnce( + Object.assign(new Error('full'), { code: 'ENOSPC' }), + ); + const remove = vi.spyOn(fs, 'rm'); + const store = new SessionMediaStore(); + try { + await expect(store.put(Uint8Array.of(1), 'image/png')).rejects.toThrow( + 'full', + ); + expect(remove).toHaveBeenCalledWith(expect.any(String), { force: true }); + expect(store.sizeBytes).toBe(0); + } finally { + write.mockRestore(); + remove.mockRestore(); + await store.close(); + } + }); + + it('closes cleanly after directory creation fails', async () => { + const mkdir = vi + .spyOn(fs, 'mkdtemp') + .mockRejectedValueOnce( + Object.assign(new Error('full'), { code: 'ENOSPC' }), + ); + const store = new SessionMediaStore(); + try { + await expect(store.put(Uint8Array.of(1), 'image/png')).rejects.toThrow( + 'full', + ); + await expect(store.close()).resolves.toBeUndefined(); + } finally { + mkdir.mockRestore(); + await store.close(); + } + }); + + it('removes stored media and releases its byte accounting', async () => { + const store = new SessionMediaStore(); + try { + const reference = await store.put(Uint8Array.of(1, 2), 'image/png'); + expect(store.sizeBytes).toBe(2); + await expect(store.remove(reference.mediaId)).resolves.toBe(true); + expect(store.sizeBytes).toBe(0); + await expect(store.read(reference.mediaId)).resolves.toBeUndefined(); + } finally { + await store.close(); + } + }); + + it('forgets media whose backing file disappeared', async () => { + const store = new SessionMediaStore(); + try { + const reference = await store.put(Uint8Array.of(1, 2), 'image/png'); + const read = vi + .spyOn(fs, 'readFile') + .mockRejectedValueOnce( + Object.assign(new Error('gone'), { code: 'ENOENT' }), + ); + try { + await expect(store.read(reference.mediaId)).resolves.toBeUndefined(); + expect(store.sizeBytes).toBe(0); + expect(() => store.assertReferences([reference])).toThrow( + 'Unknown or unavailable session media', + ); + } finally { + read.mockRestore(); + } + } finally { + await store.close(); + } + }); + + it('rejects duplicate references to one mediaId in a single message', async () => { + // A block count cap alone does not bound the resolved payload: the same + // mediaId repeated N times passes admission and expands per occurrence at + // dispatch, so one small upload can serialize into gigabytes. Reject the + // duplicate occurrences at admission. + const store = new SessionMediaStore(); + try { + const reference = await store.put(Uint8Array.of(1, 2, 3), 'image/png'); + expect(() => + store.assertReferences([reference, { ...reference }]), + ).toThrow(/more than once/); + // A single occurrence is still valid. + expect(() => store.assertReferences([reference])).not.toThrow(); + } finally { + await store.close(); + } + }); + + it('rejects references from another session store', async () => { + const first = new SessionMediaStore(); + const second = new SessionMediaStore(); + try { + const reference = await first.put(Uint8Array.of(1), 'image/png'); + expect(() => second.assertReferences([reference])).toThrow( + 'Unknown or unavailable session media', + ); + } finally { + await Promise.all([first.close(), second.close()]); + } + }); + + it('bounds the number of stored objects', async () => { + const store = new SessionMediaStore(); + try { + await Promise.all( + Array.from({ length: SESSION_MEDIA_MAX_ITEMS }, async () => + store.put(Uint8Array.of(1), 'image/png'), + ), + ); + await expect(store.put(Uint8Array.of(1), 'image/png')).rejects.toThrow( + `${SESSION_MEDIA_MAX_ITEMS}-item session limit`, + ); + } finally { + await store.close(); + } + }); + + it('bounds the total bytes stored by one session', async () => { + const write = vi.spyOn(fs, 'writeFile').mockResolvedValue(undefined); + const store = new SessionMediaStore(); + try { + const item = new Uint8Array(SESSION_MEDIA_MAX_ITEM_BYTES); + const accepted = Math.floor( + SESSION_MEDIA_MAX_TOTAL_BYTES / SESSION_MEDIA_MAX_ITEM_BYTES, + ); + for (let index = 0; index < accepted; index += 1) { + await store.put(item, 'image/png'); + } + + await expect(store.put(item, 'image/png')).rejects.toThrow( + /session limit/, + ); + expect(store.sizeBytes).toBe(accepted * item.byteLength); + } finally { + write.mockRestore(); + await store.close(); + } + }); + + it('evicts a rejected memo entry so siblings and retries read again', async () => { + // A transient non-ENOENT read failure must not be cached in a shared + // memo: every message referencing the same mediaId would otherwise await + // the cached rejection although the store still holds the bytes. + const store = new SessionMediaStore(); + const readFile = vi + .spyOn(fs, 'readFile') + .mockRejectedValueOnce( + Object.assign(new Error('too many open files'), { code: 'EMFILE' }), + ); + try { + const reference = await store.put(Uint8Array.of(9, 9), 'image/png'); + const memo = new Map>(); + + await expect(store.resolveContent([reference], memo)).rejects.toThrow( + 'too many open files', + ); + // The failed entry must not stay cached: the next resolution re-reads + // from disk and succeeds. + await expect(store.resolveContent([reference], memo)).resolves.toEqual([ + { type: 'image', data: 'CQk=', mimeType: 'image/png' }, + ]); + expect(readFile).toHaveBeenCalledTimes(2); + } finally { + readFile.mockRestore(); + await store.close(); + } + }); + + it('resolveContentDegrading drops only the unresolvable reference', async () => { + const store = new SessionMediaStore(); + try { + const live = await store.put(Uint8Array.of(1, 2), 'image/png'); + const gone = await store.put(Uint8Array.of(3, 4), 'image/png'); + await store.remove(gone.mediaId); + const text = { type: 'text', text: 'both' } as ContentBlock; + + const result = await store.resolveContentDegrading([text, gone, live]); + + expect(result.degraded).toBe(1); + expect(result.retainedBlocks).toEqual([text, live]); + expect(result.resolvedBlocks).toEqual([ + text, + { type: 'image', data: 'AQI=', mimeType: 'image/png' }, + ]); + } finally { + await store.close(); + } + }); + + it('does not make byte accounting negative when close races put', async () => { + let finishWrite: (() => void) | undefined; + const write = vi.spyOn(fs, 'writeFile').mockImplementationOnce( + async () => + await new Promise((resolve) => { + finishWrite = resolve; + }), + ); + const store = new SessionMediaStore(); + try { + const pending = store.put(Uint8Array.of(1), 'image/png'); + await vi.waitFor(() => expect(write).toHaveBeenCalled()); + await store.close(); + finishWrite?.(); + await expect(pending).rejects.toThrow('Session media store is closed'); + expect((store as unknown as { totalBytes: number }).totalBytes).toBe(0); + } finally { + write.mockRestore(); + await store.close(); + } + }); +}); diff --git a/packages/acp-bridge/src/sessionMedia.ts b/packages/acp-bridge/src/sessionMedia.ts new file mode 100644 index 00000000000..8e746ed993f --- /dev/null +++ b/packages/acp-bridge/src/sessionMedia.ts @@ -0,0 +1,353 @@ +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { randomUUID } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import { tmpdir } from 'node:os'; +import * as path from 'node:path'; +import type { ContentBlock } from '@agentclientprotocol/sdk'; + +export const SESSION_MEDIA_MAX_ITEM_BYTES = 8 * 1024 * 1024; +export const SESSION_MEDIA_MAX_TOTAL_BYTES = 100 * 1024 * 1024; +export const SESSION_MEDIA_MAX_ITEMS = 256; + +// Text the degrade paths substitute for media the model will not receive. The +// SDK's DaemonSessionClient.hydrateBlock and the web shell's degradation +// detection carry their own copies; keep the wording in sync. +export const SESSION_MEDIA_UNAVAILABLE_TEXT = + '[Attached media is no longer available]'; + +export class SessionMediaReferenceError extends Error { + constructor( + message: string, + readonly code: 'invalid_session_media_reference' | 'session_media_gone', + ) { + super(message); + this.name = 'SessionMediaReferenceError'; + } +} + +export interface SessionMediaReference { + type: 'image'; + mediaId: string; + mimeType: string; + size: number; +} + +interface StoredSessionMedia extends SessionMediaReference { + filePath: string; +} + +export function isSessionMediaReference( + value: unknown, +): value is SessionMediaReference { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const record = value as Record; + return ( + record['type'] === 'image' && + typeof record['mediaId'] === 'string' && + record['mediaId'].length > 0 && + typeof record['mimeType'] === 'string' && + record['mimeType'].startsWith(`${record['type']}/`) && + typeof record['size'] === 'number' && + Number.isSafeInteger(record['size']) && + record['size'] > 0 + ); +} + +// Append the unavailable marker to the last text block (or as a new text +// block) so a partially degraded prompt keeps its surviving blocks instead of +// collapsing into one wholesale placeholder. +export function withMediaDegradationMarker< + T extends ContentBlock | SessionMediaReference, +>(blocks: readonly T[]): T[] { + for (let i = blocks.length - 1; i >= 0; i--) { + const block = blocks[i]; + if (block.type === 'text') { + if (block.text.endsWith(SESSION_MEDIA_UNAVAILABLE_TEXT)) { + return [...blocks]; + } + const next = [...blocks]; + next[i] = { + type: 'text', + text: `${block.text}\n${SESSION_MEDIA_UNAVAILABLE_TEXT}`, + } as T; + return next; + } + } + return [ + ...blocks, + { type: 'text', text: SESSION_MEDIA_UNAVAILABLE_TEXT } as T, + ]; +} + +export class SessionMediaStore { + private readonly records = new Map(); + private directoryPromise?: Promise; + private totalBytes = 0; + private pendingItems = 0; + private closed = false; + + async put( + data: Uint8Array, + mimeType: string, + ): Promise { + if (this.closed) throw new Error('Session media store is closed'); + const type = 'image' as const; + if (!mimeType.startsWith('image/')) { + throw new TypeError('Session media must be image/*'); + } + if ( + data.byteLength === 0 || + data.byteLength > SESSION_MEDIA_MAX_ITEM_BYTES + ) { + throw new RangeError( + `Session media must be between 1 and ${SESSION_MEDIA_MAX_ITEM_BYTES} bytes`, + ); + } + if (this.totalBytes + data.byteLength > SESSION_MEDIA_MAX_TOTAL_BYTES) { + throw new RangeError( + `Session media exceeds the ${SESSION_MEDIA_MAX_TOTAL_BYTES}-byte session limit`, + ); + } + if (this.records.size + this.pendingItems >= SESSION_MEDIA_MAX_ITEMS) { + throw new RangeError( + `Session media exceeds the ${SESSION_MEDIA_MAX_ITEMS}-item session limit`, + ); + } + + const mediaId = randomUUID(); + let filePath: string | undefined; + this.totalBytes += data.byteLength; + this.pendingItems += 1; + try { + const directory = await this.directory(); + filePath = path.join(directory, mediaId); + await fs.writeFile(filePath, data, { flag: 'wx' }); + if (this.closed) { + throw new Error('Session media store is closed'); + } + const record: StoredSessionMedia = { + type, + mediaId, + mimeType, + size: data.byteLength, + filePath, + }; + this.records.set(mediaId, record); + return { type, mediaId, mimeType, size: data.byteLength }; + } catch (error) { + if (filePath) await fs.rm(filePath, { force: true }).catch(() => {}); + if (!this.closed) this.totalBytes -= data.byteLength; + throw error; + } finally { + if (!this.closed) this.pendingItems -= 1; + } + } + + // Validate one block against the store. Blocks without a `mediaId` (inline + // media, text) pass through untouched, matching `assertReferences`. + assertReference(block: unknown): void { + if ( + !block || + typeof block !== 'object' || + Array.isArray(block) || + !('mediaId' in block) + ) { + return; + } + if (!isSessionMediaReference(block)) { + throw new SessionMediaReferenceError( + 'Invalid session media reference', + 'invalid_session_media_reference', + ); + } + this.assertStored(block); + } + + assertReferences(content: readonly unknown[]): void { + // One occurrence per mediaId: the serializer expands every reference at + // dispatch, so repeated occurrences of one stored blob amplify the + // outbound payload without bound even though only one read is needed. + const seenMediaIds = new Set(); + for (const block of content) { + if ( + !block || + typeof block !== 'object' || + Array.isArray(block) || + !('mediaId' in block) + ) { + continue; + } + if (!isSessionMediaReference(block)) { + throw new SessionMediaReferenceError( + 'Invalid session media reference', + 'invalid_session_media_reference', + ); + } + if (seenMediaIds.has(block.mediaId)) { + throw new SessionMediaReferenceError( + `Session media referenced more than once: ${block.mediaId}`, + 'invalid_session_media_reference', + ); + } + seenMediaIds.add(block.mediaId); + this.assertStored(block); + } + } + + async resolveContent( + content: ReadonlyArray, + memo?: Map>, + ): Promise { + // Resolve each distinct mediaId once: duplicate references share the read + // and base64 encode instead of amplifying heap per occurrence. Callers + // resolving several messages in one batch can pass a shared `memo` so a + // mediaId referenced from different messages is also read only once. + const pendingByMediaId = memo ?? new Map>(); + return await Promise.all( + content.map(async (block) => { + if (!isSessionMediaReference(block)) return block; + let pending = pendingByMediaId.get(block.mediaId); + if (!pending) { + const created = this.resolve(block); + pendingByMediaId.set(block.mediaId, created); + // A transient read failure must not poison later resolutions of the + // same mediaId: a cached rejection would hand every sibling message + // (and every later lookup) the failure although the store still + // holds the bytes. Evict it so the next lookup reads again. + void created.catch(() => { + if (pendingByMediaId.get(block.mediaId) === created) { + pendingByMediaId.delete(block.mediaId); + } + }); + pending = created; + } + return await pending; + }), + ); + } + + // Per-block variant of `resolveContent` for degrade paths: one unresolvable + // reference drops only itself, keeping the sibling blocks a wholesale + // fallback would discard. Non-media errors still propagate. + async resolveContentDegrading( + content: ReadonlyArray, + memo?: Map>, + ): Promise<{ + retainedBlocks: Array; + resolvedBlocks: ContentBlock[]; + degraded: number; + }> { + const retainedBlocks: Array = []; + const resolvedBlocks: ContentBlock[] = []; + let degraded = 0; + for (const block of content) { + if (!isSessionMediaReference(block)) { + retainedBlocks.push(block); + resolvedBlocks.push(block); + continue; + } + try { + const [resolved] = await this.resolveContent([block], memo); + if (resolved) resolvedBlocks.push(resolved); + retainedBlocks.push(block); + } catch (error) { + if (!(error instanceof SessionMediaReferenceError)) throw error; + degraded += 1; + } + } + return { retainedBlocks, resolvedBlocks, degraded }; + } + + async read( + mediaId: string, + ): Promise<{ data: Buffer; mimeType: string } | undefined> { + const record = this.records.get(mediaId); + if (!record) return undefined; + try { + return { + data: await fs.readFile(record.filePath), + mimeType: record.mimeType, + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + if (this.records.delete(mediaId)) this.totalBytes -= record.size; + return undefined; + } + throw error; + } + } + + async remove(mediaId: string): Promise { + const record = this.records.get(mediaId); + if (!record) return false; + this.records.delete(mediaId); + this.totalBytes -= record.size; + await fs.rm(record.filePath, { force: true }); + return true; + } + + get sizeBytes(): number { + return this.totalBytes; + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + this.records.clear(); + this.totalBytes = 0; + this.pendingItems = 0; + if (!this.directoryPromise) return; + const directory = await this.directoryPromise.catch(() => undefined); + if (!directory) return; + await fs.rm(directory, { recursive: true, force: true }); + } + + private assertStored(reference: SessionMediaReference): void { + const stored = this.records.get(reference.mediaId); + if ( + !stored || + stored.type !== reference.type || + stored.mimeType !== reference.mimeType || + stored.size !== reference.size + ) { + throw new SessionMediaReferenceError( + `Unknown or unavailable session media: ${reference.mediaId}`, + 'session_media_gone', + ); + } + } + + private async resolve( + reference: SessionMediaReference, + ): Promise { + const media = await this.read(reference.mediaId); + if (!media || media.mimeType !== reference.mimeType) { + throw new SessionMediaReferenceError( + `Unknown or unavailable session media: ${reference.mediaId}`, + 'session_media_gone', + ); + } + return { + type: reference.type, + data: media.data.toString('base64'), + mimeType: media.mimeType, + } as ContentBlock; + } + + private async directory(): Promise { + if (!this.directoryPromise) { + const pending = fs.mkdtemp(path.join(tmpdir(), 'qwen-session-media-')); + this.directoryPromise = pending; + void pending.catch(() => { + if (this.directoryPromise === pending) + this.directoryPromise = undefined; + }); + } + return await this.directoryPromise; + } +} diff --git a/packages/acp-bridge/src/transcript-replay.test.ts b/packages/acp-bridge/src/transcript-replay.test.ts index f9e5707157f..8e40455ed2e 100644 --- a/packages/acp-bridge/src/transcript-replay.test.ts +++ b/packages/acp-bridge/src/transcript-replay.test.ts @@ -585,6 +585,43 @@ describe('createTranscriptReplayMachine', () => { const tagged = '\ninjected hook context\n'; + it('replays daemon media references without embedding base64', () => { + const projected = updates( + createTranscriptReplayMachine(), + record('user-media-ref', 'user', { + message: { role: 'user', parts: [{ text: 'describe this' }] }, + systemPayload: { + displayText: 'describe this', + hookContext: '', + mediaReferences: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }, + }), + ); + + expect(projected).toMatchObject([ + { + sessionUpdate: 'user_message_chunk', + content: { type: 'text', text: 'describe this' }, + }, + { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + }, + ]); + }); + it('replaces text parts with displayText while preserving image parts', () => { // displayText must replace all model-facing text while the image part // survives (the previous early-return path dropped it). @@ -868,6 +905,116 @@ describe('createTranscriptReplayMachine', () => { }); }); + it('replays media references from a mid-turn user record', () => { + const projected = updates( + createTranscriptReplayMachine(), + record('mid-turn-media', 'user', { + subtype: 'mid_turn_user_message', + message: { role: 'user', parts: [{ text: 'inspect image' }] }, + systemPayload: { + displayText: 'inspect image', + mediaReferences: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }, + }), + ); + + expect(projected).toMatchObject([ + { + sessionUpdate: 'user_message_chunk', + content: { type: 'text', text: 'inspect image' }, + _meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + }, + { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + _meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + }, + ]); + }); + + it('replays an image-only mid-turn record without its synthetic prefix', () => { + const projected = updates( + createTranscriptReplayMachine(), + record('mid-turn-image-only', 'user', { + subtype: 'mid_turn_user_message', + message: { + role: 'user', + parts: [{ text: '[User message received during tool execution]: ' }], + }, + systemPayload: { + displayText: '', + mediaReferences: [ + { + type: 'image', + mediaId: 'media-only', + mimeType: 'image/png', + size: 3, + }, + ], + }, + }), + ); + + expect(projected).toMatchObject([ + { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'media-only', + mimeType: 'image/png', + size: 3, + }, + _meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + }, + ]); + }); + + it('falls back to inline parts for an image-only mid-turn record without references', () => { + const projected = updates( + createTranscriptReplayMachine(), + record('mid-turn-inline-image-only', 'user', { + subtype: 'mid_turn_user_message', + message: { + role: 'user', + parts: [{ inlineData: { data: 'AQID', mimeType: 'image/png' } }], + }, + systemPayload: { displayText: '' }, + }), + ); + + expect(projected).toMatchObject([ + { + sessionUpdate: 'user_message_chunk', + content: { type: 'image', data: 'AQID', mimeType: 'image/png' }, + _meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + }, + ]); + }); + it('projects ordered message parts with source metadata', () => { const machine = createTranscriptReplayMachine(); const projected = updates( diff --git a/packages/acp-bridge/src/transcript-replay.ts b/packages/acp-bridge/src/transcript-replay.ts index 436c9896207..3c93043b45c 100644 --- a/packages/acp-bridge/src/transcript-replay.ts +++ b/packages/acp-bridge/src/transcript-replay.ts @@ -259,6 +259,31 @@ export function createTranscriptImageUpdate( } as SessionUpdate; } +function createTranscriptMediaReferenceUpdate( + reference: Record, + options: UpdateMetaOptions, +): SessionUpdate | undefined { + if ( + (reference['type'] !== 'image' && reference['type'] !== 'audio') || + typeof reference['mediaId'] !== 'string' || + typeof reference['mimeType'] !== 'string' || + typeof reference['size'] !== 'number' + ) { + return undefined; + } + const meta = buildUpdateMeta(options); + return { + sessionUpdate: 'user_message_chunk', + content: { + type: reference['type'], + mediaId: reference['mediaId'], + mimeType: reference['mimeType'], + size: reference['size'], + }, + ...(meta ? { _meta: meta } : {}), + } as unknown as SessionUpdate; +} + export function createTranscriptUsageUpdate( usageMetadata: TranscriptUsageMetadataInput, options: TranscriptUsageUpdateOptions = {}, @@ -550,6 +575,17 @@ class DefaultTranscriptReplayMachine implements TranscriptReplayMachine { const payload = isObjectRecord(record.systemPayload) ? record.systemPayload : undefined; + const replayMeta: UpdateMetaOptions = + record.subtype === 'mid_turn_user_message' + ? { + ...meta, + extra: { + ...meta.extra, + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + } + : meta; if ( record.subtype === 'goal_runtime' || record.subtype === 'notification' || @@ -560,6 +596,15 @@ class DefaultTranscriptReplayMachine implements TranscriptReplayMachine { payload && typeof payload['displayText'] === 'string' ? payload['displayText'] : undefined; + if (record.subtype === 'mid_turn_user_message' && displayText === '') { + const media = [ + ...this.projectUserMediaReferences(payload, emit, replayMeta), + ]; + if (media.length > 0) { + yield* media; + return; + } + } if (displayText) { const isNotification = record.subtype === 'notification'; const backgroundTask = @@ -570,7 +615,7 @@ class DefaultTranscriptReplayMachine implements TranscriptReplayMachine { createTranscriptMessageUpdate({ role: 'user', text: displayText, - ...meta, + ...replayMeta, ...(isNotification ? { extra: { @@ -584,6 +629,7 @@ class DefaultTranscriptReplayMachine implements TranscriptReplayMachine { : {}), }), ); + yield* this.projectUserMediaReferences(payload, emit, replayMeta); return; } if (record.subtype !== 'mid_turn_user_message') return; @@ -595,13 +641,14 @@ class DefaultTranscriptReplayMachine implements TranscriptReplayMachine { record, 'user', emit, - meta, + replayMeta, undefined, replaceTextPartsForDisplay( record.message?.parts, projection.displayText, ), ); + yield* this.projectUserMediaReferences(payload, emit, replayMeta); return; } @@ -609,10 +656,25 @@ class DefaultTranscriptReplayMachine implements TranscriptReplayMachine { record, 'user', emit, - meta, + replayMeta, undefined, projection.parts, ); + yield* this.projectUserMediaReferences(payload, emit, replayMeta); + } + + private *projectUserMediaReferences( + payload: Record | undefined, + emit: (update: SessionUpdate) => TranscriptReplayEmission, + meta: UpdateMetaOptions, + ): Iterable { + const references = payload?.['mediaReferences']; + if (!Array.isArray(references)) return; + for (const reference of references) { + if (!isObjectRecord(reference)) continue; + const update = createTranscriptMediaReferenceUpdate(reference, meta); + if (update) yield emit(update); + } } private *projectAssistantRecord( diff --git a/packages/cli/src/acp-integration/session/Session.test.ts b/packages/cli/src/acp-integration/session/Session.test.ts index 2c5bda0ec47..77f4a6758de 100644 --- a/packages/cli/src/acp-integration/session/Session.test.ts +++ b/packages/cli/src/acp-integration/session/Session.test.ts @@ -4559,6 +4559,65 @@ describe('Session', () => { ]); }); + it('records daemon media references for transcript replay', async () => { + const mediaReference = { + type: 'image' as const, + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }; + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'describe this' }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + _meta: { + 'qwen.daemon.mediaReferences': [mediaReference], + }, + }); + + expect(mockChatRecordingService.recordUserMessage).toHaveBeenCalledWith( + 'describe this', + undefined, + { + displayText: 'describe this', + hookContext: '', + mediaReferences: [mediaReference], + }, + ); + }); + + it('records every media reference allowed by the session store', async () => { + const mediaReferences = Array.from({ length: 256 }, (_, index) => ({ + type: 'image' as const, + mediaId: `media-${index}`, + mimeType: 'image/png', + size: 3, + })); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [{ type: 'text', text: 'describe these' }], + _meta: { + 'qwen.daemon.mediaReferences': mediaReferences, + }, + }); + + expect(mockChatRecordingService.recordUserMessage).toHaveBeenCalledWith( + 'describe these', + undefined, + expect.objectContaining({ mediaReferences }), + ); + }); + it('rejects model-only prompts without trusted invocation context', async () => { await expect( session.prompt( @@ -10581,6 +10640,32 @@ describe('Session', () => { }, ], displayText: 'please inspect this image', + mediaReferences: [ + { + type: 'image', + mediaId: 'image-1', + mimeType: 'image/png', + size: 8, + }, + ], + }, + { + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'cHVyZS1pbWFnZQ==', + }, + ], + displayText: '', + mediaReferences: [ + { + type: 'image', + mediaId: 'image-2', + mimeType: 'image/png', + size: 10, + }, + ], }, ], }); @@ -10655,12 +10740,438 @@ describe('Session', () => { ); expect( mockChatRecordingService.recordMidTurnUserMessage, - ).toHaveBeenCalledWith(midTurnParts, 'please inspect this image'); + ).toHaveBeenCalledWith( + [midTurnParts[0], midTurnParts[2]], + 'please inspect this image', + undefined, + [ + { + type: 'image', + mediaId: 'image-1', + mimeType: 'image/png', + size: 8, + }, + ], + ); + expect( + mockChatRecordingService.recordMidTurnUserMessage, + ).toHaveBeenCalledWith( + [{ text: '\n[User message received during tool execution]: ' }], + '', + undefined, + [ + { + type: 'image', + mediaId: 'image-2', + mimeType: 'image/png', + size: 10, + }, + ], + ); expect(debugLoggerWarnSpy).toHaveBeenCalledWith( 'Unknown ContentBlock type: video', ); }); + it('records inline-media-only mid-turn messages with a placeholder display text', async () => { + // An inline image with no text and no references must not record an + // empty displayText: resume and replay would otherwise fall back to + // the raw internal prefix carried by the recorded parts. + const executeSpy = vi.fn().mockResolvedValue({ + llmContent: 'file contents', + returnDisplay: 'file contents', + }); + const tool = { + name: 'read_file', + kind: core.Kind.Read, + build: vi.fn().mockReturnValue({ + params: { path: '/tmp/test.txt' }, + getDefaultPermission: vi.fn().mockResolvedValue('allow'), + getDescription: vi.fn().mockReturnValue('Read file'), + toolLocations: vi.fn().mockReturnValue([]), + execute: executeSpy, + }), + }; + + mockToolRegistry.getTool.mockReturnValue(tool); + mockConfig.getApprovalMode = vi.fn().mockReturnValue(ApprovalMode.YOLO); + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'vision-agent', + baseUrl: 'https://vision.example.com/v1', + agentCapable: true, + }); + mockClient.extMethod = vi.fn().mockResolvedValue({ + items: [ + { + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW5saW5lLW9ubHk=', + }, + ], + displayText: '', + }, + ], + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValueOnce( + createStreamWithChunks([ + { + type: core.StreamEventType.CHUNK, + value: { + functionCalls: [ + { + id: 'call-1', + name: 'read_file', + args: { path: '/tmp/test.txt' }, + }, + ], + }, + }, + ]), + ) + .mockResolvedValueOnce(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [{ type: 'text', text: 'read file' }], + }); + + expect( + mockChatRecordingService.recordMidTurnUserMessage, + ).toHaveBeenCalledWith( + [ + { + text: '\n[User message received during tool execution]: [User message with attachments]', + }, + { + inlineData: { + mimeType: 'image/png', + data: 'aW5saW5lLW9ubHk=', + }, + }, + ], + '[User message with attachments]', + ); + }); + + it('records a partially-referenced mid-turn message with the placeholder, never an empty displayText', async () => { + // A message whose media references cover only a SUBSET of its image + // blocks will NOT persist references (#buildMidTurnParts' count gate). + // The display-text gate must agree and emit the attachments + // placeholder — never '' — or replay/resume fall back to the recorded + // parts and leak the raw internal prefix. + const executeSpy = vi.fn().mockResolvedValue({ + llmContent: 'file contents', + returnDisplay: 'file contents', + }); + const tool = { + name: 'read_file', + kind: core.Kind.Read, + build: vi.fn().mockReturnValue({ + params: { path: '/tmp/test.txt' }, + getDefaultPermission: vi.fn().mockResolvedValue('allow'), + getDescription: vi.fn().mockReturnValue('Read file'), + toolLocations: vi.fn().mockReturnValue([]), + execute: executeSpy, + }), + }; + + mockToolRegistry.getTool.mockReturnValue(tool); + mockConfig.getApprovalMode = vi.fn().mockReturnValue(ApprovalMode.YOLO); + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'vision-agent', + baseUrl: 'https://vision.example.com/v1', + agentCapable: true, + }); + mockClient.extMethod = vi.fn().mockResolvedValue({ + items: [ + { + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW1nMQ==', + }, + { + type: 'image', + mimeType: 'image/png', + data: 'aW1nMg==', + }, + ], + displayText: '', + // One reference for two image blocks -> references will NOT be + // persisted, so displayText must not be ''. + mediaReferences: [ + { + type: 'image', + mediaId: 'ref-1', + mimeType: 'image/png', + size: 4, + }, + ], + }, + ], + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValueOnce( + createStreamWithChunks([ + { + type: core.StreamEventType.CHUNK, + value: { + functionCalls: [ + { + id: 'call-1', + name: 'read_file', + args: { path: '/tmp/test.txt' }, + }, + ], + }, + }, + ]), + ) + .mockResolvedValueOnce(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [{ type: 'text', text: 'read file' }], + }); + + expect( + mockChatRecordingService.recordMidTurnUserMessage, + ).toHaveBeenCalledWith( + [ + { + text: '\n[User message received during tool execution]: [User message with attachments]', + }, + { + inlineData: { + mimeType: 'image/png', + data: 'aW1nMQ==', + }, + }, + { + inlineData: { + mimeType: 'image/png', + data: 'aW1nMg==', + }, + }, + ], + '[User message with attachments]', + ); + }); + + it('keeps uncovered audio bytes in the transcript record', async () => { + // References are image-only: a drain whose only media block is audio + // must NOT take the reference-recording path (the gate would strip + // the audio bytes the model is about to see). + const executeSpy = vi.fn().mockResolvedValue({ + llmContent: 'file contents', + returnDisplay: 'file contents', + }); + const tool = { + name: 'read_file', + kind: core.Kind.Read, + build: vi.fn().mockReturnValue({ + params: { path: '/tmp/test.txt' }, + getDefaultPermission: vi.fn().mockResolvedValue('allow'), + getDescription: vi.fn().mockReturnValue('Read file'), + toolLocations: vi.fn().mockReturnValue([]), + execute: executeSpy, + }), + }; + + mockToolRegistry.getTool.mockReturnValue(tool); + mockConfig.getApprovalMode = vi.fn().mockReturnValue(ApprovalMode.YOLO); + mockConfig.getEffectiveInputModalities = vi + .fn() + .mockReturnValue({ audio: true }); + mockClient.extMethod = vi.fn().mockResolvedValue({ + items: [ + { + content: [ + { type: 'text', text: 'voice note' }, + { + type: 'audio', + mimeType: 'audio/wav', + data: 'UklGRgAAAA==', + }, + ], + displayText: 'voice note', + mediaReferences: [ + { + type: 'image', + mediaId: 'image-1', + mimeType: 'image/png', + size: 4, + }, + ], + }, + ], + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValueOnce( + createStreamWithChunks([ + { + type: core.StreamEventType.CHUNK, + value: { + functionCalls: [ + { + id: 'call-1', + name: 'read_file', + args: { path: '/tmp/test.txt' }, + }, + ], + }, + }, + ]), + ) + .mockResolvedValueOnce(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [{ type: 'text', text: 'read file' }], + }); + + expect( + mockChatRecordingService.recordMidTurnUserMessage, + ).toHaveBeenCalledWith( + [ + { + text: '\n[User message received during tool execution]: voice note', + }, + { + inlineData: { + mimeType: 'audio/wav', + data: 'UklGRgAAAA==', + }, + }, + ], + 'voice note', + ); + }); + + it('keeps @-mentioned image bytes in the record when references cover only the drained image', async () => { + // The reference-recording path must strip only the inline bytes the + // references replace — never the extra inline parts #resolvePrompt + // adds for @-mentioned files, which the model also sees. + const tempDir = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), 'qwen-acp-midturn-media-')), + ); + const mentionedPath = path.join(tempDir, 'mentioned.png'); + await fs.writeFile(mentionedPath, 'image'); + const executeSpy = vi.fn().mockResolvedValue({ + llmContent: 'file contents', + returnDisplay: 'file contents', + }); + const tool = { + name: 'read_file', + kind: core.Kind.Read, + build: vi.fn().mockReturnValue({ + params: { path: '/tmp/test.txt' }, + getDefaultPermission: vi.fn().mockResolvedValue('allow'), + getDescription: vi.fn().mockReturnValue('Read file'), + toolLocations: vi.fn().mockReturnValue([]), + execute: executeSpy, + }), + }; + + mockToolRegistry.getTool.mockReturnValue(tool); + mockConfig.getApprovalMode = vi.fn().mockReturnValue(ApprovalMode.YOLO); + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getProjectRoot = vi.fn().mockReturnValue(tempDir); + mockConfig.getWorkspaceContext = vi.fn().mockReturnValue({ + isPathWithinWorkspace: (pathSpec: string) => + path.resolve(tempDir, pathSpec).startsWith(`${tempDir}${path.sep}`), + }); + const readManyFilesSpy = vi + .spyOn(core, 'readManyFiles') + .mockResolvedValue({ + contentParts: { + inlineData: { mimeType: 'image/png', data: 'bWVudGlvbmVk' }, + }, + } as Awaited>); + mockClient.extMethod = vi.fn().mockResolvedValue({ + items: [ + { + content: [ + { type: 'text', text: `compare with @${mentionedPath}` }, + { + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=', + }, + ], + displayText: 'compare with image', + mediaReferences: [ + { + type: 'image', + mediaId: 'image-1', + mimeType: 'image/png', + size: 8, + }, + ], + }, + ], + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValueOnce( + createStreamWithChunks([ + { + type: core.StreamEventType.CHUNK, + value: { + functionCalls: [ + { + id: 'call-1', + name: 'read_file', + args: { path: '/tmp/test.txt' }, + }, + ], + }, + }, + ]), + ) + .mockResolvedValueOnce(createEmptyStream()); + + try { + await session.prompt({ + sessionId: 'test-session-id', + prompt: [{ type: 'text', text: 'read file' }], + }); + + expect(readManyFilesSpy).toHaveBeenCalled(); + expect( + mockChatRecordingService.recordMidTurnUserMessage, + ).toHaveBeenCalledWith( + expect.arrayContaining([ + { + inlineData: { mimeType: 'image/png', data: 'bWVudGlvbmVk' }, + }, + ]), + 'compare with image', + undefined, + [ + { + type: 'image', + mediaId: 'image-1', + mimeType: 'image/png', + size: 8, + }, + ], + ); + } finally { + readManyFilesSpy.mockRestore(); + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + it('keeps later structured mid-turn messages when one resolution fails', async () => { const clampSpy = vi .spyOn(core, 'clampInlineMediaPart') diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index 971762fbb12..cea4d3dacb0 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -197,11 +197,13 @@ import { ENV_ACP_REPEATED_TOOL_FAILURE_GUARD } from '../../config/shared-env-key import { type ActiveWorkHoldV1, DAEMON_CHANNEL_DELIVERY_META_KEY, + DAEMON_MEDIA_REFERENCES_META_KEY, DAEMON_PROMPT_DISPLAY_TEXT_META_KEY, MID_TURN_QUEUE_DRAIN_METHOD, isValidTrustedModelPrompt, TODO_STOP_GUARD_CONTINUATION_CLAIM_METHOD, } from '@qwen-code/acp-bridge/bridgeTypes'; +import type { SessionMediaReference } from '@qwen-code/acp-bridge/sessionMedia'; import { SERVE_CONTROL_EXT_METHODS } from '@qwen-code/acp-bridge/status'; import { getCommandSubcommandNames } from '../../services/commandMetadata.js'; import { cleanupReviewWorktreeLeases } from '../../services/review-worktree-lease.js'; @@ -347,6 +349,47 @@ const NEW_PROMPT_ABORT_REASON = 'qwen:new-prompt'; const SESSION_DISPOSE_ABORT_REASON = 'qwen:session-dispose'; const DAEMON_RETRY_META_KEY = 'qwen.daemon.retry'; const DAEMON_CONTINUE_META_KEY = 'qwen.daemon.continueLastTurn'; +const MAX_DAEMON_MEDIA_REFERENCES = 256; + +function readDaemonMediaReferences( + value: unknown, +): SessionMediaReference[] | undefined { + if ( + !Array.isArray(value) || + value.length === 0 || + value.length > MAX_DAEMON_MEDIA_REFERENCES + ) { + return undefined; + } + const references: SessionMediaReference[] = []; + for (const item of value) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return undefined; + } + const reference = item as Record; + if ( + reference['type'] !== 'image' || + typeof reference['mediaId'] !== 'string' || + reference['mediaId'].length === 0 || + reference['mediaId'].length > 128 || + typeof reference['mimeType'] !== 'string' || + reference['mimeType'].length === 0 || + reference['mimeType'].length > 128 || + typeof reference['size'] !== 'number' || + !Number.isSafeInteger(reference['size']) || + reference['size'] <= 0 + ) { + return undefined; + } + references.push({ + type: reference['type'], + mediaId: reference['mediaId'], + mimeType: reference['mimeType'], + size: reference['size'], + }); + } + return references; +} const TODO_STOP_GUARD_PROMPT_PREFIX = '[Todo Stop Guard] '; const TODO_STOP_GUARD_PROMPT_BODY_SUFFIX = ' todo item(s) are still pending or in progress. Continue executing the current task now. Do not ask the user whether to continue. If progress requires user input, use the structured question or permission flow. If progress depends on external state, report the blocker explicitly.'; @@ -895,7 +938,12 @@ const TRANSIENT_FS_CODES: readonly string[] = [ type DrainedMidTurnMessage = | { kind: 'text'; message: string } - | { kind: 'structured'; content: ContentBlock[]; displayText: string }; + | { + kind: 'structured'; + content: ContentBlock[]; + displayText: string; + mediaReferences?: SessionMediaReference[]; + }; function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object'; @@ -1005,6 +1053,30 @@ function hasInlineMediaContentBlock(content: ContentBlock[]): boolean { return content.some((part) => part.type === 'image' || part.type === 'audio'); } +function stripReferencedInlineDataParts( + parts: Part[], + content: ContentBlock[], +): Part[] { + const coveredByKey = new Map(); + for (const block of content) { + if (block.type !== 'image') continue; + const key = `${block.mimeType}\u0000${block.data}`; + coveredByKey.set(key, (coveredByKey.get(key) ?? 0) + 1); + } + if (coveredByKey.size === 0) return parts; + return parts.filter((part) => { + const inlineData = part.inlineData; + if (inlineData === undefined || typeof inlineData.data !== 'string') { + return true; + } + const key = `${inlineData.mimeType ?? ''}\u0000${inlineData.data}`; + const remaining = coveredByKey.get(key) ?? 0; + if (remaining === 0) return true; + coveredByKey.set(key, remaining - 1); + return false; + }); +} + function capMidTurnDrainItems(items: T[], fieldName: string): T[] { if (items.length <= MAX_MID_TURN_DRAIN_ITEMS) return items; @@ -1050,6 +1122,7 @@ function getValidMidTurnContentBlocks( function getStructuredMidTurnDisplayText( content: ContentBlock[], displayText: unknown, + willPersistReferences: boolean, ): string { if (typeof displayText === 'string' && displayText.trim().length > 0) { return displayText.trim(); @@ -1064,7 +1137,19 @@ function getStructuredMidTurnDisplayText( .join('\n') .trim(); - return text || '[User message with attachments]'; + if (text) return text; + + // Only records that WILL persist media references keep '' (replay then + // projects the media ids). The gate must match #buildMidTurnParts' + // persistence condition exactly; a record that will not carry references + // needs the visible placeholder, because resume and replay fall back to the + // recorded parts — which start with the raw internal prefix — when + // displayText is empty. + if (!willPersistReferences && hasInlineMediaContentBlock(content)) { + return '[User message with attachments]'; + } + + return text; } function parseMidTurnDrainResponse(response: unknown): DrainedMidTurnMessage[] { @@ -1081,6 +1166,17 @@ function parseMidTurnDrainResponse(response: unknown): DrainedMidTurnMessage[] { item['displayText'], ); if (content.length === 0) return []; + const mediaReferences = readDaemonMediaReferences( + item['mediaReferences'], + ); + // Same gate #buildMidTurnParts uses to decide whether references are + // persisted; display text must agree or a mixed inline+reference + // message records displayText:'' with NO references — a shape replay + // and resume cannot project. + const willPersistReferences = + mediaReferences !== undefined && + mediaReferences.length === + content.filter((block) => block.type === 'image').length; return [ { kind: 'structured', @@ -1088,7 +1184,9 @@ function parseMidTurnDrainResponse(response: unknown): DrainedMidTurnMessage[] { displayText: getStructuredMidTurnDisplayText( content, item['displayText'], + willPersistReferences, ), + ...(mediaReferences ? { mediaReferences } : {}), }, ]; }, @@ -1130,7 +1228,9 @@ function isValidMidTurnDrainResponse( isRecord(item) && Array.isArray(item['content']) && item['content'].length > 0 && - item['content'].every(isContentBlock), + item['content'].every(isContentBlock) && + (item['mediaReferences'] === undefined || + readDaemonMediaReferences(item['mediaReferences']) !== undefined), ); } @@ -4081,10 +4181,11 @@ export class Session implements SessionContext { .filter((block) => block.type === 'text') .map((block) => (block.type === 'text' ? block.text : '')) .join(' '); + const promptDisplayTextValue = + promptMetadata?.[DAEMON_PROMPT_DISPLAY_TEXT_META_KEY]; const promptDisplayText = - typeof promptMetadata?.[DAEMON_PROMPT_DISPLAY_TEXT_META_KEY] === - 'string' - ? promptMetadata[DAEMON_PROMPT_DISPLAY_TEXT_META_KEY] + typeof promptDisplayTextValue === 'string' + ? promptDisplayTextValue : undefined; const modelPromptBlocks: PromptRequest['prompt'] = modelPrompt === undefined @@ -4194,11 +4295,15 @@ export class Session implements SessionContext { this.#getCurrentChat().stripOrphanedUserEntriesFromHistory(); } else { // record user message for session management + const mediaReferences = readDaemonMediaReferences( + promptMetadata?.[DAEMON_MEDIA_REFERENCES_META_KEY], + ); const recorder = this.config.getChatRecordingService(); - if (promptDisplayText !== undefined) { + if (promptDisplayText !== undefined || mediaReferences) { recorder?.recordUserMessage(promptText, goalTurn?.permit, { - displayText: promptDisplayText, + displayText: promptDisplayText ?? promptText, hookContext: '', + ...(mediaReferences ? { mediaReferences } : {}), }); } else if (goalTurn) { recorder?.recordUserMessage(promptText, goalTurn.permit); @@ -6626,9 +6731,24 @@ export class Session implements SessionContext { } } const built = prefixMidTurnUserMessageParts(rawParts, displayText); - this.config - .getChatRecordingService() - ?.recordMidTurnUserMessage(built, displayText); + const recorder = this.config.getChatRecordingService(); + if (message.kind === 'structured' && message.mediaReferences) { + const everyMediaBlockHasAReference = + message.mediaReferences.length === + message.content.filter((block) => block.type === 'image').length; + if (everyMediaBlockHasAReference) { + recorder?.recordMidTurnUserMessage( + stripReferencedInlineDataParts(built, message.content), + displayText, + undefined, + message.mediaReferences, + ); + } else { + recorder?.recordMidTurnUserMessage(built, displayText); + } + } else { + recorder?.recordMidTurnUserMessage(built, displayText); + } parts.push(...built); } return parts; diff --git a/packages/cli/src/acp-integration/session/history-replayer.test.ts b/packages/cli/src/acp-integration/session/history-replayer.test.ts index 62df9ce7c8a..8c5e447e400 100644 --- a/packages/cli/src/acp-integration/session/history-replayer.test.ts +++ b/packages/cli/src/acp-integration/session/history-replayer.test.ts @@ -204,7 +204,10 @@ describe('HistoryReplayer', () => { expect(sendUpdateSpy).toHaveBeenCalledWith({ sessionUpdate: 'user_message_chunk', content: { type: 'text', text: 'save logs' }, - _meta: replayMeta(record), + _meta: replayMeta(record, { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }), }); }); }); diff --git a/packages/cli/src/serve/acp-session-bridge.ts b/packages/cli/src/serve/acp-session-bridge.ts index 0144b4c830a..5dcba2a292d 100644 --- a/packages/cli/src/serve/acp-session-bridge.ts +++ b/packages/cli/src/serve/acp-session-bridge.ts @@ -82,6 +82,7 @@ export type { BridgeGenerationModelSource, BridgeGenerationStreamEvent, BridgeWorkspaceGenerationStreamEvent, + BridgePromptContentBlock, BridgeSessionSummary, SessionMetadataUpdate, BridgeClientRequestContext, diff --git a/packages/cli/src/serve/capabilities.ts b/packages/cli/src/serve/capabilities.ts index 39a60718cbf..8a552ea92a0 100644 --- a/packages/cli/src/serve/capabilities.ts +++ b/packages/cli/src/serve/capabilities.ts @@ -52,6 +52,10 @@ export const SERVE_CAPABILITY_REGISTRY = { session_source_metadata: { since: 'v1' }, session_side_task: { since: 'v1' }, session_prompt: { since: 'v1' }, + // Prompts and mid-turn messages support session-scoped media uploaded once + // and referenced by `mediaId`. The bridge resolves bytes only when ACP input + // is dispatched, keeping base64 out of JSON and SSE payloads. + session_media: { since: 'v1' }, session_mid_turn_message_mutation: { since: 'v1' }, // Daemon-owned reconciliation surface for mid-turn messages: // `GET /session/:id/mid-turn-messages` returns the messages still waiting diff --git a/packages/cli/src/serve/routes/session.ts b/packages/cli/src/serve/routes/session.ts index 246b25f8010..89afe16153a 100644 --- a/packages/cli/src/serve/routes/session.ts +++ b/packages/cli/src/serve/routes/session.ts @@ -43,7 +43,13 @@ import { } from '../conversations/session-source.js'; import type { ConversationRuntimeActivityGate } from '../conversations/conversation-runtime-activity.js'; import { ConversationRuntimeOwnershipError } from '../conversations/conversation-runtime-errors.js'; -import type { Application, Request, RequestHandler, Response } from 'express'; +import express, { + type Application, + type ErrorRequestHandler, + type Request, + type RequestHandler, + type Response, +} from 'express'; import { writeStderrLine } from '../../utils/stdioHelpers.js'; import { parseCallerSuppliedSessionId } from '../../config/session-id.js'; import { isChannelDeliveryError } from '../../runtime/channel-delivery-ipc.js'; @@ -59,6 +65,7 @@ import { SessionShellClientRequiredError, SessionShellDisabledError, type AcpSessionBridge, + type BridgePromptContentBlock, } from '../acp-session-bridge.js'; import type { DaemonLogger } from '../daemon-logger.js'; import type { SendBridgeError } from '../server/error-response.js'; @@ -204,6 +211,78 @@ const TRANSCRIPT_CURSOR_TOO_LARGE_REPLAY_ERROR = 'Transcript pagination state exceeds the safe limit'; // Must exceed CHANNEL_DELIVERY_IPC_TIMEOUT_MS (30 s, channel-delivery-ipc.ts) plus scheduling slack. const CHANNEL_DELIVERY_AUTHORIZATION_GRACE_MS = 60_000; +// Media blocks are resolved into inline bytes at dispatch, so an unbounded +// content array lets one small request fan out into gigabytes of heap (a +// repeated reference resolves to the same 8 MiB image once per occurrence). +// 256 matches the session media store's item cap, so a message can still +// reference every stored item. +const MEDIA_CONTENT_MAX_BLOCKS = 256; + +// SVG can carry scripts; this origin also hosts the daemon API and Web +// Shell UI, and stored bytes are served back to browsers — raster formats +// only. Compare the normalized media type: standards-conformant spelling +// variants (`image/svg+xml;charset=utf-8`, `image/SVG+XML`) must not slip +// past an exact-string match. +function isSvgMimeType(mimeType: string | undefined): boolean { + return mimeType?.split(';', 1)[0]?.trim().toLowerCase() === 'image/svg+xml'; +} + +// Shared per-block validation for the prompt and mid-turn routes. SVG can +// carry scripts; this origin also hosts the daemon API and Web Shell UI, and +// stored bytes are served back to browsers — raster formats only, matching +// the upload route's policy. +type MediaBlockParseResult = + | { valid: true; block: BridgePromptContentBlock } + | { valid: false; code: 'not-object' | 'invalid-shape' | 'svg' }; + +function parseMediaContentBlock(block: unknown): MediaBlockParseResult { + if (typeof block !== 'object' || block === null || Array.isArray(block)) { + return { valid: false, code: 'not-object' }; + } + const record = block as Record; + const type = record['type']; + const data = record['data']; + const mediaId = record['mediaId']; + const mimeType = record['mimeType']; + const size = record['size']; + const inline = typeof data === 'string' && data.length > 0; + const reference = + typeof mediaId === 'string' && + mediaId.length > 0 && + typeof size === 'number' && + Number.isSafeInteger(size) && + size > 0; + if ( + type !== 'image' || + inline === reference || + typeof mimeType !== 'string' || + !mimeType.startsWith(`${type}/`) + ) { + return { valid: false, code: 'invalid-shape' }; + } + if (isSvgMimeType(mimeType)) { + return { valid: false, code: 'svg' }; + } + return { + valid: true, + block: inline + ? ({ type, data, mimeType } as BridgePromptContentBlock) + : ({ type, mediaId, mimeType, size } as BridgePromptContentBlock), + }; +} + +function mediaBlockParseError( + code: 'not-object' | 'invalid-shape' | 'svg', + entryLabel: string, +): string { + if (code === 'not-object') { + return `each ${entryLabel} must be a media content block`; + } + if (code === 'svg') { + return 'SVG images are not supported'; + } + return `each ${entryLabel} must be an image block with either \`data\`, or \`mediaId\` and \`size\`, plus a matching \`mimeType\``; +} const PRIMARY_ONLY_LIVE_SESSION_ROUTES = ['POST /session/:id/cd'] as const; const PRIMARY_OR_INTERNAL_LIVE_SESSION_ROUTES = [ 'POST /session/:id/branch', @@ -4182,6 +4261,123 @@ export function registerSessionRoutes( ), ); + app.post( + '/session/:id/media', + mutate(), + express.raw({ type: 'image/*', limit: '8mb' }), + ((error, _req, res, next) => { + if ( + error && + typeof error === 'object' && + 'status' in error && + error.status === 413 + ) { + res.status(413).json({ error: 'Request body too large (max 8 MiB)' }); + return; + } + next(error); + }) satisfies ErrorRequestHandler, + withOwnerMutableSession( + 'POST /session/:id/media', + async (req, res, sessionId, runtime) => { + const contentType = req.headers['content-type'] + ?.split(';', 1)[0] + ?.trim() + .toLowerCase(); + // SVG can carry scripts; this origin also hosts the daemon API and + // Web Shell UI, and the bytes are served back to browsers, so an + // inline SVG would run same-origin. Raster formats only. + if (isSvgMimeType(contentType)) { + res.status(415).json({ error: 'SVG uploads are not supported' }); + return; + } + if ( + !contentType || + !contentType.startsWith('image/') || + !Buffer.isBuffer(req.body) || + req.body.byteLength === 0 + ) { + res.status(400).json({ + error: + 'request body must contain image/* bytes with a matching Content-Type', + }); + return; + } + const clientId = parseClientIdHeader(req, res); + if (clientId === null) return; + try { + const reference = await runtime.bridge.storeSessionMedia( + sessionId, + req.body, + contentType, + clientId !== undefined ? { clientId } : undefined, + ); + res.status(201).json(reference); + } catch (error) { + if (error instanceof RangeError) { + res.status(413).json({ error: error.message }); + return; + } + throw error; + } + }, + ), + ); + + app.get( + '/session/:id/media/:mediaId', + withOwnerReadSession( + 'GET /session/:id/media/:mediaId', + async (req, res, sessionId, runtime) => { + const mediaId = req.params['mediaId']; + if (!mediaId) { + res.status(400).json({ error: '`mediaId` is required' }); + return; + } + const clientId = parseClientIdHeader(req, res); + if (clientId === null) return; + const media = await runtime.bridge.readSessionMedia( + sessionId, + mediaId, + clientId !== undefined ? { clientId } : undefined, + ); + if (!media) { + res.status(404).json({ error: 'session media not found' }); + return; + } + res.setHeader('Content-Type', media.mimeType); + res.setHeader('Content-Length', String(media.data.byteLength)); + res.setHeader('Cache-Control', 'private, max-age=300'); + res.setHeader('Content-Disposition', 'attachment'); + res.setHeader('X-Content-Type-Options', 'nosniff'); + res.status(200).send(media.data); + }, + ), + ); + + app.delete( + '/session/:id/media/:mediaId', + mutate(), + withOwnerMutableSession( + 'DELETE /session/:id/media/:mediaId', + async (req, res, sessionId, runtime) => { + const mediaId = req.params['mediaId']; + if (!mediaId) { + res.status(400).json({ error: '`mediaId` is required' }); + return; + } + const clientId = parseClientIdHeader(req, res); + if (clientId === null) return; + const removed = await runtime.bridge.removeSessionMedia( + sessionId, + mediaId, + clientId !== undefined ? { clientId } : undefined, + ); + res.status(200).json({ removed }); + }, + ), + ); + app.post( '/session/:id/prompt', mutate(), @@ -4209,6 +4405,31 @@ export function registerSessionRoutes( }); return; } + const mediaBlockCount = prompt.filter( + (item: unknown) => + (item as Record)['type'] !== 'text', + ).length; + if (mediaBlockCount > MEDIA_CONTENT_MAX_BLOCKS) { + res.status(400).json({ + error: `\`prompt\` must carry at most ${MEDIA_CONTENT_MAX_BLOCKS} media blocks`, + }); + return; + } + // Same per-block validation as the mid-turn route, scoped to image + // blocks: a malformed image admitted here only fails the ACP child's + // schema parse later, surfacing an async turn error instead of a + // synchronous 400. Other non-text blocks (legacy inline audio, + // embedded resources) keep their pre-existing child-side validation. + for (const item of prompt) { + if ((item as Record)['type'] !== 'image') continue; + const parsed = parseMediaContentBlock(item); + if (!parsed.valid) { + res.status(400).json({ + error: mediaBlockParseError(parsed.code, '`prompt` image block'), + }); + return; + } + } const rawRequestDeadline = body['deadlineMs']; let requestDeadlineMs: number | undefined; if (rawRequestDeadline !== undefined && rawRequestDeadline !== null) { @@ -5619,9 +5840,42 @@ export function registerSessionRoutes( // stores the trimmed string, so checking the raw length would reject input // whose real content fits but is padded with whitespace. const trimmed = typeof message === 'string' ? message.trim() : ''; - if (trimmed.length === 0) { + // Optional image blocks injected mid-turn alongside the + // text. Validate strictly here — the ACP child silently drops blocks + // that fail its own `isContentBlock` check, so a malformed block would + // vanish from the turn without any error. Media size is bounded by the + // global request body limit. + const rawContent = body['content']; + let mediaBlocks: BridgePromptContentBlock[] | undefined; + if (rawContent !== undefined) { + if (!Array.isArray(rawContent) || rawContent.length === 0) { + res.status(400).json({ + error: '`content` must be a non-empty array of media blocks', + }); + return; + } + if (rawContent.length > MEDIA_CONTENT_MAX_BLOCKS) { + res.status(400).json({ + error: `\`content\` must carry at most ${MEDIA_CONTENT_MAX_BLOCKS} media blocks`, + }); + return; + } + mediaBlocks = []; + for (const block of rawContent) { + const parsed = parseMediaContentBlock(block); + if (!parsed.valid) { + res.status(400).json({ + error: mediaBlockParseError(parsed.code, '`content` entry'), + }); + return; + } + mediaBlocks.push(parsed.block); + } + } + if (trimmed.length === 0 && mediaBlocks === undefined) { res.status(400).json({ - error: '`message` is required and must be a non-empty string', + error: + '`message` must be a non-empty string, or `content` must carry at least one media block', }); return; } @@ -5654,6 +5908,7 @@ export function registerSessionRoutes( trimmed, clientId !== undefined ? { clientId } : undefined, typeof messageId === 'string' ? messageId : undefined, + mediaBlocks ? { content: mediaBlocks } : undefined, ); res.status(200).json(result); }, diff --git a/packages/cli/src/serve/server.test.ts b/packages/cli/src/serve/server.test.ts index 59056ec1c2b..4f1cc0d9433 100644 --- a/packages/cli/src/serve/server.test.ts +++ b/packages/cli/src/serve/server.test.ts @@ -533,6 +533,7 @@ const EXPECTED_STAGE1_FEATURES = [ 'session_source_metadata', 'session_side_task', 'session_prompt', + 'session_media', 'session_mid_turn_message_mutation', 'session_mid_turn_message_query', 'session_cancel', @@ -1292,6 +1293,7 @@ function fakeBridge(opts: FakeBridgeOpts = {}): FakeBridge { worktree: { slug: string; path: string; branch: string }; }> = []; const enqueueMidTurnCalls: FakeBridge['enqueueMidTurnCalls'] = []; + const sessionMedia = new Map(); const enqueueMidTurnImpl = opts.enqueueMidTurnImpl ?? (() => ({ accepted: true, messageId: 'mid-default' })); @@ -2292,12 +2294,29 @@ function fakeBridge(opts: FakeBridgeOpts = {}): FakeBridge { async isWorkspaceMemoryRememberAvailable() { return true; }, - enqueueMidTurnMessage(sessionId, message, context, messageId) { + async storeSessionMedia(_sessionId, data, mimeType) { + const mediaId = `media-${sessionMedia.size + 1}`; + sessionMedia.set(mediaId, { data: Buffer.from(data), mimeType }); + return { + type: 'image', + mediaId, + mimeType, + size: data.byteLength, + }; + }, + async readSessionMedia(_sessionId, mediaId) { + return sessionMedia.get(mediaId); + }, + async removeSessionMedia(_sessionId, mediaId) { + return sessionMedia.delete(mediaId); + }, + enqueueMidTurnMessage(sessionId, message, context, messageId, options) { enqueueMidTurnCalls.push({ sessionId, message, ...(context ? { context } : {}), ...(messageId ? { messageId } : {}), + ...(options ? { options } : {}), }); return enqueueMidTurnImpl(sessionId, message, context, messageId); }, @@ -9326,6 +9345,134 @@ describe('createServeApp', () => { }); }); + describe('session media', () => { + it('uploads and reads session-scoped binary media', async () => { + const app = createServeApp( + { ...baseOpts, token: 'secret', workspace: WS_BOUND }, + undefined, + { bridge: fakeBridge() }, + ); + const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + const uploaded = await request(app) + .post('/session/s-1/media') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret') + .set('Content-Type', 'image/png') + .send(bytes); + + expect(uploaded.status).toBe(201); + expect(uploaded.body).toEqual({ + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: bytes.length, + }); + + const downloaded = await request(app) + .get('/session/s-1/media/media-1') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret') + .buffer(true); + expect(downloaded.status).toBe(200); + expect(downloaded.headers['content-type']).toBe('image/png'); + expect(downloaded.body).toEqual(bytes); + + const removed = await request(app) + .delete('/session/s-1/media/media-1') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret'); + expect(removed.status).toBe(200); + expect(removed.body).toEqual({ removed: true }); + + const missing = await request(app) + .get('/session/s-1/media/media-1') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret'); + expect(missing.status).toBe(404); + }); + + it('rejects non-image uploads', async () => { + const app = createServeApp( + { ...baseOpts, token: 'secret', workspace: WS_BOUND }, + undefined, + { bridge: fakeBridge() }, + ); + const response = await request(app) + .post('/session/s-1/media') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret') + .set('Content-Type', 'audio/wav') + .send(Buffer.from([1])); + + expect(response.status).toBe(400); + }); + + it('rejects SVG uploads', async () => { + // SVG can carry scripts and the bytes are served back to browsers on + // the same origin as the daemon API and Web Shell UI. + const app = createServeApp( + { ...baseOpts, token: 'secret', workspace: WS_BOUND }, + undefined, + { bridge: fakeBridge() }, + ); + const response = await request(app) + .post('/session/s-1/media') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret') + .set('Content-Type', 'image/svg+xml') + .send(Buffer.from('')); + + expect(response.status).toBe(415); + expect(response.body).toEqual({ + error: 'SVG uploads are not supported', + }); + }); + + it('serves stored media with download-safe headers', async () => { + const app = createServeApp( + { ...baseOpts, token: 'secret', workspace: WS_BOUND }, + undefined, + { bridge: fakeBridge() }, + ); + const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + const uploaded = await request(app) + .post('/session/s-1/media') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret') + .set('Content-Type', 'image/png') + .send(bytes); + expect(uploaded.status).toBe(201); + + const downloaded = await request(app) + .get('/session/s-1/media/media-1') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret') + .buffer(true); + expect(downloaded.status).toBe(200); + expect(downloaded.headers['content-disposition']).toBe('attachment'); + expect(downloaded.headers['x-content-type-options']).toBe('nosniff'); + }); + + it('reports the media route 8 MiB body limit accurately', async () => { + const app = createServeApp( + { ...baseOpts, token: 'secret', workspace: WS_BOUND }, + undefined, + { bridge: fakeBridge() }, + ); + const response = await request(app) + .post('/session/s-1/media') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .set('Authorization', 'Bearer secret') + .set('Content-Type', 'image/png') + .send(Buffer.alloc(8 * 1024 * 1024 + 1)); + + expect(response.status).toBe(413); + expect(response.body).toEqual({ + error: 'Request body too large (max 8 MiB)', + }); + }); + }); + describe('POST /session/:id/mid-turn-message', () => { const midTurnPost = ( app: ReturnType, @@ -9428,6 +9575,108 @@ describe('createServeApp', () => { expect(bridge.enqueueMidTurnCalls).toEqual([]); }); + it('forwards validated media blocks to the bridge', async () => { + const bridge = fakeBridge(); + const res = await midTurnPost(midTurnApp(bridge), 's-1', { + message: 'see this', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }); + expect(res.status).toBe(200); + expect(bridge.enqueueMidTurnCalls).toEqual([ + { + sessionId: 's-1', + message: 'see this', + options: { + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }, + }, + ]); + }); + + it('admits an empty message when media blocks are present', async () => { + const bridge = fakeBridge(); + const res = await midTurnPost(midTurnApp(bridge), 's-1', { + message: '', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }); + expect(res.status).toBe(200); + expect(bridge.enqueueMidTurnCalls).toHaveLength(1); + }); + + it.each([ + ['not an array', { message: 'hi', content: 'nope' }], + ['empty array', { message: 'hi', content: [] }], + ['non-object block', { message: 'hi', content: ['block'] }], + [ + 'unknown block type', + { message: 'hi', content: [{ type: 'text', text: 'hi' }] }, + ], + [ + 'missing data', + { message: 'hi', content: [{ type: 'image', mimeType: 'image/png' }] }, + ], + [ + 'mismatched mimeType', + { + message: 'hi', + content: [{ type: 'image', data: 'aW1n', mimeType: 'audio/mp3' }], + }, + ], + ])('400 when `content` is invalid: %s', async (_label, body) => { + const bridge = fakeBridge(); + const res = await midTurnPost(midTurnApp(bridge), 's-1', body); + expect(res.status).toBe(400); + expect(bridge.enqueueMidTurnCalls).toEqual([]); + }); + + it.each([ + ['exact', 'image/svg+xml'], + ['parameter suffix', 'image/svg+xml;charset=utf-8'], + ['uppercase', 'image/SVG+XML'], + ])( + '400 when `content` carries an SVG block: %s (raster-only policy)', + async (_label, mimeType) => { + // The upload route rejects SVG after normalizing the media type; + // the inline-block gate must reject the same spelling variants — an + // exact-string match lets standards-conformant variants through. + const bridge = fakeBridge(); + const res = await midTurnPost(midTurnApp(bridge), 's-1', { + message: 'hi', + content: [{ type: 'image', data: 'PHN2Zz4=', mimeType }], + }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('SVG images are not supported'); + expect(bridge.enqueueMidTurnCalls).toEqual([]); + }, + ); + + it('forwards reference-form media blocks to the bridge verbatim', async () => { + const bridge = fakeBridge(); + const res = await midTurnPost(midTurnApp(bridge), 's-1', { + message: 'see this', + content: [ + { type: 'image', mediaId: 'media-1', mimeType: 'image/png', size: 4 }, + ], + }); + expect(res.status).toBe(200); + expect(bridge.enqueueMidTurnCalls).toEqual([ + { + sessionId: 's-1', + message: 'see this', + options: { + content: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 4, + }, + ], + }, + }, + ]); + }); + it('400 when the trimmed message exceeds the 16 KB cap', async () => { const bridge = fakeBridge(); const res = await midTurnPost(midTurnApp(bridge), 's-1', { @@ -9437,6 +9686,22 @@ describe('createServeApp', () => { expect(bridge.enqueueMidTurnCalls).toEqual([]); }); + it('400 when `content` carries more than 256 media blocks', async () => { + // Media blocks are resolved into inline bytes at dispatch; an unbounded + // array amplifies one small request into gigabytes of heap. + const bridge = fakeBridge(); + const res = await midTurnPost(midTurnApp(bridge), 's-1', { + message: 'hi', + content: Array.from({ length: 257 }, () => ({ + type: 'image', + data: 'aW1n', + mimeType: 'image/png', + })), + }); + expect(res.status).toBe(400); + expect(bridge.enqueueMidTurnCalls).toEqual([]); + }); + it('maps a bridge SessionNotFoundError to 404', async () => { const bridge = fakeBridge({ enqueueMidTurnImpl: (sessionId) => { @@ -12887,6 +13152,129 @@ describe('createServeApp', () => { expect(bridge.promptCalls[0]?.req.sessionId).toBe('session-A'); }); + it('400 when the prompt carries more than 256 media blocks', async () => { + // Same amplification guard as the mid-turn route: repeated media + // references resolve into per-occurrence inline bytes at dispatch. + const bridge = fakeBridge({ + promptImpl: async () => ({ stopReason: 'end_turn' }), + }); + const app = createServeApp(baseOpts, undefined, { bridge }); + const res = await request(app) + .post('/session/session-A/prompt') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .send({ + prompt: [ + { type: 'text', text: 'hi' }, + ...Array.from({ length: 257 }, () => ({ + type: 'image', + data: 'aW1n', + mimeType: 'image/png', + })), + ], + }); + expect(res.status).toBe(400); + expect(bridge.promptCalls).toHaveLength(0); + }); + + it('400 when a non-text prompt block is malformed (validated before admission)', async () => { + // Without per-block validation the block is admitted and only fails + // the ACP child's schema parse later, surfacing an async turn error + // instead of a synchronous 400. + const bridge = fakeBridge({ + promptImpl: async () => ({ stopReason: 'end_turn' }), + }); + const app = createServeApp(baseOpts, undefined, { bridge }); + const res = await request(app) + .post('/session/session-A/prompt') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .send({ + prompt: [ + { type: 'text', text: 'hi' }, + { type: 'image', data: 'aW1n' }, + ], + }); + expect(res.status).toBe(400); + expect(bridge.promptCalls).toHaveLength(0); + }); + + it.each([ + ['exact', 'image/svg+xml'], + ['parameter suffix', 'image/svg+xml;charset=utf-8'], + ['uppercase', 'image/SVG+XML'], + ])( + '400 when a prompt media block is SVG: %s (raster-only policy)', + async (_label, mimeType) => { + // Same normalizing gate as the mid-turn route and the upload route. + const bridge = fakeBridge({ + promptImpl: async () => ({ stopReason: 'end_turn' }), + }); + const app = createServeApp(baseOpts, undefined, { bridge }); + const res = await request(app) + .post('/session/session-A/prompt') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .send({ + prompt: [ + { type: 'text', text: 'hi' }, + { type: 'image', data: 'PHN2Zz4=', mimeType }, + ], + }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('SVG images are not supported'); + expect(bridge.promptCalls).toHaveLength(0); + }, + ); + + it('202 still admits legacy inline audio blocks (child-side validation)', async () => { + // The per-block validation is scoped to image blocks; legacy audio + // prompts keep their pre-existing behavior (the ACP child validates). + const bridge = fakeBridge({ + promptImpl: async () => ({ stopReason: 'end_turn' }), + }); + const app = createServeApp(baseOpts, undefined, { bridge }); + const res = await request(app) + .post('/session/session-A/prompt') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .send({ + prompt: [ + { type: 'text', text: 'hi' }, + { type: 'audio', data: 'YXVk', mimeType: 'audio/wav' }, + ], + }); + expect(res.status).toBe(202); + await new Promise((r) => setTimeout(r, 20)); + expect(bridge.promptCalls).toHaveLength(1); + }); + + it('202 accepts valid inline and reference media blocks', async () => { + const bridge = fakeBridge({ + promptImpl: async () => ({ stopReason: 'end_turn' }), + }); + const app = createServeApp(baseOpts, undefined, { bridge }); + const res = await request(app) + .post('/session/session-A/prompt') + .set('Host', `127.0.0.1:${baseOpts.port}`) + .send({ + prompt: [ + { type: 'text', text: 'hi' }, + { type: 'image', data: 'aW1n', mimeType: 'image/png' }, + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 4, + }, + ], + }); + expect(res.status).toBe(202); + await new Promise((r) => setTimeout(r, 20)); + expect(bridge.promptCalls).toHaveLength(1); + expect(bridge.promptCalls[0]?.req.prompt).toEqual([ + { type: 'text', text: 'hi' }, + { type: 'image', data: 'aW1n', mimeType: 'image/png' }, + { type: 'image', mediaId: 'media-1', mimeType: 'image/png', size: 4 }, + ]); + }); + it('202 envelope carries eventEpoch alongside lastEventId (DAEMON-001)', async () => { // A client seeding its SSE resume cursor from this 202 must also // learn the bus epoch, or a daemon restart between the 202 and the diff --git a/packages/cli/src/serve/server/error-response.test.ts b/packages/cli/src/serve/server/error-response.test.ts index 8a16f99e093..d21a80896cd 100644 --- a/packages/cli/src/serve/server/error-response.test.ts +++ b/packages/cli/src/serve/server/error-response.test.ts @@ -122,4 +122,20 @@ describe('sendBridgeError session writer errors', () => { errorKind: 'session_writer_unavailable', }); }); + + it.each([ + ['invalid_session_media_reference', 400], + ['session_media_gone', 410], + ] as const)('maps %s to %i', (code, expectedStatus) => { + const { response, status, json } = responseMock(); + const error = Object.assign(new Error('media reference failed'), { code }); + + sendBridgeError(response, error); + + expect(status).toHaveBeenCalledWith(expectedStatus); + expect(json).toHaveBeenCalledWith({ + error: 'media reference failed', + code, + }); + }); }); diff --git a/packages/cli/src/serve/server/error-response.ts b/packages/cli/src/serve/server/error-response.ts index 11e79ac405f..565fcc3d4dc 100644 --- a/packages/cli/src/serve/server/error-response.ts +++ b/packages/cli/src/serve/server/error-response.ts @@ -235,6 +235,17 @@ export function sendBridgeError( return; } if (sendGenerationClosedError(res, err)) return; + if ( + err instanceof Error && + 'code' in err && + (err.code === 'session_media_gone' || + err.code === 'invalid_session_media_reference') + ) { + res + .status(err.code === 'session_media_gone' ? 410 : 400) + .json({ error: err.message, code: err.code }); + return; + } if (err instanceof SessionWriterError) { res.status(err.httpStatus).json({ error: err.message, diff --git a/packages/cli/src/serve/server/telemetry-catalog.test.ts b/packages/cli/src/serve/server/telemetry-catalog.test.ts index 3c0a192f488..d47dee8eb5c 100644 --- a/packages/cli/src/serve/server/telemetry-catalog.test.ts +++ b/packages/cli/src/serve/server/telemetry-catalog.test.ts @@ -98,7 +98,7 @@ describe('legacy session telemetry route drift guard', () => { .map(({ method, path }) => `${method} ${path}`) .sort(); - expect(registered).toHaveLength(54); + expect(registered).toHaveLength(57); expect(registered).toEqual(catalog); }); }); diff --git a/packages/cli/src/serve/server/telemetry.test.ts b/packages/cli/src/serve/server/telemetry.test.ts index 47353dcd6f1..2e8f4e8e5af 100644 --- a/packages/cli/src/serve/server/telemetry.test.ts +++ b/packages/cli/src/serve/server/telemetry.test.ts @@ -812,17 +812,17 @@ describe('daemonTelemetryMiddleware — recordRequest seam', () => { }); describe('legacy session telemetry route catalog', () => { - it('contains 54 unique routes with the audited 52/2 attribution split', () => { + it('contains 57 unique routes with the audited 55/2 attribution split', () => { const keys = legacySessionTelemetryRoutes.map( ({ method, path }) => `${method} ${path}`, ); - expect(keys).toHaveLength(54); - expect(new Set(keys).size).toBe(54); + expect(keys).toHaveLength(57); + expect(new Set(keys).size).toBe(57); expect( legacySessionTelemetryRoutes.filter( ({ attribution }) => attribution === 'handler_resolved', ), - ).toHaveLength(52); + ).toHaveLength(55); expect( legacySessionTelemetryRoutes.filter( ({ attribution }) => attribution === 'pre_resolved', diff --git a/packages/cli/src/serve/server/telemetry.ts b/packages/cli/src/serve/server/telemetry.ts index 4c3e0167f33..e66484be794 100644 --- a/packages/cli/src/serve/server/telemetry.ts +++ b/packages/cli/src/serve/server/telemetry.ts @@ -179,6 +179,24 @@ export const legacySessionTelemetryRoutes = [ attribution: 'handler_resolved', route: 'POST /session/:id/continue', }, + { + method: 'POST', + path: '/session/:id/media', + attribution: 'handler_resolved', + route: 'POST /session/:id/media', + }, + { + method: 'GET', + path: '/session/:id/media/:mediaId', + attribution: 'handler_resolved', + route: 'GET /session/:id/media/:mediaId', + }, + { + method: 'DELETE', + path: '/session/:id/media/:mediaId', + attribution: 'handler_resolved', + route: 'DELETE /session/:id/media/:mediaId', + }, { method: 'POST', path: '/session/:id/prompt', diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts index 44487572302..7b62676e2e9 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts @@ -634,6 +634,96 @@ describe('resumeHistoryUtils', () => { }); }); + it('restores media-reference mid-turn messages as an attachment placeholder', () => { + // Image-only mid-turn messages are recorded with an empty displayText and + // mediaReferences; resuming must not fall back to the raw internal prefix. + const conversation = { + messages: [ + { + type: 'user', + subtype: 'mid_turn_user_message', + message: { + parts: [ + { + text: '\n[User message received during tool execution]: ', + } as Part, + ], + }, + systemPayload: { + displayText: '', + mediaReferences: [ + { + type: 'image', + mediaId: 'image-1', + mimeType: 'image/png', + size: 8, + }, + ], + }, + }, + ], + } as unknown as ConversationRecord; + + const session: ResumedSessionData = { + conversation, + } as ResumedSessionData; + + const items = buildResumedHistoryItems( + session, + makeConfig({ replace: mockTool }), + 40, + ); + + expect(items).toContainEqual({ + id: 41, + type: 'user', + text: '[User message with attachments]', + sentToModel: false, + }); + }); + + it('restores media-reference ordinary user messages as an attachment placeholder', () => { + // Image-only prompts are recorded with an empty displayText and + // mediaReferences; resuming must keep the prompt visible instead of + // dropping it from the restored history. + const conversation = { + messages: [ + { + type: 'user', + message: { + parts: [ + { + inlineData: { mimeType: 'image/png', data: 'aW1n' }, + } as Part, + ], + }, + systemPayload: { + displayText: '', + hookContext: '', + mediaReferences: [ + { + type: 'image', + mediaId: 'image-1', + mimeType: 'image/png', + size: 8, + }, + ], + }, + }, + ], + } as unknown as ConversationRecord; + + const session: ResumedSessionData = { + conversation, + } as ResumedSessionData; + + const items = buildResumedHistoryItems(session, makeConfig({}), 50); + + expect(items).toEqual([ + { id: 51, type: 'user', text: '[User message with attachments]' }, + ]); + }); + it('restores ordinary user messages from clean display text', () => { const conversation = { messages: [ diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.ts index d59c4c6f0c6..5a5110914d2 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.ts @@ -386,11 +386,16 @@ function convertToHistoryItems( } if (record.subtype === 'mid_turn_user_message') { const payload = record.systemPayload as - | { displayText?: string } + | { displayText?: string; mediaReferences?: unknown[] } | undefined; + const hasMediaReferences = + Array.isArray(payload?.mediaReferences) && + payload.mediaReferences.length > 0; const text = payload?.displayText || - extractTextFromParts(record.message?.parts as Part[]); + (hasMediaReferences + ? '[User message with attachments]' + : extractTextFromParts(record.message?.parts as Part[])); if (text) { items.push({ type: MessageType.USER, text, sentToModel: false }); } @@ -434,8 +439,17 @@ function convertToHistoryItems( } const projection = projectUserTranscriptForDisplay(record); + const payload = record.systemPayload as + | { mediaReferences?: unknown[] } + | undefined; + const hasMediaReferences = + Array.isArray(payload?.mediaReferences) && + payload.mediaReferences.length > 0; const text = - projection.displayText ?? extractTextFromParts(projection.parts); + projection.displayText || + (hasMediaReferences + ? '[User message with attachments]' + : extractTextFromParts(projection.parts)); if (text) { items.push({ type: 'user', text }); } diff --git a/packages/core/src/services/chatRecordingService.test.ts b/packages/core/src/services/chatRecordingService.test.ts index cd3dac199a0..5a884f6a339 100644 --- a/packages/core/src/services/chatRecordingService.test.ts +++ b/packages/core/src/services/chatRecordingService.test.ts @@ -328,6 +328,60 @@ describe('ChatRecordingService', () => { expect(record.systemPayload).toEqual({ displayText: 'save logs' }); }); + it('records mid-turn media references without inline bytes', async () => { + const mediaReferences = [ + { + type: 'image' as const, + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ]; + + chatRecordingService.recordMidTurnUserMessage( + [{ text: 'inspect image' }], + 'inspect image', + undefined, + mediaReferences, + ); + await chatRecordingService.flush(); + + const record = vi.mocked(jsonl.writeLine).mock.calls[0][1] as ChatRecord; + expect(record.message).toEqual({ + role: 'user', + parts: [{ text: 'inspect image' }], + }); + expect(record.systemPayload).toEqual({ + displayText: 'inspect image', + mediaReferences, + }); + }); + + it('records media references when the mid-turn display text is empty', async () => { + const mediaReferences = [ + { + type: 'image' as const, + mediaId: 'media-only', + mimeType: 'image/png', + size: 3, + }, + ]; + + chatRecordingService.recordMidTurnUserMessage( + [{ text: '[User message received during tool execution]: ' }], + '', + undefined, + mediaReferences, + ); + await chatRecordingService.flush(); + + const record = vi.mocked(jsonl.writeLine).mock.calls[0][1] as ChatRecord; + expect(record.systemPayload).toEqual({ + displayText: '', + mediaReferences, + }); + }); + it('records defensive Goal context on real user messages', async () => { const topLevelPermit: GoalTurnPermit = { goalId: 'goal-1', diff --git a/packages/core/src/services/chatRecordingService.ts b/packages/core/src/services/chatRecordingService.ts index 00643eeedfd..6ef3b930180 100644 --- a/packages/core/src/services/chatRecordingService.ts +++ b/packages/core/src/services/chatRecordingService.ts @@ -408,6 +408,7 @@ export interface ChatRecord { export interface NotificationRecordPayload { displayText: string; + mediaReferences?: UserPromptMediaReference[]; backgroundTask?: { taskId: string; status: string; @@ -429,6 +430,15 @@ export interface UserPromptRecordPayload { displayText: string; /** Sanitized hook context duplicated from the tagged model-bound part. */ hookContext: string; + /** Daemon-owned media references used to restore prompt previews. */ + mediaReferences?: UserPromptMediaReference[]; +} + +export interface UserPromptMediaReference { + type: 'image' | 'audio'; + mediaId: string; + mimeType: string; + size: number; } export interface AgentBootstrapRecordPayload { @@ -1571,8 +1581,9 @@ export class ChatRecordingService { */ recordMidTurnUserMessage( message: PartListUnion, - displayText?: string, + displayText: string, goalContext?: GoalTurnPermit, + mediaReferences?: UserPromptMediaReference[], ): void { try { const record: ChatRecord = { @@ -1580,9 +1591,10 @@ export class ChatRecordingService { subtype: 'mid_turn_user_message', ...(goalContext ? { goalContext: copyGoalContext(goalContext) } : {}), message: createUserContent(message), - systemPayload: displayText - ? ({ displayText } as NotificationRecordPayload) - : undefined, + systemPayload: { + displayText, + ...(mediaReferences ? { mediaReferences } : {}), + }, }; this.appendRecord(record); } catch (error) { @@ -1684,7 +1696,10 @@ export class ChatRecordingService { ...(goalContext ? { goalContext: copyGoalContext(goalContext) } : {}), message: createUserContent(message), systemPayload: displayText - ? ({ displayText, backgroundTask } as NotificationRecordPayload) + ? { + displayText, + ...(backgroundTask ? { backgroundTask } : {}), + } : undefined, }; } diff --git a/packages/sdk-typescript/scripts/build.js b/packages/sdk-typescript/scripts/build.js index 72a0923d755..d3ffa560ef9 100755 --- a/packages/sdk-typescript/scripts/build.js +++ b/packages/sdk-typescript/scripts/build.js @@ -87,9 +87,12 @@ const rootDir = join(__dirname, '..'); // APIs merged in from main. // Bumped from 189KB to 190KB for historical branch sessions and transcript // branch-point projection merged with the upload and reasoning APIs. -// Bumped from 190KB to 191KB for the composer text-file attachment metadata -// (#9180) on the local optimistic user transcript surface. -const MAX_DAEMON_BROWSER_BUNDLE_BYTES = 191 * 1024; +// Bumped from 190KB to 195KB for session media upload, cleanup, and hydration +// merged with the branch-session APIs and the composer text-file attachment +// metadata (#9180). +// Bumped from 195KB to 196KB for transient-vs-gone media hydration errors and +// the reference-only replay placeholder. +const MAX_DAEMON_BROWSER_BUNDLE_BYTES = 196 * 1024; // The opt-in `daemon/transports` browser bundle legitimately ships the concrete // ACP transports (AcpHttpTransport/AcpWsTransport/AutoReconnect + negotiate), so // it's larger than the default barrel — but still budgeted so a future PR can't diff --git a/packages/sdk-typescript/src/daemon/DaemonClient.ts b/packages/sdk-typescript/src/daemon/DaemonClient.ts index 263888c83f7..63c73161de0 100644 --- a/packages/sdk-typescript/src/daemon/DaemonClient.ts +++ b/packages/sdk-typescript/src/daemon/DaemonClient.ts @@ -157,6 +157,8 @@ import type { DaemonMcpManageAction, DaemonMcpManageResult, DaemonSessionBtwResult, + DaemonSessionMediaData, + DaemonSessionMediaReference, DaemonSessionGenerationEvent, DaemonMidTurnMessageResult, DaemonMidTurnMessagesResult, @@ -3252,16 +3254,105 @@ export class DaemonClient { return (await res.json()) as DaemonSessionBtwResult; } + async uploadSessionMedia( + sessionId: string, + data: Blob, + mimeType: string, + opts?: { signal?: AbortSignal; clientId?: string }, + ): Promise { + return await this.fetchWithTimeout( + `${this.baseUrl}/session/${urlEncode(sessionId)}/media`, + { + method: 'POST', + headers: this.headers({ 'Content-Type': mimeType }, opts?.clientId), + body: data, + signal: opts?.signal, + }, + async (res) => { + if (!res.ok) { + throw await this.failOnError(res, 'POST /session/:id/media'); + } + return (await res.json()) as DaemonSessionMediaReference; + }, + ); + } + + async readSessionMedia( + sessionId: string, + mediaId: string, + opts?: { signal?: AbortSignal; clientId?: string }, + ): Promise { + return await this.fetchWithTimeout( + `${this.baseUrl}/session/${urlEncode(sessionId)}/media/${urlEncode(mediaId)}`, + { + method: 'GET', + headers: this.headers({}, opts?.clientId), + signal: opts?.signal, + }, + async (res) => { + if (!res.ok) { + throw await this.failOnError(res, 'GET /session/:id/media/:mediaId'); + } + const bytes = new Uint8Array(await res.arrayBuffer()); + // This package also targets browsers, where Node's Buffer is absent. + // Chunking keeps the spread call below the engine's argument limit. + let binary = ''; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode( + ...bytes.subarray(offset, offset + 0x8000), + ); + } + return { + data: btoa(binary), + mimeType: + res.headers.get('content-type') ?? 'application/octet-stream', + }; + }, + ); + } + + async removeSessionMedia( + sessionId: string, + mediaId: string, + opts?: { signal?: AbortSignal; clientId?: string }, + ): Promise { + return await this.fetchWithTimeout( + `${this.baseUrl}/session/${urlEncode(sessionId)}/media/${urlEncode(mediaId)}`, + { + method: 'DELETE', + headers: this.headers({}, opts?.clientId), + signal: opts?.signal, + }, + async (res) => { + if (!res.ok) { + throw await this.failOnError( + res, + 'DELETE /session/:id/media/:mediaId', + ); + } + return ((await res.json()) as { removed?: unknown }).removed === true; + }, + ); + } + /** * Queue a user message typed while the session's turn is still running. The * ACP child drains it between tool batches so the model sees it before the * turn ends. Every accepted request is daemon-owned; a caller-supplied id - * makes ambiguous retries idempotent. + * makes ambiguous retries idempotent. `opts.content` carries media content + * image blocks alongside the text — pre-flight the + * `session_media` capability; older daemons ignore the + * field and drop the media. */ async enqueueMidTurnMessage( sessionId: string, message: string, - opts?: { signal?: AbortSignal; clientId?: string; messageId?: string }, + opts?: { + signal?: AbortSignal; + clientId?: string; + messageId?: string; + content?: PromptContentBlock[]; + }, ): Promise { // Route through `fetchWithTimeout` like every other method so a hung daemon // can't wedge this promise forever (the caller in `actions.ts` awaits it). @@ -3275,7 +3366,13 @@ export class DaemonClient { { 'Content-Type': 'application/json' }, opts?.clientId, ), - body: JSON.stringify({ message, messageId: opts?.messageId }), + body: JSON.stringify({ + message, + messageId: opts?.messageId, + ...(opts?.content && opts.content.length > 0 + ? { content: opts.content } + : {}), + }), signal: opts?.signal, }, async (res) => { diff --git a/packages/sdk-typescript/src/daemon/DaemonSessionClient.ts b/packages/sdk-typescript/src/daemon/DaemonSessionClient.ts index f7157eb23ae..903e86e9ca0 100644 --- a/packages/sdk-typescript/src/daemon/DaemonSessionClient.ts +++ b/packages/sdk-typescript/src/daemon/DaemonSessionClient.ts @@ -23,6 +23,10 @@ import type { DaemonRewindResult, DaemonRewindSnapshotInfo, DaemonSessionBtwResult, + DaemonSessionMediaData, + DaemonSessionMediaReference, + DaemonSessionTranscriptPage, + DaemonSessionTranscriptPageOptions, DaemonSessionGenerationEvent, DaemonMidTurnMessageResult, DaemonMidTurnMessagesResult, @@ -47,6 +51,7 @@ import type { DaemonSessionTasksStatus, HeartbeatResult, PermissionResponse, + PromptContentBlock, PromptResult, SetModelResult, SessionMetadataResult, @@ -129,6 +134,26 @@ export interface DaemonSessionSubscribeOptions resume?: boolean; } +function isSessionMediaReference( + value: unknown, +): value is DaemonSessionMediaReference { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const record = value as Record; + return ( + record['type'] === 'image' && + typeof record['mediaId'] === 'string' && + record['mediaId'].length > 0 && + typeof record['mimeType'] === 'string' && + record['mimeType'].startsWith(`${record['type']}/`) && + typeof record['size'] === 'number' && + Number.isSafeInteger(record['size']) && + record['size'] > 0 + ); +} + +const MAX_MEDIA_CACHE_BYTES = 32 * 1024 * 1024; +const MAX_MEDIA_CACHE_ENTRIES = 128; + /** * Session-scoped wrapper around `DaemonClient`. * @@ -178,6 +203,11 @@ export class DaemonSessionClient { private reattaching?: Promise; private cancelling?: Promise; private readonly promptLimit: number; + private readonly mediaCache = new Map< + string, + { pending: Promise; size: number } + >(); + private mediaCacheBytes = 0; private readonly _pendingPrompts = new Map< string, { @@ -290,7 +320,7 @@ export class DaemonSessionClient { eventEpoch, ...session } = restored; - return new DaemonSessionClient({ + const result = new DaemonSessionClient({ client, session, hasActivePrompt, @@ -308,6 +338,8 @@ export class DaemonSessionClient { historyAnchorRecordId, replayDegraded, }); + await result.hydrateReplaySnapshot(); + return result; } /** @@ -465,6 +497,35 @@ export class DaemonSessionClient { return accepted; } + async uploadMedia( + data: Blob, + mimeType: string, + signal?: AbortSignal, + ): Promise { + return await this.withClientIdSelfHeal(() => + this.client.uploadSessionMedia(this.sessionId, data, mimeType, { + ...(signal ? { signal } : {}), + ...(this.clientId ? { clientId: this.clientId } : {}), + }), + ); + } + + async removeMedia(mediaId: string): Promise { + const removed = await this.withClientIdSelfHeal(() => + this.client.removeSessionMedia( + this.sessionId, + mediaId, + this.clientId ? { clientId: this.clientId } : undefined, + ), + ); + if (removed) { + const cached = this.mediaCache.get(mediaId); + this.mediaCache.delete(mediaId); + this.mediaCacheBytes -= cached?.size ?? 0; + } + return removed; + } + /** * Run a prompt-admission call, recovering from a stale `clientId`. * @@ -652,11 +713,18 @@ export class DaemonSessionClient { */ async enqueueMidTurnMessage( message: string, - opts?: { signal?: AbortSignal; messageId?: string }, + opts?: { + signal?: AbortSignal; + messageId?: string; + content?: PromptContentBlock[]; + }, ): Promise { return await this.client.enqueueMidTurnMessage(this.sessionId, message, { ...(opts?.signal ? { signal: opts.signal } : {}), ...(opts?.messageId ? { messageId: opts.messageId } : {}), + ...(opts?.content && opts.content.length > 0 + ? { content: opts.content } + : {}), ...(this.clientId ? { clientId: this.clientId } : {}), }); } @@ -679,16 +747,52 @@ export class DaemonSessionClient { async getMidTurnMessages(opts?: { signal?: AbortSignal; }): Promise { - return await this.client.getMidTurnMessages(this.sessionId, { + const result = await this.client.getMidTurnMessages(this.sessionId, { ...(opts?.signal ? { signal: opts.signal } : {}), ...(this.clientId ? { clientId: this.clientId } : {}), }); + return { + ...result, + messages: await Promise.all( + result.messages.map(async (message) => ({ + ...message, + ...(message.content + ? { content: await this.hydrateContent(message.content) } + : {}), + })), + ), + }; } async getPendingPrompts(): Promise { - return await this.client.getPendingPrompts(this.sessionId, { + const result = await this.client.getPendingPrompts(this.sessionId, { ...(this.clientId ? { clientId: this.clientId } : {}), }); + return { + pendingPrompts: await Promise.all( + result.pendingPrompts.map(async (prompt) => ({ + ...prompt, + ...(prompt.content + ? { content: await this.hydrateContent(prompt.content) } + : {}), + })), + ), + }; + } + + async getTranscriptPage( + opts: DaemonSessionTranscriptPageOptions = {}, + ): Promise { + const page = await this.client.getSessionTranscriptPage(this.sessionId, { + ...opts, + clientId: opts.clientId ?? this.clientId, + }); + return { + ...page, + events: await Promise.all( + page.events.map(async (event) => await this.hydrateEvent(event)), + ), + }; } async removePendingPrompt( @@ -928,8 +1032,9 @@ export class DaemonSessionClient { callerOnEpoch?.(learned); }, })) { - this._dispatchTurnEvent(event); - yield event; + const hydratedEvent = await this.hydrateEvent(event); + this._dispatchTurnEvent(hydratedEvent); + yield hydratedEvent; if (event.id !== undefined) { this.lastSeenEventId = Math.max( this.lastSeenEventId ?? 0, @@ -943,6 +1048,125 @@ export class DaemonSessionClient { } } + private async hydrateReplaySnapshot(): Promise { + this.replaySnapshot.compactedReplay = await Promise.all( + this.replaySnapshot.compactedReplay.map( + async (event) => await this.hydrateEvent(event), + ), + ); + this.replaySnapshot.liveJournal = await Promise.all( + this.replaySnapshot.liveJournal.map( + async (event) => await this.hydrateEvent(event), + ), + ); + } + + private async hydrateEvent(event: DaemonEvent): Promise { + if (!event.data || typeof event.data !== 'object') return event; + const data = event.data as Record; + if (event.type === 'session_update') { + const update = data['update']; + if (update && typeof update === 'object' && !Array.isArray(update)) { + const content = (update as Record)['content']; + const hydrated = await this.hydrateBlock(content); + if (hydrated === content) return event; + return { + ...event, + data: { ...data, update: { ...update, content: hydrated } }, + }; + } + const content = data['content']; + const hydrated = await this.hydrateBlock(content); + if (hydrated === content) return event; + return { ...event, data: { ...data, content: hydrated } }; + } + if (event.type !== 'mid_turn_message_injected') return event; + const items = data['items']; + if (!Array.isArray(items)) return event; + return { + ...event, + data: { + ...data, + items: await Promise.all( + items.map(async (item) => { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return item; + } + const record = item as Record; + return Array.isArray(record['content']) + ? { + ...record, + content: await this.hydrateContent(record['content']), + } + : item; + }), + ), + }, + }; + } + + private async hydrateContent( + content: readonly unknown[], + ): Promise { + return await Promise.all( + content.map(async (block) => await this.hydrateBlock(block)), + ); + } + + private async hydrateBlock(block: unknown): Promise { + if (!isSessionMediaReference(block)) { + return block as PromptContentBlock; + } + let cached = this.mediaCache.get(block.mediaId); + if (cached) { + this.mediaCache.delete(block.mediaId); + this.mediaCache.set(block.mediaId, cached); + } else { + const pending = this.withClientIdSelfHeal(() => + this.client.readSessionMedia(this.sessionId, block.mediaId, { + ...(this.clientId ? { clientId: this.clientId } : {}), + }), + ); + cached = { pending, size: block.size }; + this.mediaCache.set(block.mediaId, cached); + this.mediaCacheBytes += block.size; + while ( + this.mediaCache.size > MAX_MEDIA_CACHE_ENTRIES || + this.mediaCacheBytes > MAX_MEDIA_CACHE_BYTES + ) { + const oldestId = this.mediaCache.keys().next().value; + if (oldestId === undefined) break; + const evicted = this.mediaCache.get(oldestId); + this.mediaCache.delete(oldestId); + this.mediaCacheBytes -= evicted?.size ?? 0; + } + void pending.catch(() => { + if (this.mediaCache.get(block.mediaId)?.pending !== pending) return; + this.mediaCache.delete(block.mediaId); + this.mediaCacheBytes -= block.size; + }); + } + try { + const media = await cached.pending; + return { type: block.type, data: media.data, mimeType: media.mimeType }; + } catch (err) { + // 404/410 means the daemon no longer holds the blob, so pin the + // placeholder. Any other failure is transient: return the reference + // unchanged so the snapshot keeps its mediaId and a later hydration + // pass can retry (the failed cache entry evicted itself above). + if ( + err instanceof DaemonHttpError && + (err.status === 404 || err.status === 410) + ) { + return { + type: 'text', + text: '[Attached media is no longer available]', + }; + } + return block; + } + } + private _dispatchTurnEvent(event: DaemonEvent): void { if (event.type !== 'turn_complete' && event.type !== 'turn_error') return; const promptId = (event.data as { promptId?: string } | null | undefined) diff --git a/packages/sdk-typescript/src/daemon/events.ts b/packages/sdk-typescript/src/daemon/events.ts index 19ac96fa505..03bfe2d065c 100644 --- a/packages/sdk-typescript/src/daemon/events.ts +++ b/packages/sdk-typescript/src/daemon/events.ts @@ -11,6 +11,7 @@ import type { DaemonMcpTransport, DaemonSessionArtifactChange, PermissionOutcome, + PromptContentBlock, } from './types.js'; // Single source of truth: the daemon publisher owns the wire literal in // acp-bridge's dependency-free `daemonEventTypes` module. We re-export it so the @@ -306,15 +307,21 @@ export interface DaemonArtifactChangedData { /** * `mid_turn_message_injected` payload. Emitted when the daemon drains * browser-queued mid-turn messages into the running turn (web-shell mid-turn - * drain). It is a transient dedupe signal, not a transcript item: consumers - * move these messages out of their pending queue so they aren't resent as the - * next turn. They are not rendered from this event — the message already reached - * the model mid-turn, and the persisted transcript shows it on reload. + * drain). Consumers move these messages out of their pending queue so they + * aren't resent as the next turn; UI adapters may also render the attached + * text/media as the immediate mid-turn echo. */ export interface DaemonMidTurnMessageInjectedData { sessionId: string; messages: string[]; messageIds?: string[]; + /** + * Parallel array to `messages` — one entry per drained message. Each entry + * may carry image content blocks the daemon attached to the + * original queued payload, so the browser echo renderer can show them + * alongside the message text. Older daemons omit this field. + */ + items?: Array<{ content?: PromptContentBlock[] }>; /** * Present only on events from older daemons. New daemons publish one * session-wide batch and clients reconcile it by message id. diff --git a/packages/sdk-typescript/src/daemon/index.ts b/packages/sdk-typescript/src/daemon/index.ts index b6775df914a..dd8dd48f7b4 100644 --- a/packages/sdk-typescript/src/daemon/index.ts +++ b/packages/sdk-typescript/src/daemon/index.ts @@ -698,6 +698,8 @@ export type { PermissionOutcomeSelected, PermissionResponse, PromptContentBlock, + DaemonSessionMediaData, + DaemonSessionMediaReference, PromptResult, PromptTextContent, SetModelResult, diff --git a/packages/sdk-typescript/src/daemon/types.ts b/packages/sdk-typescript/src/daemon/types.ts index 4c5eac910eb..9b79bfc21ec 100644 --- a/packages/sdk-typescript/src/daemon/types.ts +++ b/packages/sdk-typescript/src/daemon/types.ts @@ -3146,11 +3146,14 @@ export interface DaemonRemoveMidTurnMessageResult { /** * One entry still waiting in the daemon's mid-turn queue (projection of the - * bridge's `MidTurnQueueEntry`). The queue is session-global. + * bridge's `MidTurnQueueEntry`). The queue is session-global. `content` carries + * any image blocks attached to the message, so a refreshed client + * can rebuild its queued row with the attachments intact. */ export interface DaemonMidTurnMessageSummary { messageId: string; text: string; + content?: PromptContentBlock[]; } /** @@ -3172,11 +3175,14 @@ export interface DaemonMidTurnMessagesResult { /** * One entry in the daemon's pending prompt queue. The `state` is * `'running'` for the currently dispatching prompt and `'queued'` - * for prompts waiting in the FIFO. + * for prompts waiting in the FIFO. `content` carries any image blocks attached + * to the prompt, so a refreshed client can restore + * the full payload (text + images) instead of just the text. */ export interface DaemonPendingPromptSummary { promptId: string; text: string; + content?: PromptContentBlock[]; queuedAt: number; state: 'queued' | 'running'; originatorClientId?: string; @@ -3853,6 +3859,18 @@ export interface PromptTextContent { text: string; } +export type DaemonSessionMediaReference = Record & { + type: 'image'; + mediaId: string; + mimeType: string; + size: number; +}; + +export interface DaemonSessionMediaData { + data: string; + mimeType: string; +} + /** * The set of content blocks the daemon's prompt route accepts. The full ACP * `ContentBlock` union is wider; SDK clients can pass any of those shapes diff --git a/packages/sdk-typescript/src/daemon/ui/normalizer.ts b/packages/sdk-typescript/src/daemon/ui/normalizer.ts index 1ecc62fb5b7..9978d7f0dde 100644 --- a/packages/sdk-typescript/src/daemon/ui/normalizer.ts +++ b/packages/sdk-typescript/src/daemon/ui/normalizer.ts @@ -62,6 +62,8 @@ const MAX_DETAILS_LENGTH = 4096; const SESSION_RECORDING_DEGRADED_MESSAGE = 'Session recording stopped after a write failure. New messages for the affected session will not be saved. Check disk space and permissions, then start a new session to resume recording.'; +const MEDIA_UNAVAILABLE_TEXT = '[Attached media is no longer available]'; + export function normalizeDaemonEvent( event: DaemonEvent, opts: NormalizeDaemonEventOptions = {}, @@ -575,24 +577,67 @@ function normalizeMidTurnMessageInjected( if (!isRecord(event.data)) { return fallbackDebug(event, base, 'malformed mid_turn_message_injected'); } - const messages = Array.isArray(event.data['messages']) - ? event.data['messages'].filter( - (message): message is string => - typeof message === 'string' && message.length > 0, - ) - : []; - if (messages.length === 0) { + const data = event.data; + const rawMessages = data['messages']; + const messages = + Array.isArray(rawMessages) && + rawMessages.every( + (message): message is string => typeof message === 'string', + ) + ? rawMessages + : []; + const items = data['items']; + // An injected message is renderable when its text is non-empty OR its + // content carries an image or a non-empty text block. The drain's + // degraded-media path publishes `messages: ['']` whose items hold only the + // '[Attached media is no longer available]' text block — dropping that + // frame as malformed would erase the echo of the user's message. + const hasRenderableItemContent = + Array.isArray(items) && + items.some( + (item) => + isRecord(item) && + Array.isArray(item['content']) && + item['content'].some( + (block) => + isRecord(block) && + (block['type'] === 'image' || + (block['type'] === 'text' && + typeof block['text'] === 'string' && + (block['text'] as string).length > 0)), + ), + ); + if ( + messages.length === 0 || + (!messages.some(Boolean) && !hasRenderableItemContent) + ) { return fallbackDebug(event, base, 'malformed mid_turn_message_injected'); } - return [ - { + const messageIds = Array.isArray(data['messageIds']) + ? data['messageIds'] + : []; + return messages.map((text, index) => { + const item = Array.isArray(items) ? items[index] : undefined; + const messageId = messageIds[index]; + return { ...base, type: 'status', - text: `Inserted message: ${messages.join('\n')}`, + text, source: 'mid_turn_message_injected', - data: event.data, - }, - ]; + data: { + ...data, + messages: [text], + ...(Array.isArray(items) + ? { items: item !== undefined ? [item] : [] } + : {}), + ...(typeof messageId === 'string' + ? { messageIds: [messageId] } + : Array.isArray(data['messageIds']) + ? { messageIds: [] } + : {}), + }, + }; + }); } function createBase( @@ -691,6 +736,24 @@ function parseTimestamp(value: unknown): number | undefined { return Number.isFinite(parsed) ? parsed : undefined; } +/** + * True for the session-media reference shape (`mediaId` instead of inline + * data/url/source) that replay producers persist for uploaded attachments. + * `extractContentPart` cannot render it; see the `user_message_chunk` case + * below for how it degrades instead of vanishing. + */ +function isMediaReferenceContent(value: unknown): boolean { + return ( + isRecord(value) && + value['type'] === 'image' && + typeof value['mediaId'] === 'string' && + (value['mediaId'] as string).length > 0 && + value['data'] === undefined && + value['url'] === undefined && + value['source'] === undefined + ); +} + function normalizeSessionUpdate( event: DaemonEvent, base: NormalizedEventBase, @@ -737,7 +800,15 @@ function normalizeSessionUpdate( else if (prefix.startsWith('UklGR')) mimeType = 'image/webp'; } if (data) { - return [{ ...base, type: 'user.image.delta', data, mimeType }]; + return [ + { + ...base, + type: 'user.image.delta', + data, + mimeType, + ...(meta ? { meta } : {}), + }, + ]; } return []; } @@ -755,6 +826,19 @@ function normalizeSessionUpdate( } return []; } + // Live consumers hydrate reference blocks before normalization; a path + // that reaches this point with one (offline record projection, failed + // hydrate) keeps the user's message visible via the placeholder. + if (isMediaReferenceContent(content)) { + return [ + { + ...base, + type: 'user.text.delta', + text: MEDIA_UNAVAILABLE_TEXT, + ...(meta ? { meta } : {}), + }, + ]; + } const text = getTextContent(content); return text ? [ diff --git a/packages/sdk-typescript/src/daemon/ui/transcript.ts b/packages/sdk-typescript/src/daemon/ui/transcript.ts index 31121d52209..7b3e58066c6 100644 --- a/packages/sdk-typescript/src/daemon/ui/transcript.ts +++ b/packages/sdk-typescript/src/daemon/ui/transcript.ts @@ -249,7 +249,7 @@ function applyDaemonTranscriptEvent( '', event.eventId, event.serverTimestamp, - undefined, + event.meta, event.sourceRecordIds, event.promptId, ) as DaemonTextTranscriptBlock; @@ -262,6 +262,7 @@ function applyDaemonTranscriptEvent( | DaemonTextTranscriptBlock | undefined; if (block && block.kind === 'user') { + if (event.meta) block.meta = { ...block.meta, ...event.meta }; // Use immutable update to avoid mutating a shared array reference block.images = [ ...(block.images ?? []), diff --git a/packages/sdk-typescript/src/daemon/ui/types.ts b/packages/sdk-typescript/src/daemon/ui/types.ts index b7776b90826..96480fd388e 100644 --- a/packages/sdk-typescript/src/daemon/ui/types.ts +++ b/packages/sdk-typescript/src/daemon/ui/types.ts @@ -131,6 +131,7 @@ export interface DaemonUiUserImageEvent extends DaemonUiEventBase { type: 'user.image.delta'; data: string; mimeType: string; + meta?: DaemonTextDeltaMeta; } export interface DaemonUiUserShellCommandEvent extends DaemonUiEventBase { diff --git a/packages/sdk-typescript/test/unit/DaemonClient.test.ts b/packages/sdk-typescript/test/unit/DaemonClient.test.ts index ef8060cd504..90ed955a613 100644 --- a/packages/sdk-typescript/test/unit/DaemonClient.test.ts +++ b/packages/sdk-typescript/test/unit/DaemonClient.test.ts @@ -4396,6 +4396,33 @@ describe('DaemonClient', () => { expect(result.accepted).toBe(false); }); + it('includes media content blocks in the POST body when provided', async () => { + const { fetch, calls } = recordingFetch(() => + jsonResponse(200, { accepted: true, messageId: 'mid-1' }), + ); + const client = new DaemonClient({ baseUrl: 'http://daemon', fetch }); + await client.enqueueMidTurnMessage('s-1', 'see this', { + messageId: 'client-mid-1', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }); + expect(JSON.parse(calls[0]?.body as string)).toEqual({ + message: 'see this', + messageId: 'client-mid-1', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }); + }); + + it('omits the content field when no media blocks are attached', async () => { + const { fetch, calls } = recordingFetch(() => + jsonResponse(200, { accepted: true }), + ); + const client = new DaemonClient({ baseUrl: 'http://daemon', fetch }); + await client.enqueueMidTurnMessage('s-1', 'plain', { content: [] }); + expect(JSON.parse(calls[0]?.body as string)).toEqual({ + message: 'plain', + }); + }); + it('URL-encodes the session id, forwards client id, and propagates the abort signal', async () => { const { fetch, calls } = recordingFetch(() => jsonResponse(200, { accepted: true }), diff --git a/packages/sdk-typescript/test/unit/DaemonSessionClient.test.ts b/packages/sdk-typescript/test/unit/DaemonSessionClient.test.ts index 41b99b9484a..653243799e9 100644 --- a/packages/sdk-typescript/test/unit/DaemonSessionClient.test.ts +++ b/packages/sdk-typescript/test/unit/DaemonSessionClient.test.ts @@ -334,6 +334,375 @@ describe('DaemonSessionClient', () => { expect(calls[1]?.headers['last-event-id']).toBe('42'); }); + it('hydrates media references in a replay snapshot', async () => { + const { fetch, calls } = recordingFetch((req) => { + if (req.url.endsWith('/session/s-1/load')) { + return jsonResponse(200, { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: false, + clientId: 'client-1', + compactedReplay: [ + { + id: 1, + v: 1, + type: 'session_update', + data: { + update: { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + }, + }, + }, + { + id: 2, + v: 1, + type: 'session_update', + data: { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + }, + }, + ], + }); + } + if (req.url.endsWith('/session/s-1/media/media-1')) { + return new Response(Uint8Array.from([1, 2, 3]), { + status: 200, + headers: { 'content-type': 'image/png' }, + }); + } + if (req.url.endsWith('/session/s-1/transcript')) { + return jsonResponse(200, { + v: 1, + sessionId: 's-1', + hasMore: false, + events: [ + { + v: 1, + type: 'session_update', + data: { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + }, + }, + ], + }); + } + return jsonResponse(500, { error: `unexpected ${req.url}` }); + }); + const client = new DaemonClient({ baseUrl: 'http://daemon', fetch }); + + const session = await DaemonSessionClient.load(client, 's-1'); + + expect(session.replaySnapshot.compactedReplay[0]?.data).toEqual({ + update: { + sessionUpdate: 'user_message_chunk', + content: { type: 'image', data: 'AQID', mimeType: 'image/png' }, + }, + }); + expect(session.replaySnapshot.compactedReplay[1]?.data).toEqual({ + sessionUpdate: 'user_message_chunk', + content: { type: 'image', data: 'AQID', mimeType: 'image/png' }, + }); + const page = await session.getTranscriptPage(); + expect(page.events[0]?.data).toEqual({ + sessionUpdate: 'user_message_chunk', + content: { type: 'image', data: 'AQID', mimeType: 'image/png' }, + }); + expect(calls[1]?.headers['x-qwen-client-id']).toBe('client-1'); + expect( + calls.filter((call) => call.url.endsWith('/media/media-1')), + ).toHaveLength(1); + }); + + it('keeps a visible placeholder when replay media is unavailable', async () => { + const { fetch, calls } = recordingFetch((req) => { + if (req.url.endsWith('/session/s-1/load')) { + return jsonResponse(200, { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: false, + clientId: 'client-1', + compactedReplay: [ + { + id: 1, + v: 1, + type: 'session_update', + data: { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'missing-media', + mimeType: 'image/png', + size: 3, + }, + }, + }, + ], + }); + } + if (req.url.endsWith('/session/s-1/media/missing-media')) { + return jsonResponse(410, { error: 'gone' }); + } + return jsonResponse(500, { error: `unexpected ${req.url}` }); + }); + const client = new DaemonClient({ baseUrl: 'http://daemon', fetch }); + + const session = await DaemonSessionClient.load(client, 's-1'); + + expect(session.replaySnapshot.compactedReplay[0]?.data).toEqual({ + sessionUpdate: 'user_message_chunk', + content: { + type: 'text', + text: '[Attached media is no longer available]', + }, + }); + const hydrateBlock = ( + session as unknown as { + hydrateBlock(block: unknown): Promise; + } + ).hydrateBlock.bind(session); + await hydrateBlock({ + type: 'image', + mediaId: 'missing-media', + mimeType: 'image/png', + size: 3, + }); + expect( + calls.filter((call) => call.url.endsWith('/media/missing-media')), + ).toHaveLength(2); + }); + + it('keeps replay media references retryable after a transient media failure', async () => { + let mediaRequests = 0; + const reference = { + type: 'image', + mediaId: 'flaky-media', + mimeType: 'image/png', + size: 3, + }; + const { fetch, calls } = recordingFetch((req) => { + if (req.url.endsWith('/session/s-1/load')) { + return jsonResponse(200, { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: false, + clientId: 'client-1', + compactedReplay: [ + { + id: 1, + v: 1, + type: 'session_update', + data: { + update: { + sessionUpdate: 'user_message_chunk', + content: reference, + }, + }, + }, + { + id: 2, + v: 1, + type: 'mid_turn_message_injected', + data: { + sessionId: 's-1', + messages: [''], + items: [{ content: [reference] }], + }, + }, + ], + }); + } + if (req.url.endsWith('/session/s-1/media/flaky-media')) { + mediaRequests += 1; + if (mediaRequests === 1) { + return jsonResponse(500, { error: 'boom' }); + } + return new Response(Uint8Array.from([1, 2, 3]), { + status: 200, + headers: { 'content-type': 'image/png' }, + }); + } + if (req.url.endsWith('/session/s-1/transcript')) { + return jsonResponse(200, { + v: 1, + sessionId: 's-1', + hasMore: false, + events: [ + { + v: 1, + type: 'session_update', + data: { + sessionUpdate: 'user_message_chunk', + content: reference, + }, + }, + ], + }); + } + return jsonResponse(500, { error: `unexpected ${req.url}` }); + }); + const client = new DaemonClient({ baseUrl: 'http://daemon', fetch }); + + const session = await DaemonSessionClient.load(client, 's-1'); + + // A transient failure must keep the reference (and its mediaId) in the + // snapshot so a later hydration pass can retry instead of pinning the + // permanent placeholder for the client's lifetime. + expect(session.replaySnapshot.compactedReplay[0]?.data).toEqual({ + update: { sessionUpdate: 'user_message_chunk', content: reference }, + }); + expect(session.replaySnapshot.compactedReplay[1]?.data).toMatchObject({ + items: [{ content: [reference] }], + }); + expect( + calls.filter((call) => call.url.endsWith('/media/flaky-media')), + ).toHaveLength(1); + + const page = await session.getTranscriptPage(); + expect(page.events[0]?.data).toEqual({ + sessionUpdate: 'user_message_chunk', + content: { type: 'image', data: 'AQID', mimeType: 'image/png' }, + }); + expect( + calls.filter((call) => call.url.endsWith('/media/flaky-media')), + ).toHaveLength(2); + }); + + it('evicts least-recently-used media when the cache byte cap is exceeded', async () => { + const { fetch, calls } = recordingFetch((req) => { + if (req.url.includes('/session/s-1/media/')) { + return new Response(Uint8Array.from([1]), { + status: 200, + headers: { 'content-type': 'image/png' }, + }); + } + return jsonResponse(500, { error: `unexpected ${req.url}` }); + }); + const session = new DaemonSessionClient({ + client: new DaemonClient({ baseUrl: 'http://daemon', fetch }), + session: { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: true, + }, + }); + const hydrateBlock = ( + session as unknown as { + hydrateBlock(block: unknown): Promise; + } + ).hydrateBlock.bind(session); + + for (let index = 0; index < 4; index += 1) { + await hydrateBlock({ + type: 'image', + mediaId: `media-${index}`, + mimeType: 'image/png', + size: 8 * 1024 * 1024, + }); + } + await hydrateBlock({ + type: 'image', + mediaId: 'media-0', + mimeType: 'image/png', + size: 8 * 1024 * 1024, + }); + await hydrateBlock({ + type: 'image', + mediaId: 'media-4', + mimeType: 'image/png', + size: 8 * 1024 * 1024, + }); + await hydrateBlock({ + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 8 * 1024 * 1024, + }); + + expect( + calls.filter((call) => call.url.endsWith('/media/media-0')), + ).toHaveLength(1); + expect( + calls.filter((call) => call.url.endsWith('/media/media-1')), + ).toHaveLength(2); + }); + + it('uploads session media through the authenticated session route', async () => { + const reference = { + type: 'image' as const, + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }; + const { fetch, calls } = recordingFetch((req) => + req.method === 'POST' + ? jsonResponse(201, reference) + : jsonResponse(500, { error: `unexpected ${req.url}` }), + ); + const session = new DaemonSessionClient({ + client: new DaemonClient({ baseUrl: 'http://daemon', fetch }), + session: { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: true, + clientId: 'client-1', + }, + }); + + await expect( + session.uploadMedia(new Blob([Uint8Array.of(1, 2, 3)]), 'image/png'), + ).resolves.toEqual(reference); + expect(calls[0]).toMatchObject({ + method: 'POST', + headers: expect.objectContaining({ + 'content-type': 'image/png', + 'x-qwen-client-id': 'client-1', + }), + }); + expect(calls[0]?.url).toContain('/session/s-1/media'); + }); + + it('removes session media through the authenticated session route', async () => { + const { fetch, calls } = recordingFetch((req) => + req.method === 'DELETE' + ? jsonResponse(200, { removed: true }) + : jsonResponse(500, { error: `unexpected ${req.url}` }), + ); + const session = new DaemonSessionClient({ + client: new DaemonClient({ baseUrl: 'http://daemon', fetch }), + session: { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: true, + clientId: 'client-1', + }, + }); + + await expect(session.removeMedia('media-1')).resolves.toBe(true); + expect(calls[0]).toMatchObject({ + method: 'DELETE', + headers: expect.objectContaining({ 'x-qwen-client-id': 'client-1' }), + }); + expect(calls[0]?.url).toContain('/session/s-1/media/media-1'); + }); + it('loads restored prompt activity from hasActivePrompt responses', async () => { const { fetch } = recordingFetch((req) => { if (req.url.endsWith('/session/s-1/load')) { @@ -717,6 +1086,86 @@ describe('DaemonSessionClient', () => { expect(calls[0]?.headers['x-qwen-client-id']).toBe('client-1'); }); + it('hydrates media references in pending and mid-turn snapshots', async () => { + const reference = { + type: 'image' as const, + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }; + const { fetch } = recordingFetch((req) => { + if (req.url.endsWith('/pending-prompts')) { + return jsonResponse(200, { + pendingPrompts: [ + { + promptId: 'prompt-1', + text: 'look', + state: 'queued', + queuedAt: 1, + content: [reference], + }, + ], + }); + } + if (req.url.endsWith('/mid-turn-messages')) { + return jsonResponse(200, { + messages: [ + { messageId: 'mid-1', text: 'look', content: [reference] }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + } + if (requestPathEndsWith(req, '/events')) { + return sseResponse( + `id: 1\nevent: mid_turn_message_injected\ndata: ${JSON.stringify({ + id: 1, + v: 1, + type: 'mid_turn_message_injected', + data: { + sessionId: 's-1', + messages: ['look'], + messageIds: ['mid-1'], + items: [{ content: [reference] }], + }, + })}\n\n`, + ); + } + if (req.url.endsWith('/media/media-1')) { + return new Response(Uint8Array.of(1, 2, 3), { + status: 200, + headers: { 'content-type': 'image/png' }, + }); + } + return jsonResponse(500, { error: `unexpected ${req.url}` }); + }); + const session = new DaemonSessionClient({ + client: new DaemonClient({ baseUrl: 'http://daemon', fetch }), + session: { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: true, + clientId: 'client-1', + }, + }); + const image = { type: 'image', data: 'AQID', mimeType: 'image/png' }; + + await expect(session.getPendingPrompts()).resolves.toMatchObject({ + pendingPrompts: [{ content: [image] }], + }); + await expect(session.getMidTurnMessages()).resolves.toMatchObject({ + messages: [{ content: [image] }], + }); + const events = []; + for await (const event of session.events()) events.push(event); + expect(events).toMatchObject([ + { + type: 'mid_turn_message_injected', + data: { items: [{ content: [image] }] }, + }, + ]); + }); + it('forwards pending prompt removals with encoded ids and clientId', async () => { const { fetch, calls } = recordingFetch(() => jsonResponse(200, { removed: false }), @@ -2218,6 +2667,85 @@ describe('DaemonSessionClient clientId self-heal', () => { expect(resumeReq?.body).toBe(JSON.stringify({ cwd: '/work/a' })); }); + it('re-registers and retries media upload, removal, and hydration', async () => { + let resumeCalls = 0; + const attempts = new Map(); + const { fetch, calls } = recordingFetch((req) => { + if (req.url.endsWith('/session/s-1/resume')) { + resumeCalls += 1; + return jsonResponse(200, { + sessionId: 's-1', + workspaceCwd: '/work/a', + attached: true, + clientId: `client-${resumeCalls + 1}`, + state: {}, + }); + } + const operation = `${req.method} ${new URL(req.url).pathname}`; + const attempt = (attempts.get(operation) ?? 0) + 1; + attempts.set(operation, attempt); + if (attempt === 1) return invalidClientIdResponse(); + if (req.method === 'POST') { + return jsonResponse(201, { + type: 'image', + mediaId: 'media-uploaded', + mimeType: 'image/png', + size: 3, + }); + } + if (req.method === 'DELETE') { + return jsonResponse(200, { removed: true }); + } + if (req.method === 'GET') { + return new Response(Uint8Array.of(1, 2, 3), { + status: 200, + headers: { 'content-type': 'image/png' }, + }); + } + return jsonResponse(500, { error: `unexpected ${req.url}` }); + }); + const session = newSession( + new DaemonClient({ baseUrl: 'http://daemon', fetch }), + ); + + await expect( + session.uploadMedia(new Blob([Uint8Array.of(1, 2, 3)]), 'image/png'), + ).resolves.toMatchObject({ mediaId: 'media-uploaded' }); + await expect(session.removeMedia('media-uploaded')).resolves.toBe(true); + const hydrateBlock = ( + session as unknown as { + hydrateBlock(block: unknown): Promise; + } + ).hydrateBlock.bind(session); + await expect( + hydrateBlock({ + type: 'image', + mediaId: 'media-read', + mimeType: 'image/png', + size: 3, + }), + ).resolves.toEqual({ + type: 'image', + data: 'AQID', + mimeType: 'image/png', + }); + + expect(resumeCalls).toBe(3); + expect(session.clientId).toBe('client-4'); + expect( + calls + .filter((call) => !call.url.endsWith('/resume')) + .map((call) => call.headers['x-qwen-client-id']), + ).toEqual([ + 'client-1', + 'client-2', + 'client-2', + 'client-3', + 'client-3', + 'client-4', + ]); + }); + it('re-registers and retries once on the non-blocking prompt path', async () => { let promptCalls = 0; let resumeCalls = 0; diff --git a/packages/sdk-typescript/test/unit/daemon-transcript-projection.test.ts b/packages/sdk-typescript/test/unit/daemon-transcript-projection.test.ts index 171838b6d3c..d44198e68aa 100644 --- a/packages/sdk-typescript/test/unit/daemon-transcript-projection.test.ts +++ b/packages/sdk-typescript/test/unit/daemon-transcript-projection.test.ts @@ -578,4 +578,56 @@ describe('projectChatRecordsToDaemonTranscript', () => { }), ); }); + + it('projects reference-only media records as unavailable placeholders', () => { + const projection = projectChatRecordsToDaemonTranscript([ + record('mid-text-plus-image', null, { + subtype: 'mid_turn_user_message', + message: { role: 'user', parts: [{ text: 'look at this' }] }, + systemPayload: { + displayText: 'look at this', + mediaReferences: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }, + }), + record('mid-image-only', 'mid-text-plus-image', { + subtype: 'mid_turn_user_message', + message: { + role: 'user', + parts: [{ text: '[User message received during tool execution]: ' }], + }, + systemPayload: { + displayText: '', + mediaReferences: [ + { + type: 'image', + mediaId: 'media-2', + mimeType: 'image/png', + size: 3, + }, + ], + }, + }), + ]); + + const userBlocks = projection.blocks.filter( + (block) => block.kind === 'user', + ); + expect(userBlocks.map((block) => block.text)).toEqual([ + 'look at this', + '[Attached media is no longer available]', + '[Attached media is no longer available]', + ]); + expect(userBlocks.map((block) => block.sourceRecordIds)).toEqual([ + ['mid-text-plus-image'], + ['mid-text-plus-image'], + ['mid-image-only'], + ]); + }); }); diff --git a/packages/sdk-typescript/test/unit/daemonUi.test.ts b/packages/sdk-typescript/test/unit/daemonUi.test.ts index d4e49da1f58..5ca62739966 100644 --- a/packages/sdk-typescript/test/unit/daemonUi.test.ts +++ b/packages/sdk-typescript/test/unit/daemonUi.test.ts @@ -6415,13 +6415,191 @@ describe('R5 review batch — coverage additions', () => { expect(events).toEqual([ expect.objectContaining({ type: 'status', - text: 'Inserted message: 你好', + text: '你好', source: 'mid_turn_message_injected', data: { sessionId: 's1', messages: ['你好'] }, }), ]); }); + it('keeps each message in an injected mid-turn batch separate', () => { + const events = normalizeDaemonEvent({ + id: 3, + v: 1, + type: 'mid_turn_message_injected', + data: { + sessionId: 's1', + messages: ['with image', 'text only'], + messageIds: ['mid-1', 'mid-2'], + items: [ + { + content: [{ type: 'image', data: 'AQID', mimeType: 'image/png' }], + }, + {}, + ], + }, + }); + + expect(events).toMatchObject([ + { + type: 'status', + text: 'with image', + data: { + messages: ['with image'], + messageIds: ['mid-1'], + items: [ + { + content: [{ type: 'image', data: 'AQID', mimeType: 'image/png' }], + }, + ], + }, + }, + { + type: 'status', + text: 'text only', + data: { + messages: ['text only'], + messageIds: ['mid-2'], + items: [{}], + }, + }, + ]); + }); + + it('preserves replay source metadata on an image-only user block', () => { + const events = normalizeDaemonEvent({ + id: 4, + v: 1, + type: 'session_update', + data: { + update: { + sessionUpdate: 'user_message_chunk', + content: { type: 'image', data: 'AQID', mimeType: 'image/png' }, + _meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + qwenTranscript: { sourceRecordIds: ['record-1'] }, + }, + }, + }, + }); + const state = reduceDaemonTranscriptEvents( + createDaemonTranscriptState({ now: 1 }), + events, + ); + + expect(state.blocks).toMatchObject([ + { + kind: 'user', + images: [{ data: 'AQID', mimeType: 'image/png' }], + meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + }, + ]); + }); + + it('normalizes a reference-only image block into the media-unavailable placeholder', () => { + // Replay producers persist uploaded attachments as media references + // (`mediaId`, no inline bytes). Paths that normalize without hydrating + // (offline record projections) must degrade to a visible placeholder + // instead of silently dropping the user's message. + expect( + normalizeDaemonEvent({ + id: 7, + v: 1, + type: 'session_update', + data: { + update: { + sessionUpdate: 'user_message_chunk', + content: { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + _meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + qwenTranscript: { sourceRecordIds: ['record-1'] }, + }, + }, + }, + }), + ).toEqual([ + expect.objectContaining({ + type: 'user.text.delta', + text: '[Attached media is no longer available]', + sourceRecordIds: ['record-1'], + meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + }), + ]); + }); + + it('normalizes an image-only mid-turn message without dropping its slot', () => { + const data = { + sessionId: 's1', + messages: [''], + items: [ + { + content: [{ type: 'image', data: 'AQID', mimeType: 'image/png' }], + }, + ], + }; + expect( + normalizeDaemonEvent({ + id: 2, + v: 1, + type: 'mid_turn_message_injected', + data, + }), + ).toEqual([ + expect.objectContaining({ + type: 'status', + text: '', + source: 'mid_turn_message_injected', + data, + }), + ]); + }); + + it('normalizes a degraded-media mid-turn echo instead of dropping it', () => { + // The drain's media-failure path publishes `messages: ['']` whose item + // content is the media-unavailable text block (no image blocks); the + // guard must keep the user's injected echo renderable. + const data = { + sessionId: 's1', + messages: [''], + messageIds: ['mid-gone'], + items: [ + { + content: [ + { type: 'text', text: '[Attached media is no longer available]' }, + ], + }, + ], + }; + expect( + normalizeDaemonEvent({ + id: 5, + v: 1, + type: 'mid_turn_message_injected', + data, + }), + ).toEqual([ + expect.objectContaining({ + type: 'status', + text: '', + source: 'mid_turn_message_injected', + data, + }), + ]); + }); + it('store.clearAwaitingResync clears latch', async () => { const { createDaemonTranscriptStore } = await import( '../../src/daemon/ui/index.js' diff --git a/packages/web-shell/client/App.module.css b/packages/web-shell/client/App.module.css index e259740c1af..2ddf87fd185 100644 --- a/packages/web-shell/client/App.module.css +++ b/packages/web-shell/client/App.module.css @@ -1073,6 +1073,30 @@ white-space: nowrap; } +.queuedPromptImages { + display: inline-flex; + align-items: center; + gap: 3px; + flex-shrink: 0; +} + +.queuedPromptImage { + width: 18px; + height: 18px; + object-fit: cover; + border-radius: 4px; + border: 1px solid var(--border); +} + +.queuedPromptImageInteractive { + cursor: pointer; + transition: opacity 0.15s ease; +} + +.queuedPromptImageInteractive:hover { + opacity: 0.85; +} + .queuedPromptState { display: inline-flex; align-items: center; diff --git a/packages/web-shell/client/App.tsx b/packages/web-shell/client/App.tsx index 108ecabf767..05cabd3ae97 100644 --- a/packages/web-shell/client/App.tsx +++ b/packages/web-shell/client/App.tsx @@ -4101,6 +4101,9 @@ export function App({ (!failedPromptRetry.admitted || failedPromptRetry.settled), ); const streamingStateRef = useRef(streamingState); + useEffect(() => { + streamingStateRef.current = streamingState; + }, [streamingState]); // Cleared in three places: the session-switch effect, the drain loop, and // handleCancel. Bumping drainGenerationRef at each clear site also cancels // any in-flight inline ! command whose ensureSessionForPrompt is resolving. @@ -6123,6 +6126,8 @@ export function App({ connection.capabilities?.features.includes( 'session_mid_turn_message_query', ) === true; + const canInjectMidTurnMedia = + connection.capabilities?.features.includes('session_media') === true; const { queuedPrompts, queuedTexts, @@ -6141,6 +6146,7 @@ export function App({ clientId: connection.clientId, canMutateMidTurn, canQueryMidTurn, + canInjectMidTurnMedia, streamingState, sessionActions, store, @@ -7033,10 +7039,6 @@ export function App({ ], ); - useEffect(() => { - streamingStateRef.current = streamingState; - }, [streamingState]); - // Drop queued commands on a session switch so the drain never runs a // command against a different workspace's daemon (mirrors useQueuedPrompts). const prevQueueSessionIdRef = useRef(logicalSessionKey); @@ -12484,6 +12486,7 @@ export function App({ onEdit={editQueuedPrompt} onRestoreUnknown={restoreUnknownQueuedPrompt} onDiscardUnknown={discardUnknownQueuedPrompt} + onImagePreview={openImagePanel} /> {CustomComposerHeader && (
diff --git a/packages/web-shell/client/adapters/messageTypes.ts b/packages/web-shell/client/adapters/messageTypes.ts index 51b5534ac10..aa4fb1c3765 100644 --- a/packages/web-shell/client/adapters/messageTypes.ts +++ b/packages/web-shell/client/adapters/messageTypes.ts @@ -142,6 +142,7 @@ export interface DaemonSystemMessage extends DaemonMessageMeta { retryable?: boolean; source?: string; data?: unknown; + images?: Array<{ data: string; mimeType: string }>; } export interface DaemonUserShellMessage extends DaemonMessageMeta { diff --git a/packages/web-shell/client/adapters/transcriptToMessages.test.ts b/packages/web-shell/client/adapters/transcriptToMessages.test.ts index 76bdfd22f4c..80780e5fc81 100644 --- a/packages/web-shell/client/adapters/transcriptToMessages.test.ts +++ b/packages/web-shell/client/adapters/transcriptToMessages.test.ts @@ -526,31 +526,172 @@ describe('transcriptBlocksToDaemonMessages', () => { ]); }); - it('localizes structured mid-turn inserted status blocks', () => { - const messages = transcriptBlocksToDaemonMessages( - [ - statusBlock('mid-1', 'Inserted message: hello', 1, { - source: 'mid_turn_message_injected', - data: { sessionId: 's1', messages: ['你好'] }, - }), - ], - { - labels: { - midTurnInserted: (message) => `已插入消息：${message}`, - }, - }, - ); + it('shows structured mid-turn inserted text without a status prefix', () => { + const messages = transcriptBlocksToDaemonMessages([ + statusBlock('mid-1', '你好', 1, { + source: 'mid_turn_message_injected', + data: { sessionId: 's1', messages: ['你好'] }, + }), + ]); expect(messages).toEqual([ expect.objectContaining({ role: 'system', - content: '已插入消息：你好', + content: '你好', source: 'mid_turn_message_injected', data: { sessionId: 's1', messages: ['你好'] }, }), ]); }); + it('extracts images from mid-turn injected message items', () => { + const messages = transcriptBlocksToDaemonMessages([ + statusBlock('mid-1', 'look at this', 1, { + source: 'mid_turn_message_injected', + data: { + sessionId: 's1', + messages: ['look at this'], + items: [ + { + content: [ + { type: 'image', data: 'base64data', mimeType: 'image/png' }, + ], + }, + ], + }, + }), + ]); + + expect(messages).toEqual([ + expect.objectContaining({ + role: 'system', + content: 'look at this', + source: 'mid_turn_message_injected', + images: [{ data: 'base64data', mimeType: 'image/png' }], + }), + ]); + }); + + it('shows the degraded-media notice when the echo text is empty', () => { + // When the stored media is gone at drain, the daemon echoes an empty + // messages array whose items carry only the placeholder text block; the + // notice must be surfaced instead of rendering an empty bubble. + const messages = transcriptBlocksToDaemonMessages([ + statusBlock('mid-1', '', 1, { + source: 'mid_turn_message_injected', + data: { + sessionId: 's1', + messages: [''], + messageIds: ['mid-gone'], + items: [ + { + content: [ + { + type: 'text', + text: '[Attached media is no longer available]', + }, + ], + }, + ], + }, + }), + ]); + + expect(messages).toEqual([ + expect.objectContaining({ + role: 'system', + content: '[Attached media is no longer available]', + source: 'mid_turn_message_injected', + }), + ]); + }); + + it('keeps mid-turn injected echoes that look like status noise', () => { + // User content that merely starts like a filtered status line must not be + // dropped by the noise filter. + const messages = transcriptBlocksToDaemonMessages([ + statusBlock('mid-1', 'Model switched: check this too', 1, { + source: 'mid_turn_message_injected', + data: { + sessionId: 's1', + messages: ['Model switched: check this too'], + }, + }), + ]); + + expect(messages).toEqual([ + expect.objectContaining({ + role: 'system', + content: 'Model switched: check this too', + source: 'mid_turn_message_injected', + }), + ]); + }); + + it('keeps mid-turn injected echoes that start like plan JSON', () => { + // User content starting with the plan projection shape must not be + // misrendered as a plan card (which also drops the attached images). + const planLikeText = + 'plan: {"sessionUpdate":"plan","entries":[{"content":"step"}]}'; + const messages = transcriptBlocksToDaemonMessages([ + statusBlock('mid-1', planLikeText, 1, { + source: 'mid_turn_message_injected', + data: { + sessionId: 's1', + messages: [planLikeText], + items: [ + { + content: [ + { type: 'image', data: 'base64data', mimeType: 'image/png' }, + ], + }, + ], + }, + }), + ]); + + expect(messages).toEqual([ + expect.objectContaining({ + role: 'system', + content: planLikeText, + source: 'mid_turn_message_injected', + images: [{ data: 'base64data', mimeType: 'image/png' }], + }), + ]); + }); + + it('keeps replayed mid-turn user blocks as inserted messages', () => { + const messages = transcriptBlocksToDaemonMessages([ + textBlock('mid-1', 'user', 'with image', 1, false, { + meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + images: [{ data: 'AQID', mimeType: 'image/png' }], + }), + textBlock('mid-2', 'user', 'text only', 2, false, { + meta: { + source: 'mid_turn_message_injected', + qwenDiscreteMessage: true, + }, + }), + ]); + + expect(messages).toMatchObject([ + { + role: 'system', + content: 'with image', + source: 'mid_turn_message_injected', + images: [{ data: 'AQID', mimeType: 'image/png' }], + }, + { + role: 'system', + content: 'text only', + source: 'mid_turn_message_injected', + }, + ]); + }); + it('ignores daemon plan entries without content', () => { const plan = { sessionUpdate: 'plan', diff --git a/packages/web-shell/client/adapters/transcriptToMessages.ts b/packages/web-shell/client/adapters/transcriptToMessages.ts index ab3788188a0..18909771714 100644 --- a/packages/web-shell/client/adapters/transcriptToMessages.ts +++ b/packages/web-shell/client/adapters/transcriptToMessages.ts @@ -44,7 +44,6 @@ type ExtendedDaemonTextTranscriptBlock = DaemonTextTranscriptBlock & { interface TranscriptMessageLabels { promptCancelled?: string; branchSuccess?: (name: string) => string; - midTurnInserted?: (message: string) => string; modelStreamInterrupted?: string; loopDetected?: string; } @@ -246,15 +245,68 @@ function getSessionBranchDisplayName(data: unknown): string | null { : null; } -function getMidTurnInjectedText(data: unknown): string | null { - if (!data || typeof data !== 'object') return null; - const messages = (data as { messages?: unknown }).messages; - if (!Array.isArray(messages)) return null; - const text = messages - .filter((message): message is string => typeof message === 'string') - .join('\n') - .trim(); - return text || null; +/** + * Extract image content blocks from mid-turn injected message items. + * Returns an array of {data, mimeType} objects for rendering in the transcript. + */ +function getMidTurnInjectedImages( + data: unknown, +): Array<{ data: string; mimeType: string }> | undefined { + if (!data || typeof data !== 'object') return undefined; + const items = (data as { items?: unknown }).items; + if (!Array.isArray(items) || items.length === 0) return undefined; + + const images: Array<{ data: string; mimeType: string }> = []; + for (const item of items) { + if (!item || typeof item !== 'object') continue; + const content = (item as { content?: unknown }).content; + if (!Array.isArray(content)) continue; + + for (const block of content) { + if (!block || typeof block !== 'object') continue; + const type = (block as { type?: unknown }).type; + const blockData = (block as { data?: unknown }).data; + const mimeType = (block as { mimeType?: unknown }).mimeType; + + if ( + type === 'image' && + typeof blockData === 'string' && + typeof mimeType === 'string' + ) { + images.push({ data: blockData, mimeType }); + } + } + } + + return images.length > 0 ? images : undefined; +} + +/** + * Collect text content blocks from mid-turn injected message items. The + * degraded-media drain echo ships an empty `messages` array whose items carry + * only the unavailability notice, so the echo text can be empty while the + * items still hold renderable text. + */ +function getMidTurnInjectedItemText(data: unknown): string | undefined { + if (!data || typeof data !== 'object') return undefined; + const items = (data as { items?: unknown }).items; + if (!Array.isArray(items) || items.length === 0) return undefined; + + const texts: string[] = []; + for (const item of items) { + if (!item || typeof item !== 'object') continue; + const content = (item as { content?: unknown }).content; + if (!Array.isArray(content)) continue; + + for (const block of content) { + if (!block || typeof block !== 'object') continue; + if ((block as { type?: unknown }).type !== 'text') continue; + const text = (block as { text?: unknown }).text; + if (typeof text === 'string' && text.length > 0) texts.push(text); + } + } + + return texts.length > 0 ? texts.join('\n') : undefined; } function isBackgroundNotificationBlock( @@ -381,6 +433,23 @@ export function transcriptBlocksToDaemonMessages( const inputAnnotations = Array.isArray(meta?.inputAnnotations) ? (meta.inputAnnotations as DaemonInputAnnotation[]) : undefined; + const images = textBlock.images?.map((img) => ({ + data: img.data, + mimeType: img.mimeType || 'image/*', + })); + if (source === 'mid_turn_message_injected') { + messages.push({ + id: block.id, + role: 'system', + content: textBlock.text, + variant: 'info', + source, + timestamp: blockTime, + ...(images && images.length > 0 ? { images } : {}), + }); + needsNewContentMessage = true; + break; + } const msg: DaemonUserMessage = { id: block.id, role: 'user', @@ -390,11 +459,8 @@ export function transcriptBlocksToDaemonMessages( ...(inputAnnotations ? { inputAnnotations } : {}), }; // Attach images if present - if (textBlock.images && textBlock.images.length > 0) { - msg.images = textBlock.images.map((img) => ({ - data: img.data, - mimeType: img.mimeType || 'image/*', - })); + if (images && images.length > 0) { + msg.images = images; } if (textBlock.files && textBlock.files.length > 0) { msg.files = textBlock.files.map((file) => ({ @@ -761,20 +827,41 @@ export function transcriptBlocksToDaemonMessages( case 'debug': { const statusBlock = block; if (isUnrecognizedDaemonDebug(statusBlock)) break; + // Mid-turn injected echoes are user content, not daemon diagnostics: + // run them past no filter, or an injected message that merely starts + // like a status line ("Model switched: …") or plan JSON would be + // dropped or misrendered. + if (statusBlock.source === 'mid_turn_message_injected') { + const midTurnInjectedImages = getMidTurnInjectedImages( + statusBlock.data, + ); + messages.push({ + id: block.id, + role: 'system', + content: + statusBlock.text.length > 0 + ? statusBlock.text + : (getMidTurnInjectedItemText(statusBlock.data) ?? + statusBlock.text), + variant: 'info', + timestamp: blockTime, + source: statusBlock.source, + ...(statusBlock.data !== undefined + ? { data: statusBlock.data } + : {}), + ...(midTurnInjectedImages ? { images: midTurnInjectedImages } : {}), + }); + needsNewContentMessage = true; + break; + } const branchDisplayName = statusBlock.source === 'session_branched' ? getSessionBranchDisplayName(statusBlock.data) : null; - const midTurnInsertedText = - statusBlock.source === 'mid_turn_message_injected' - ? getMidTurnInjectedText(statusBlock.data) - : null; const text = branchDisplayName && options.labels?.branchSuccess ? options.labels.branchSuccess(branchDisplayName) - : midTurnInsertedText && options.labels?.midTurnInserted - ? options.labels.midTurnInserted(midTurnInsertedText) - : statusBlock.text; + : statusBlock.text; if (isIgnoredWebShellStatus(text)) break; const todos = parsePlanTodos(text); if (todos) { diff --git a/packages/web-shell/client/components/ChatPane.tsx b/packages/web-shell/client/components/ChatPane.tsx index dba968eac29..6c066bf667b 100644 --- a/packages/web-shell/client/components/ChatPane.tsx +++ b/packages/web-shell/client/components/ChatPane.tsx @@ -525,6 +525,8 @@ export function ChatPane({ connection.capabilities?.features.includes( 'session_mid_turn_message_query', ) === true; + const canInjectMidTurnMedia = + connection.capabilities?.features.includes('session_media') === true; const { queuedPrompts, queuedTexts, @@ -542,6 +544,7 @@ export function ChatPane({ clientId: connection.clientId, canMutateMidTurn, canQueryMidTurn, + canInjectMidTurnMedia, streamingState, sessionActions: actions, store, @@ -1058,6 +1061,7 @@ export function ChatPane({ onEdit={editQueuedPrompt} onRestoreUnknown={restoreUnknownQueuedPrompt} onDiscardUnknown={discardUnknownQueuedPrompt} + onImagePreview={handleImagePreview} /> {unknownPromptAdmission && (
{ switch (message.role) { case 'user': @@ -138,7 +142,9 @@ export const MessageItem = memo(function MessageItem({ variant={message.variant} source={message.source} data={message.data} + images={message.images} onShowContextDetail={onShowContextDetail} + onImagePreview={onImagePreview} isLatest={isLatest} showRetryHint={showRetryHint && message.retryable === true} onRetryClick={onRetryClick} @@ -190,9 +196,7 @@ export const MessageItem = memo(function MessageItem({ - } + fallback={} > {body} @@ -239,9 +243,9 @@ export const MessageItem = memo(function MessageItem({ return ( {selectableSafeBody} @@ -351,7 +355,8 @@ function areMessagesEqual(prev: Message, next: Message): boolean { prev.variant === next.variant && prev.retryable === next.retryable && prev.source === next.source && - prev.data === next.data + prev.data === next.data && + stableImagesEqual(prev.images, next.images) ); case 'user_shell': return ( diff --git a/packages/web-shell/client/components/MessageList.test.ts b/packages/web-shell/client/components/MessageList.test.ts index 621acf4def0..918a6ac05e2 100644 --- a/packages/web-shell/client/components/MessageList.test.ts +++ b/packages/web-shell/client/components/MessageList.test.ts @@ -2040,22 +2040,22 @@ describe('applyTurnCollapse', () => { expect(collapseOf(out, 0)?.hiddenCount).toBe(1); }); - it('hides mid-turn injected debug rows with collapsed tool steps', () => { + it('keeps mid-turn injected user messages visible with collapsed tool steps', () => { const items = groupParallelAgents([ makeUserMessage('u1'), makeMultiToolGroup('g1'), { id: 's1', role: 'system', - content: '已插入消息：hi', + content: 'hi', variant: 'info', source: 'mid_turn_message_injected', }, makeAssistantMessage('a1'), ]); const out = collapseItems(items); - expect(rowIds(out)).toEqual(['u1', 'tc-u1', 'a1']); - expect(collapseOf(out, 0)?.hiddenCount).toBe(2); + expect(rowIds(out)).toEqual(['u1', 'tc-u1', 's1', 'a1']); + expect(collapseOf(out, 0)?.hiddenCount).toBe(1); }); it('does not collapse a turn whose only response is a system row', () => { diff --git a/packages/web-shell/client/components/MessageList.tsx b/packages/web-shell/client/components/MessageList.tsx index c04b8455093..918a3c8c41d 100644 --- a/packages/web-shell/client/components/MessageList.tsx +++ b/packages/web-shell/client/components/MessageList.tsx @@ -699,7 +699,7 @@ function isHideableStep(item: DisplayItem, isFinalAnswer: boolean): boolean { if (item.message.source === 'background_notification') { return !isFinalAnswer; } - return isMidTurnInjectedDebugMessage(item.message); + return false; case 'user': case 'user_shell': case 'btw': diff --git a/packages/web-shell/client/components/QueuedPromptDisplay.test.tsx b/packages/web-shell/client/components/QueuedPromptDisplay.test.tsx index 224057dea57..96884e83e76 100644 --- a/packages/web-shell/client/components/QueuedPromptDisplay.test.tsx +++ b/packages/web-shell/client/components/QueuedPromptDisplay.test.tsx @@ -75,6 +75,82 @@ describe('QueuedPromptDisplay', () => { expect(container.textContent).toContain('排队消息二'); }); + it('renders image thumbnails after the text', () => { + const { container } = setup({ + prompts: [ + { + id: 1, + text: '带图消息', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }, + ], + }); + expect(container.textContent).toContain('带图消息'); + const img = container.querySelector( + 'img[class*="queuedPromptImage"]', + ); + expect(img).not.toBeNull(); + expect(img?.getAttribute('src')).toBe('data:image/png;base64,aW1n'); + expect(img?.getAttribute('alt')).toBe('用户上传的图片 1'); + // The thumbnail sits in the row after the text span. + const row = container.querySelector('[class*="queuedPromptText"]'); + expect(row).not.toBeNull(); + const position = row?.compareDocumentPosition(img as Node) ?? 0; + expect(position & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + + it('does not render unsafe image data URIs', () => { + const { container } = setup({ + prompts: [ + { + id: 1, + text: '不安全图片', + images: [{ data: 'PHNjcmlwdD4=', media_type: 'text/html' }], + }, + ], + }); + + expect( + container.querySelector('img[class*="queuedPromptImage"]'), + ).toBeNull(); + expect(container.querySelector('[class*="queuedPromptImages"]')).toBeNull(); + }); + + it('calls onImagePreview when a thumbnail is clicked', () => { + const onImagePreview = vi.fn(); + const { container } = setup({ + prompts: [ + { + id: 1, + text: '带图消息', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }, + ], + onImagePreview, + }); + const img = container.querySelector( + 'img[class*="queuedPromptImage"]', + ); + expect(img).not.toBeNull(); + expect(img?.classList.toString()).toContain('queuedPromptImageInteractive'); + act(() => img?.click()); + expect(onImagePreview).toHaveBeenCalledWith( + 'data:image/png;base64,aW1n', + '用户上传的图片 1', + ); + expect(img?.getAttribute('role')).toBe('button'); + expect(img?.tabIndex).toBe(0); + act(() => { + img?.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }), + ); + img?.dispatchEvent( + new KeyboardEvent('keydown', { key: ' ', bubbles: true }), + ); + }); + expect(onImagePreview).toHaveBeenCalledTimes(3); + }); + it('shows server queue status without an insert action', () => { const { container } = setup({ prompts: [{ id: 1, text: '等待处理', serverState: 'queued' }], diff --git a/packages/web-shell/client/components/QueuedPromptDisplay.tsx b/packages/web-shell/client/components/QueuedPromptDisplay.tsx index 06c3c44717f..f85935f75ad 100644 --- a/packages/web-shell/client/components/QueuedPromptDisplay.tsx +++ b/packages/web-shell/client/components/QueuedPromptDisplay.tsx @@ -22,6 +22,7 @@ import { } from '../utils/composerTag'; import { cssUrlVar } from '../utils/cssUrlVar'; import { ReadonlyComposerTag } from './messages/UserMessage'; +import { isSafeImageSrc } from './messages/Markdown'; import styles from '../App.module.css'; const MAX_QUEUED_PROMPT_PREVIEW_CHARS = 240; @@ -146,6 +147,7 @@ export function QueuedPromptDisplay({ onEdit, onRestoreUnknown, onDiscardUnknown, + onImagePreview, }: { prompts: readonly QueuedPrompt[]; t: ReturnType; @@ -154,6 +156,7 @@ export function QueuedPromptDisplay({ onEdit: (id: number) => void; onRestoreUnknown?: (id: number) => void; onDiscardUnknown?: (id: number) => void; + onImagePreview?: (src: string, alt?: string) => void; }) { const { parseUserMessageContent, @@ -192,7 +195,11 @@ export function QueuedPromptDisplay({ const preview = truncateQueuedPromptParts( getQueuedPromptParts(prompt, parseUserMessageContent), ); - const imageCount = prompt.images?.length ?? 0; + const safeImages = (prompt.images ?? []).flatMap((image, index) => { + const src = `data:${image.media_type};base64,${image.data}`; + return isSafeImageSrc(src) ? [{ index, src }] : []; + }); + const imageCount = safeImages.length; const fileCount = prompt.files?.length ?? 0; const isSubmitting = prompt.serverState === 'submitting'; const isQueued = prompt.serverState === 'queued'; @@ -256,9 +263,6 @@ export function QueuedPromptDisplay({ ), )} {preview.truncated ? '...' : null} - {imageCount > 0 - ? ` ${t('queue.imageCount', { count: imageCount })}` - : ''} {fileCount > 0 ? ` ${t('queue.fileCount', { count: fileCount })}` : ''} @@ -266,6 +270,46 @@ export function QueuedPromptDisplay({ ? ` ${t('queue.localCopyDiscarded')}` : ''} + {imageCount > 0 ? ( + + {safeImages.map(({ index, src }) => { + const alt = t('user.uploadedImage', { index: index + 1 }); + return ( + {alt} onImagePreview(src, alt) + : undefined + } + onKeyDown={ + onImagePreview + ? (event) => { + if (event.key !== 'Enter' && event.key !== ' ') + return; + event.preventDefault(); + onImagePreview(src, alt); + } + : undefined + } + /> + ); + })} + + ) : null} {isSubmitting || isQueued || isMidTurnPending || diff --git a/packages/web-shell/client/components/messages/SystemMessage.test.tsx b/packages/web-shell/client/components/messages/SystemMessage.test.tsx index 63dee06b21f..f37aff12a6a 100644 --- a/packages/web-shell/client/components/messages/SystemMessage.test.tsx +++ b/packages/web-shell/client/components/messages/SystemMessage.test.tsx @@ -379,3 +379,65 @@ describe('SystemMessage — goal status activation', () => { window.removeEventListener('web-shell-goal-status-active', handler); }); }); + +describe('SystemMessage — inline images', () => { + it('renders image thumbnails when images prop is provided', () => { + const container = render( + , + ); + + const img = container.querySelector('img'); + expect(img).not.toBeNull(); + expect(img?.getAttribute('src')).toBe('data:image/png;base64,base64data'); + expect(img?.className).toContain('chatImageThumb'); + }); + + it('makes images clickable when onImagePreview is provided', () => { + const onImagePreview = vi.fn(); + const container = render( + , + ); + + const img = container.querySelector('img'); + expect(img?.className).toContain('chatImageThumbInteractive'); + + act(() => { + img?.click(); + }); + + expect(onImagePreview).toHaveBeenCalledWith( + 'data:image/png;base64,base64data', + 'User uploaded image 1', + ); + }); + + it('renders multiple images in a row', () => { + const container = render( + , + ); + + const imgs = container.querySelectorAll('img'); + expect(imgs).toHaveLength(2); + expect(imgs[0]?.getAttribute('src')).toBe('data:image/png;base64,img1'); + expect(imgs[1]?.getAttribute('src')).toBe('data:image/jpeg;base64,img2'); + }); +}); diff --git a/packages/web-shell/client/components/messages/SystemMessage.tsx b/packages/web-shell/client/components/messages/SystemMessage.tsx index 994236312a5..86a3e3287f6 100644 --- a/packages/web-shell/client/components/messages/SystemMessage.tsx +++ b/packages/web-shell/client/components/messages/SystemMessage.tsx @@ -19,6 +19,7 @@ import { } from './TasksStatusMessage'; import { GoalStatusMessage, parseGoalStatusMessage } from './GoalStatusMessage'; import { Markdown } from './Markdown'; +import { UserMessage } from './UserMessage'; import styles from './SystemMessage.module.css'; interface SystemMessageProps { @@ -26,8 +27,11 @@ interface SystemMessageProps { variant: 'info' | 'error' | 'warning'; source?: string; data?: unknown; + images?: Array<{ data: string; mimeType: string }>; /** Run /context detail, exactly like typing it (context-usage panels). */ onShowContextDetail?: () => void; + /** Click an image to preview it in the right panel. */ + onImagePreview?: (src: string, alt?: string) => void; isLatest?: boolean; showRetryHint?: boolean; onRetryClick?: () => void; @@ -38,12 +42,23 @@ export const SystemMessage = memo(function SystemMessage({ variant, source, data, + images, onShowContextDetail, + onImagePreview, isLatest = false, showRetryHint = false, onRetryClick, }: SystemMessageProps) { const { t } = useI18n(); + if (source === 'mid_turn_message_injected') { + return ( + + ); + } // The user ESC-cancelled a live stream. Render it right-aligned and subtle — // a user-initiated stop reads as belonging to the user side of the transcript. if (source === 'prompt_cancelled') { diff --git a/packages/web-shell/client/hooks/useMessages.ts b/packages/web-shell/client/hooks/useMessages.ts index 9b05ad587b8..ce850357670 100644 --- a/packages/web-shell/client/hooks/useMessages.ts +++ b/packages/web-shell/client/hooks/useMessages.ts @@ -55,7 +55,6 @@ export function transcriptBlocksToLocalizedMessages( labels: { promptCancelled: t('request.cancelled'), branchSuccess: (name) => t('branch.success', { name }), - midTurnInserted: (message) => t('midTurn.inserted', { message }), modelStreamInterrupted: t('error.modelStreamInterrupted'), loopDetected: t('error.loopDetected'), }, diff --git a/packages/web-shell/client/hooks/useQueuedPrompts.dom.test.tsx b/packages/web-shell/client/hooks/useQueuedPrompts.dom.test.tsx index 15a8b743fcb..d3e3b6ecdc2 100644 --- a/packages/web-shell/client/hooks/useQueuedPrompts.dom.test.tsx +++ b/packages/web-shell/client/hooks/useQueuedPrompts.dom.test.tsx @@ -115,6 +115,7 @@ function mount( canMutateMidTurn, // This suite pins the legacy local-fallback lifecycle. canQueryMidTurn: false, + canInjectMidTurnMedia: false, streamingState: state, sessionActions, store, diff --git a/packages/web-shell/client/hooks/useQueuedPrompts.midTurnReconcile.test.tsx b/packages/web-shell/client/hooks/useQueuedPrompts.midTurnReconcile.test.tsx index ed76c94a4cb..5490195939e 100644 --- a/packages/web-shell/client/hooks/useQueuedPrompts.midTurnReconcile.test.tsx +++ b/packages/web-shell/client/hooks/useQueuedPrompts.midTurnReconcile.test.tsx @@ -20,6 +20,8 @@ const sdkMock = vi.hoisted(() => { const pendingEventListeners = new Set<() => void>(); const mock = { actions: { + uploadMedia: vi.fn(), + removeMedia: vi.fn(), enqueueMidTurnMessage: vi.fn(), getMidTurnMessages: vi.fn(), submitPrompt: vi.fn(), @@ -96,6 +98,7 @@ interface HarnessOptions { clientId?: string; canMutateMidTurn?: boolean; canQueryMidTurn?: boolean; + canInjectMidTurnMedia?: boolean; streamingState?: DaemonStreamingState; } @@ -130,6 +133,7 @@ function createHarness() { clientId: opts.clientId ?? CLIENT_ID, canMutateMidTurn: opts.canMutateMidTurn ?? true, canQueryMidTurn: opts.canQueryMidTurn ?? true, + canInjectMidTurnMedia: opts.canInjectMidTurnMedia ?? true, streamingState: opts.streamingState ?? 'responding', sessionActions: sdkMock.actions as never, store: stableStore as never, @@ -181,6 +185,15 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que (_message: string, opts?: { messageId?: string }) => Promise.resolve({ accepted: true, messageId: opts?.messageId }), ); + sdkMock.actions.uploadMedia.mockImplementation( + async (image: { mimeType?: string }) => ({ + type: 'image', + mediaId: 'media-1', + mimeType: image.mimeType ?? 'image/png', + size: 3, + }), + ); + sdkMock.actions.removeMedia.mockResolvedValue(true); sdkMock.actions.getMidTurnMessages.mockResolvedValue({ messages: [], settledMessageIds: [], @@ -199,6 +212,50 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que sdkMock.pendingEvents = []; }); + it('does not restore a row from a snapshot older than its injection', async () => { + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + let resolveSnapshot: ((value: unknown) => void) | undefined; + sdkMock.actions.getMidTurnMessages.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveSnapshot = resolve; + }), + ); + + await act(async () => { + harness.result().enqueuePrompt('already injected'); + await Promise.resolve(); + }); + const messageId = + sdkMock.actions.enqueueMidTurnMessage.mock.calls[0]?.[1]?.messageId; + expect(messageId).toEqual(expect.any(String)); + + sdkMock.injectedBatches = [ + { + sessionId: 'session-a', + messages: ['already injected'], + messageIds: [messageId], + }, + ]; + await harness.render({ streamingState: 'responding' }); + expect(sdkMock.actions.getMidTurnMessages).toHaveBeenCalledTimes(3); + await act(async () => { + resolveSnapshot?.({ + messages: [{ messageId, text: 'already injected' }], + settledMessageIds: [], + promotedMessageIds: [], + }); + await Promise.resolve(); + }); + + expect(harness.result().queuedPrompts).toEqual([]); + } finally { + await harness.dispose(); + } + }); + it('restores queued rows lost to a page refresh from the daemon snapshot', async () => { sdkMock.actions.getMidTurnMessages.mockResolvedValue({ messages: [ @@ -1038,7 +1095,7 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que } }); - it('restores a rejected stable-id admission after returning to its workspace', async () => { + it('cleans up a rejected stable-id admission after switching workspaces', async () => { let resolveAdmission: | ((value: { accepted: boolean; messageId?: string }) => void) | undefined; @@ -1065,17 +1122,17 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que await Promise.resolve(); }); + expect(harness.editor.setText).toHaveBeenCalledWith( + 'rejected in workspace-a', + ); + expect(harness.reportError).toHaveBeenCalledTimes(1); + await harness.render({ streamingState: 'responding', workspaceCwd: '/workspace-a', }); - expect(harness.result().queuedPrompts).toEqual([ - expect.objectContaining({ - text: 'rejected in workspace-a', - admissionOutcome: 'unknown', - }), - ]); + expect(harness.result().queuedPrompts).toEqual([]); } finally { await harness.dispose(); } @@ -1349,7 +1406,7 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que } }); - it('restores text when a failed enqueue is absent from the reconciliation snapshot', async () => { + it('keeps an ambiguous enqueue when the first reconciliation snapshot is empty', async () => { const onComplete = vi.fn(); let failedId: string | undefined; sdkMock.actions.enqueueMidTurnMessage.mockImplementation( @@ -1369,7 +1426,12 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que await act(async () => { harness .result() - .enqueuePrompt('lost in transit', undefined, undefined, onComplete); + .enqueuePrompt( + 'lost in transit', + [{ data: 'aW1n', media_type: 'image/png' }], + undefined, + onComplete, + ); }); for (let i = 0; i < 3; i++) { await act(async () => { @@ -1377,13 +1439,22 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que }); } - // The daemon never saw the message: hand the text back for a retry. - expect(harness.editor.setText).toHaveBeenCalledWith('lost in transit'); + // An empty snapshot cannot prove that the timed-out POST will not land + // after this read. Keep both the stable admission and its uploaded media + // until a later snapshot settles the id. + expect(harness.editor.setText).not.toHaveBeenCalled(); + expect(sdkMock.actions.removeMedia).not.toHaveBeenCalled(); expect(harness.reportError).toHaveBeenCalledTimes(1); expect(onComplete).not.toHaveBeenCalled(); + expect(harness.result().queuedPrompts).toEqual([ + expect.objectContaining({ + text: 'lost in transit', + midTurnMessageId: failedId, + admissionOutcome: 'unknown', + }), + ]); - // The catch path also deregistered the callback: a later snapshot - // settling the id must not fire it. + // A later authoritative snapshot settles the retained admission once. sdkMock.actions.getMidTurnMessages.mockResolvedValue({ messages: [], settledMessageIds: [failedId], @@ -1396,7 +1467,7 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que await Promise.resolve(); }); } - expect(onComplete).not.toHaveBeenCalled(); + expect(onComplete).toHaveBeenCalledTimes(1); } finally { await harness.dispose(); } @@ -1641,4 +1712,1131 @@ describe('useQueuedPrompts mid-turn reconciliation (session_mid_turn_message_que await harness.dispose(); } }); + + it('attaches images as content blocks on the mid-turn push', async () => { + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('look at this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + await act(async () => { + await Promise.resolve(); + }); + expect(sdkMock.actions.enqueueMidTurnMessage).toHaveBeenCalledWith( + 'look at this', + expect.objectContaining({ + messageId: expect.any(String), + content: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }), + ); + expect(sdkMock.actions.submitPrompt).not.toHaveBeenCalled(); + } finally { + await harness.dispose(); + } + }); + + it('removes uploaded media when mid-turn admission is rejected', async () => { + sdkMock.actions.enqueueMidTurnMessage.mockResolvedValueOnce({ + accepted: false, + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('look at this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + await Promise.resolve(); + }); + + expect(sdkMock.actions.removeMedia).toHaveBeenCalledWith('media-1', { + sessionId: 'session-a', + }); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + ]); + } finally { + await harness.dispose(); + } + }); + + it('does not enqueue uploaded media into a different session', async () => { + let finishUpload: + | ((reference: { + type: 'image'; + mediaId: string; + mimeType: string; + size: number; + }) => void) + | undefined; + sdkMock.actions.uploadMedia.mockReturnValueOnce( + new Promise((resolve) => { + finishUpload = resolve; + }), + ); + const harness = createHarness(); + try { + await harness.render({ sessionId: 'session-a' }); + act(() => { + harness + .result() + .enqueuePrompt('look at this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + await harness.render({ sessionId: 'session-b' }); + await act(async () => { + finishUpload?.({ + type: 'image', + mediaId: 'media-a', + mimeType: 'image/png', + size: 3, + }); + await Promise.resolve(); + }); + + expect(sdkMock.actions.enqueueMidTurnMessage).not.toHaveBeenCalled(); + expect(sdkMock.actions.removeMedia).toHaveBeenCalledWith('media-a', { + sessionId: 'session-a', + }); + } finally { + await harness.dispose(); + } + }); + + it('injects an image-only message mid-turn', async () => { + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('', [{ data: 'aW1n', media_type: 'image/png' }]); + }); + await act(async () => { + await Promise.resolve(); + }); + expect(sdkMock.actions.enqueueMidTurnMessage).toHaveBeenCalledWith( + '', + expect.objectContaining({ + messageId: expect.any(String), + content: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }), + ); + expect(sdkMock.actions.submitPrompt).not.toHaveBeenCalled(); + } finally { + await harness.dispose(); + } + }); + + it('restores media immediately when upload fails before admission', async () => { + sdkMock.actions.uploadMedia.mockRejectedValueOnce( + new Error('upload failed'), + ); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('keep this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + await Promise.resolve(); + }); + + expect(sdkMock.actions.enqueueMidTurnMessage).not.toHaveBeenCalled(); + expect(harness.editor.setText).toHaveBeenCalledWith('keep this'); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + ]); + } finally { + await harness.dispose(); + } + }); + + it('removes successful uploads when another image fails', async () => { + sdkMock.actions.uploadMedia + .mockResolvedValueOnce({ + type: 'image', + mediaId: 'uploaded-before-failure', + mimeType: 'image/png', + size: 3, + }) + .mockRejectedValueOnce(new Error('second upload failed')); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness.result().enqueuePrompt('keep this', [ + { data: 'aW1nMQ==', media_type: 'image/png' }, + { data: 'aW1nMg==', media_type: 'image/png' }, + ]); + await Promise.resolve(); + }); + + expect(sdkMock.actions.removeMedia).toHaveBeenCalledWith( + 'uploaded-before-failure', + { sessionId: 'session-a' }, + ); + expect(sdkMock.actions.enqueueMidTurnMessage).not.toHaveBeenCalled(); + } finally { + await harness.dispose(); + } + }); + + it('keeps the images on an accepted media row through reconciliation', async () => { + // The daemon snapshot is text-only; the row rebuilt from it must still + // carry the images so display and edit/restore don't lose them. + sdkMock.actions.getMidTurnMessages.mockImplementation(async () => { + const messageId = + sdkMock.actions.enqueueMidTurnMessage.mock.calls[0]?.[1]?.messageId; + return { + messages: messageId ? [{ messageId, text: 'look at this' }] : [], + settledMessageIds: [], + promotedMessageIds: [], + }; + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('look at this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + await act(async () => { + await Promise.resolve(); + }); + await act(async () => { + await Promise.resolve(); + }); + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + text: 'look at this', + midTurnState: 'queued', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }); + } finally { + await harness.dispose(); + } + }); + + it('restores images to the editor when editing a queued media row', async () => { + sdkMock.actions.getMidTurnMessages.mockImplementation(async () => { + const messageId = + sdkMock.actions.enqueueMidTurnMessage.mock.calls[0]?.[1]?.messageId; + return { + messages: messageId ? [{ messageId, text: 'edit me' }] : [], + settledMessageIds: [], + promotedMessageIds: [], + }; + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('edit me', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + await act(async () => { + await Promise.resolve(); + }); + await act(async () => { + await Promise.resolve(); + }); + const row = harness.result().queuedPrompts[0]; + expect(row?.images).toEqual([{ data: 'aW1n', media_type: 'image/png' }]); + + await act(async () => { + harness.result().editQueuedPrompt(row!.id); + }); + await act(async () => { + await Promise.resolve(); + }); + + // The daemon entry is removed and the full payload (text + images) is + // restored to the editor. + expect(sdkMock.actions.removeMidTurnMessage).toHaveBeenCalled(); + expect(harness.editor.setText).toHaveBeenCalledWith('edit me'); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + ]); + } finally { + await harness.dispose(); + } + }); + + it('keeps images when a media message is promoted into the pending-prompt FIFO', async () => { + // Settle race: the turn ends while the POST is in flight, so the daemon + // promotes the message instead of draining it. It then surfaces as a + // pending-prompt (server) row — that row must still carry the images so + // the queue shows them and editing restores them. + sdkMock.actions.getMidTurnMessages.mockImplementation(async () => { + const messageId = + sdkMock.actions.enqueueMidTurnMessage.mock.calls[0]?.[1]?.messageId; + return { + messages: [], + settledMessageIds: [], + promotedMessageIds: messageId ? [messageId] : [], + }; + }); + sdkMock.actions.getPendingPrompts.mockImplementation(async () => { + const messageId = + sdkMock.actions.enqueueMidTurnMessage.mock.calls[0]?.[1]?.messageId; + return { + pendingPrompts: messageId + ? [ + { + promptId: messageId, + text: 'promoted note', + queuedAt: Date.now(), + state: 'queued' as const, + }, + ] + : [], + }; + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('promoted note', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + await act(async () => { + await Promise.resolve(); + }); + await act(async () => { + await Promise.resolve(); + }); + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + text: 'promoted note', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }); + } finally { + await harness.dispose(); + } + }); + + it('restores images from the snapshot after a refresh (no in-memory admission)', async () => { + // Page-refresh case: nothing was enqueued this mount, so there is no pending + // admission to salvage from — the daemon snapshot's media blocks are the + // only source and must rebuild the row's images. + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-refresh', + text: 'refreshed note', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + await Promise.resolve(); + }); + await act(async () => { + await Promise.resolve(); + }); + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + text: 'refreshed note', + midTurnState: 'queued', + midTurnMessageId: 'm-refresh', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }); + } finally { + await harness.dispose(); + } + }); + + it('degrades a refresh-rebuilt row when media hydration failed', async () => { + // The SDK substitutes a placeholder text block for a media reference it + // could not hydrate. The rebuilt row must surface the loss (summary-only) + // instead of silently rendering as a complete, editable row. + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-degraded', + text: 'degraded note', + content: [ + { + type: 'text', + text: '[Attached media is no longer available]', + }, + ], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + await Promise.resolve(); + }); + await act(async () => { + await Promise.resolve(); + }); + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + text: 'degraded note', + midTurnState: 'queued', + midTurnMessageId: 'm-degraded', + payloadCompleteness: 'summary-only', + }); + expect(row?.images).toBeUndefined(); + } finally { + await harness.dispose(); + } + }); + + it('restores images from pending-prompt content after a refresh', async () => { + // Page-refresh case for a promoted message: nothing was enqueued this + // mount, so there is no pending admission to salvage from — the daemon's + // getPendingPrompts content field is the only source and must rebuild the + // row's images. + sdkMock.actions.getPendingPrompts.mockResolvedValue({ + pendingPrompts: [ + { + promptId: 'p-refresh', + text: 'refreshed prompt', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + queuedAt: Date.now(), + state: 'queued' as const, + }, + ], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'idle' }); + await act(async () => { + await Promise.resolve(); + }); + await act(async () => { + await Promise.resolve(); + }); + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + text: 'refreshed prompt', + serverPromptId: 'p-refresh', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }); + // A server row rebuilt WITH hydrated images is payload-complete — it + // must not stay pinned to summary-only (which disables editing and + // leaves delete-and-retype as the only way to change the message). + expect(row?.payloadCompleteness).not.toBe('summary-only'); + + // Editing proceeds through the pending-prompt removal instead of + // early-returning, and restores text + images into the editor. + sdkMock.actions.removePendingPrompt.mockResolvedValue({ removed: true }); + await act(async () => { + void harness.result().editQueuedPrompt(row!.id); + }); + for (let i = 0; i < 3; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(sdkMock.actions.removePendingPrompt).toHaveBeenCalledWith( + 'p-refresh', + { sessionId: 'session-a' }, + ); + expect(harness.editor.setText).toHaveBeenCalledWith('refreshed prompt'); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + ]); + } finally { + await harness.dispose(); + } + }); + + it('keeps images on the next turn when the daemon lacks the media capability', async () => { + const harness = createHarness(); + try { + await harness.render({ + streamingState: 'responding', + canInjectMidTurnMedia: false, + }); + await act(async () => { + harness + .result() + .enqueuePrompt('with image', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + await act(async () => { + await Promise.resolve(); + }); + expect(sdkMock.actions.enqueueMidTurnMessage).not.toHaveBeenCalled(); + expect(sdkMock.actions.submitPrompt).toHaveBeenCalledWith( + 'with image', + expect.objectContaining({ + images: [{ data: 'aW1n', media_type: 'image/png' }], + }), + ); + } finally { + await harness.dispose(); + } + }); + + it('keeps the whole message on the next turn when an image has no concrete mime type', async () => { + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + await act(async () => { + harness + .result() + .enqueuePrompt('odd image', [ + { data: 'aW1n', media_type: 'image/*' }, + ]); + }); + await act(async () => { + await Promise.resolve(); + }); + expect(sdkMock.actions.enqueueMidTurnMessage).not.toHaveBeenCalled(); + expect(sdkMock.actions.submitPrompt).toHaveBeenCalledWith( + 'odd image', + expect.objectContaining({ + images: [{ data: 'aW1n', media_type: 'image/*' }], + }), + ); + } finally { + await harness.dispose(); + } + }); + + it('upgrades a degraded row once a later snapshot hydrates the media', async () => { + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-degraded', + text: 'degraded note', + content: [ + { + type: 'text', + text: '[Attached media is no longer available]', + }, + ], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + let row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + midTurnMessageId: 'm-degraded', + payloadCompleteness: 'summary-only', + }); + expect(row?.images).toBeUndefined(); + + // The daemon still holds the media; the next reconciliation hydrates it, + // so the provisional degradation must clear and the payload returns. + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-degraded', + text: 'degraded note', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + await harness.render({ + streamingState: 'responding', + connected: false, + }); + await harness.render({ streamingState: 'responding', connected: true }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + + row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + midTurnMessageId: 'm-degraded', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }); + expect(row?.payloadCompleteness).not.toBe('summary-only'); + + // The row is editable again: editing restores text + images. + await act(async () => { + void harness.result().editQueuedPrompt(row!.id); + }); + for (let i = 0; i < 3; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(harness.editor.setText).toHaveBeenCalledWith('degraded note'); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + ]); + } finally { + await harness.dispose(); + } + }); + + it('degrades a refresh-rebuilt row when media hydration only transiently failed', async () => { + // A transient hydration failure (anything but 404/410) leaves the raw + // reference block in the snapshot — image-shaped but without string + // `data`. The rebuilt row must degrade to summary-only like the + // placeholder case, so editing cannot silently discard attachments the + // daemon still holds. + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-flaky', + text: 'flaky note', + content: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + for (let i = 0; i < 2; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + text: 'flaky note', + midTurnState: 'queued', + midTurnMessageId: 'm-flaky', + payloadCompleteness: 'summary-only', + }); + expect(row?.images).toBeUndefined(); + + // Editing stays blocked: no daemon-message removal, no draft restore. + await act(async () => { + void harness.result().editQueuedPrompt(row!.id); + }); + for (let i = 0; i < 3; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(sdkMock.actions.removeMidTurnMessage).not.toHaveBeenCalled(); + expect(harness.editor.setText).not.toHaveBeenCalled(); + } finally { + await harness.dispose(); + } + }); + + it('self-heals a transiently degraded row once every reference hydrates', async () => { + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-flaky', + text: 'flaky note', + content: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + expect(harness.result().queuedPrompts[0]).toMatchObject({ + midTurnMessageId: 'm-flaky', + payloadCompleteness: 'summary-only', + }); + + // A partially hydrated snapshot (one reference still unhydrated) must + // NOT upgrade the row — upgrading on the hydrated subset would drop + // the other attachment. + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-flaky', + text: 'flaky note', + content: [ + { type: 'image', data: 'aW1n', mimeType: 'image/png' }, + { + type: 'image', + mediaId: 'media-2', + mimeType: 'image/png', + size: 3, + }, + ], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + await harness.render({ + streamingState: 'responding', + connected: false, + }); + await harness.render({ streamingState: 'responding', connected: true }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(harness.result().queuedPrompts[0]).toMatchObject({ + midTurnMessageId: 'm-flaky', + payloadCompleteness: 'summary-only', + }); + + // Fully hydrated: the upgrade path restores the payload and editability. + sdkMock.actions.getMidTurnMessages.mockResolvedValue({ + messages: [ + { + messageId: 'm-flaky', + text: 'flaky note', + content: [ + { type: 'image', data: 'aW1n', mimeType: 'image/png' }, + { type: 'image', data: 'aW1nMg==', mimeType: 'image/png' }, + ], + }, + ], + settledMessageIds: [], + promotedMessageIds: [], + }); + await harness.render({ + streamingState: 'responding', + connected: false, + }); + await harness.render({ streamingState: 'responding', connected: true }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + midTurnMessageId: 'm-flaky', + images: [ + { data: 'aW1n', media_type: 'image/png' }, + { data: 'aW1nMg==', media_type: 'image/png' }, + ], + }); + expect(row?.payloadCompleteness).not.toBe('summary-only'); + + // The row is editable again: editing restores text + images. + await act(async () => { + void harness.result().editQueuedPrompt(row!.id); + }); + for (let i = 0; i < 3; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(harness.editor.setText).toHaveBeenCalledWith('flaky note'); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + { data: 'aW1nMg==', media_type: 'image/png' }, + ]); + } finally { + await harness.dispose(); + } + }); + + it('echoes text and images when a promoted media message starts', async () => { + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + let messageId: string | undefined; + sdkMock.actions.enqueueMidTurnMessage.mockImplementation( + (_message: string, opts?: { messageId?: string }) => { + messageId = opts?.messageId; + return Promise.resolve({ + accepted: true, + messageId: opts?.messageId, + }); + }, + ); + await act(async () => { + harness + .result() + .enqueuePrompt('look at this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + await act(async () => { + sdkMock.publishPendingEvents([ + { + type: 'pending_prompt_started', + promptId: messageId, + originatorClientId: CLIENT_ID, + data: { + sessionId: 'session-a', + promptId: messageId, + text: 'look at this', + }, + }, + ]); + }); + expect(harness.store.appendLocalUserMessage).toHaveBeenCalledWith( + 'look at this', + [{ data: 'aW1n', mimeType: 'image/png' }], + undefined, + undefined, + ); + } finally { + await harness.dispose(); + } + }); + + it('echoes an image-only message when its promoted turn starts', async () => { + const harness = createHarness(); + try { + await harness.render({ streamingState: 'responding' }); + let messageId: string | undefined; + sdkMock.actions.enqueueMidTurnMessage.mockImplementation( + (_message: string, opts?: { messageId?: string }) => { + messageId = opts?.messageId; + return Promise.resolve({ + accepted: true, + messageId: opts?.messageId, + }); + }, + ); + await act(async () => { + harness + .result() + .enqueuePrompt('', [{ data: 'aW1n', media_type: 'image/png' }]); + }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + await act(async () => { + sdkMock.publishPendingEvents([ + { + type: 'pending_prompt_started', + promptId: messageId, + originatorClientId: CLIENT_ID, + data: { + sessionId: 'session-a', + promptId: messageId, + text: '', + }, + }, + ]); + }); + expect(harness.store.appendLocalUserMessage).toHaveBeenCalledWith( + '', + [{ data: 'aW1n', mimeType: 'image/png' }], + undefined, + undefined, + ); + } finally { + await harness.dispose(); + } + }); + + it('restores the draft when the session changes before upload reaches the daemon', async () => { + let finishUpload: + | ((reference: { + type: 'image'; + mediaId: string; + mimeType: string; + size: number; + }) => void) + | undefined; + sdkMock.actions.uploadMedia.mockReturnValueOnce( + new Promise((resolve) => { + finishUpload = resolve; + }), + ); + const harness = createHarness(); + try { + await harness.render({ sessionId: 'session-a' }); + act(() => { + harness + .result() + .enqueuePrompt('keep this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + await harness.render({ sessionId: 'session-b' }); + await act(async () => { + finishUpload?.({ + type: 'image', + mediaId: 'media-a', + mimeType: 'image/png', + size: 3, + }); + await Promise.resolve(); + }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + + // Nothing reached the daemon, so the draft comes back and the stale + // admission is dropped instead of leaking into the other session. + expect(sdkMock.actions.enqueueMidTurnMessage).not.toHaveBeenCalled(); + expect(harness.editor.setText).toHaveBeenCalledWith('keep this'); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + ]); + expect(harness.reportError).toHaveBeenCalled(); + + // Returning to session A must not materialize an unresolvable row. + await harness.render({ sessionId: 'session-a' }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(harness.result().queuedPrompts).toEqual([]); + } finally { + await harness.dispose(); + } + }); + + it('clears summary-only when a refresh restores fully hydrated images into an existing row', async () => { + // A pending prompt whose references transiently fail hydration rebuilds + // as summary-only; once a later refresh hydrates them, the existing row + // must regain its images AND its editability. + sdkMock.actions.getPendingPrompts.mockResolvedValue({ + pendingPrompts: [ + { + promptId: 'p-flaky', + text: 'flaky prompt', + content: [ + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + queuedAt: Date.now(), + state: 'queued' as const, + }, + ], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'idle' }); + for (let i = 0; i < 2; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + const degraded = harness.result().queuedPrompts[0]; + expect(degraded).toMatchObject({ + serverPromptId: 'p-flaky', + payloadCompleteness: 'summary-only', + }); + expect(degraded?.images).toBeUndefined(); + + // The next refresh hydrates fully: the row regains images and the + // summary-only flag clears. + sdkMock.actions.getPendingPrompts.mockResolvedValue({ + pendingPrompts: [ + { + promptId: 'p-flaky', + text: 'flaky prompt', + content: [{ type: 'image', data: 'aW1n', mimeType: 'image/png' }], + queuedAt: Date.now(), + state: 'queued' as const, + }, + ], + }); + await harness.render({ streamingState: 'idle', connected: false }); + await harness.render({ streamingState: 'idle', connected: true }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + serverPromptId: 'p-flaky', + images: [{ data: 'aW1n', media_type: 'image/png' }], + }); + expect(row?.payloadCompleteness).not.toBe('summary-only'); + + // Editing proceeds instead of early-returning. + sdkMock.actions.removePendingPrompt.mockResolvedValue({ removed: true }); + await act(async () => { + void harness.result().editQueuedPrompt(row!.id); + }); + for (let i = 0; i < 3; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(sdkMock.actions.removePendingPrompt).toHaveBeenCalledWith( + 'p-flaky', + { sessionId: 'session-a' }, + ); + expect(harness.editor.restoreImages).toHaveBeenCalledWith([ + { data: 'aW1n', media_type: 'image/png' }, + ]); + } finally { + await harness.dispose(); + } + }); + + it('keeps a partially hydrated pending-prompt row summary-only', async () => { + // One attachment hydrated, one still an unhydrated reference: restoring + // only the survivor and marking the row complete would let editing + // silently discard the attachment the daemon still holds. + sdkMock.actions.getPendingPrompts.mockResolvedValue({ + pendingPrompts: [ + { + promptId: 'p-partial', + text: 'look at both', + content: [ + { type: 'image', data: 'aW1n', mimeType: 'image/png' }, + { + type: 'image', + mediaId: 'media-2', + mimeType: 'image/png', + size: 3, + }, + ], + queuedAt: Date.now(), + state: 'queued' as const, + }, + ], + }); + const harness = createHarness(); + try { + await harness.render({ streamingState: 'idle' }); + for (let i = 0; i < 2; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + const row = harness.result().queuedPrompts[0]; + expect(row).toMatchObject({ + serverPromptId: 'p-partial', + payloadCompleteness: 'summary-only', + }); + expect(row?.images).toBeUndefined(); + } finally { + await harness.dispose(); + } + }); + + it('drops the pinned admission when the session changes after the enqueue was dispatched', async () => { + // Upload complete, enqueue in flight, session switched: the abort + // rejects the dispatched enqueue. The admission (with its base64 images) + // must be dropped instead of staying pinned until reload and + // materializing a stale row on return. + let rejectEnqueue: ((error: Error) => void) | undefined; + sdkMock.actions.enqueueMidTurnMessage.mockReturnValueOnce( + new Promise((_resolve, reject) => { + rejectEnqueue = reject; + }), + ); + const harness = createHarness(); + try { + await harness.render({ + sessionId: 'session-a', + streamingState: 'responding', + }); + await act(async () => { + harness + .result() + .enqueuePrompt('leak this', [ + { data: 'aW1n', media_type: 'image/png' }, + ]); + }); + // Let the upload settle so the enqueue is dispatched (enqueueStarted). + await act(async () => { + await Promise.resolve(); + }); + expect(sdkMock.actions.enqueueMidTurnMessage).toHaveBeenCalledTimes(1); + + await harness.render({ + sessionId: 'session-b', + streamingState: 'responding', + }); + await act(async () => { + rejectEnqueue?.(new DOMException('Aborted', 'AbortError')); + await Promise.resolve(); + }); + + // Returning to session-a must not materialize the stale admission row. + await harness.render({ + sessionId: 'session-a', + streamingState: 'responding', + }); + for (let i = 0; i < 4; i++) { + await act(async () => { + await Promise.resolve(); + }); + } + expect(harness.result().queuedPrompts).toEqual([]); + expect(harness.reportError).not.toHaveBeenCalled(); + } finally { + await harness.dispose(); + } + }); }); diff --git a/packages/web-shell/client/hooks/useQueuedPrompts.ts b/packages/web-shell/client/hooks/useQueuedPrompts.ts index 490e506fb0c..dd236320db4 100644 --- a/packages/web-shell/client/hooks/useQueuedPrompts.ts +++ b/packages/web-shell/client/hooks/useQueuedPrompts.ts @@ -27,7 +27,9 @@ import type { DaemonInputAnnotation, DaemonMidTurnMessagesResult, DaemonPendingPromptSummary, + DaemonSessionMediaReference, DaemonTranscriptStore, + PromptContentBlock, } from '@qwen-code/sdk/daemon'; import type { PromptFile, PromptImage } from '../adapters/promptTypes'; import type { EditorHandle } from './useComposerCore'; @@ -61,6 +63,13 @@ interface UseQueuedPromptsArgs { * the legacy local fallback used by older daemons. */ canQueryMidTurn: boolean; + /** + * Whether the daemon advertises `session_media`. With it, + * images attached to a mid-turn send travel with the message and are + * injected into the running turn; without it they stay queued for the next + * turn (an older daemon would silently drop them). + */ + canInjectMidTurnMedia: boolean; streamingState: DaemonStreamingState; sessionActions: DaemonSessionActions; store: DaemonTranscriptStore; @@ -132,6 +141,66 @@ function toStoreImages( })); } +/** + * Recover queued-row images from a reconciliation snapshot's media blocks. + * After a page refresh the in-memory pending admission is gone, so the daemon + * snapshot is the only source left for the attachments. + */ +function contentToImages( + content: readonly PromptContentBlock[] | undefined, +): PromptImage[] | undefined { + if (!content || content.length === 0) return undefined; + const images: PromptImage[] = []; + for (const block of content) { + if (typeof block !== 'object' || block === null) continue; + const record = block as Record; + if (record['type'] !== 'image') continue; + const data = record['data']; + const mimeType = record['mimeType']; + if (typeof data === 'string') { + images.push({ + data, + media_type: typeof mimeType === 'string' ? mimeType : 'image/*', + }); + } + } + return images.length > 0 ? images : undefined; +} + +// The SDK substitutes this text block for a media reference it could not +// hydrate (DaemonSessionClient.hydrateBlock); keep in sync with the SDK. +const MEDIA_UNAVAILABLE_PLACEHOLDER = '[Attached media is no longer available]'; + +function contentHasDegradedMedia( + content: readonly PromptContentBlock[] | undefined, +): boolean { + if (!content || content.length === 0) return false; + return content.some((block) => { + if (typeof block !== 'object' || block === null) return false; + const record = block as Record; + return ( + record['type'] === 'text' && + record['text'] === MEDIA_UNAVAILABLE_PLACEHOLDER + ); + }); +} + +// A transient hydration failure (anything but 404/410) returns the raw +// reference block — an image-shaped block without string `data` — instead of +// the placeholder (DaemonSessionClient.hydrateBlock). Treat it as provisional +// degradation: the daemon still holds the blob, so a later hydrated snapshot +// upgrades the row back, while editing it must stay blocked. +function contentHasUnhydratedMedia( + content: readonly PromptContentBlock[] | undefined, +): boolean { + if (!content || content.length === 0) return false; + return content.some((block) => { + if (typeof block !== 'object' || block === null) return false; + const record = block as Record; + return record['type'] === 'image' && typeof record['data'] !== 'string'; + }); +} + function toStoreFiles( files: readonly PromptFile[] | undefined, ): Array<{ name: string; mimeType: string }> | undefined { @@ -169,6 +238,7 @@ export function useQueuedPrompts({ clientId, canMutateMidTurn, canQueryMidTurn, + canInjectMidTurnMedia, streamingState, sessionActions, store, @@ -287,6 +357,15 @@ export function useQueuedPrompts({ const hasDisplayedPrompt = displayedServerPromptIdsRef.current.has( serverPrompt.promptId, ); + // Extract images from the server prompt's content field (if present) + const serverImages = contentToImages(serverPrompt.content); + // A partially hydrated payload (a loss placeholder or a raw, + // unhydrated reference) must not upgrade a row: editing it would + // silently discard the attachments the daemon still holds. Only + // fully hydrated content restores images and clears summary-only. + const contentFullyHydrated = + !contentHasDegradedMedia(serverPrompt.content) && + !contentHasUnhydratedMedia(serverPrompt.content); if (existingIndex !== -1) { if (hasDisplayedPrompt) { next.splice(existingIndex, 1); @@ -297,6 +376,13 @@ export function useQueuedPrompts({ ...(next[existingIndex]!.payloadCompleteness === 'summary-only' ? { text: serverPrompt.text } : {}), + // Restore images from server content if local row doesn't have + // them; clearing summary-only makes the restored row editable. + ...(serverImages && + contentFullyHydrated && + !next[existingIndex]!.images + ? { images: serverImages, payloadCompleteness: undefined } + : {}), midTurnState: undefined, midTurnMessageId: undefined, midTurnFailedAction: undefined, @@ -343,9 +429,19 @@ export function useQueuedPrompts({ id: nextQueuedPromptIdRef.current++, sessionId: targetSessionId, text: serverPrompt.text, + ...(serverImages && contentFullyHydrated + ? { images: serverImages } + : {}), serverPromptId: serverPrompt.promptId, serverState: serverPrompt.state, - payloadCompleteness: 'summary-only', + // A row rebuilt with fully hydrated images is payload-complete; + // pinning it to summary-only would disable editing until the user + // deletes (and loses) the attachments. A partially hydrated + // payload stays summary-only so editing cannot silently discard + // the attachments the daemon still holds — a later fully hydrated + // refresh upgrades the row. + payloadCompleteness: + serverImages && contentFullyHydrated ? undefined : 'summary-only', }); } if (areQueuedPromptsEqual(queuedPromptsRef.current, next)) return; @@ -397,7 +493,18 @@ export function useQueuedPrompts({ ): Set => { const settledIds = new Set(snapshot.settledMessageIds); const promotedIds = new Set(snapshot.promotedMessageIds); + // The daemon snapshot is text-only; salvage the images still held by the + // pending admissions before deleting them, so the restored rows stay + // payload-complete (an edited or displayed row must not lose them). + const salvagedImages = new Map(); for (const message of snapshot.messages) { + const pending = pendingMidTurnAdmissionsRef.current.get( + message.messageId, + ); + const images = pending?.prompt.images; + if (images && images.length > 0) { + salvagedImages.set(message.messageId, images); + } pendingMidTurnAdmissionsRef.current.delete(message.messageId); } for (const messageId of settledIds) { @@ -406,7 +513,17 @@ export function useQueuedPrompts({ completionCallbacksRef.current.delete(messageId); callback?.(); } + // A promoted message surfaces as a pending-prompt (server) row built from + // the text-only `getPendingPrompts` summary, so salvage its images here + // too — otherwise the promoted row displays nothing and editing it can't + // restore the attachments. + const promotedImages = new Map(); for (const messageId of promotedIds) { + const pending = pendingMidTurnAdmissionsRef.current.get(messageId); + const images = pending?.prompt.images; + if (images && images.length > 0) { + promotedImages.set(messageId, images); + } pendingMidTurnAdmissionsRef.current.delete(messageId); } const waitingIds = new Set( @@ -437,6 +554,29 @@ export function useQueuedPrompts({ } : prompt, ); + // A degraded (summary-only) row is provisional: the daemon still holds + // the media, so a later snapshot that hydrates it restores the payload. + next = next.map((prompt) => { + if ( + prompt.payloadCompleteness !== 'summary-only' || + prompt.midTurnMessageId === undefined + ) { + return prompt; + } + const message = snapshot.messages.find( + (item) => item.messageId === prompt.midTurnMessageId, + ); + if ( + !message || + contentHasDegradedMedia(message.content) || + contentHasUnhydratedMedia(message.content) + ) { + return prompt; + } + const hydrated = contentToImages(message.content); + if (!hydrated) return prompt; + return { ...prompt, images: hydrated, payloadCompleteness: undefined }; + }); if (next.length !== current.length) { const retainedIds = new Set(next.map((prompt) => prompt.id)); for (const prompt of current) { @@ -463,15 +603,38 @@ export function useQueuedPrompts({ const restoredRows: QueuedPrompt[] = []; for (const message of snapshot.messages) { if (localIds.has(message.messageId)) continue; + // Prefer the in-memory admission's images; after a refresh only the + // snapshot's media blocks remain. + const salvaged = salvagedImages.get(message.messageId); + const images = salvaged ?? contentToImages(message.content); restoredRows.push({ id: nextQueuedPromptIdRef.current++, sessionId: targetSessionId, text: message.text, + ...(images ? { images } : {}), + // A hydration-failure placeholder in the snapshot means the row's + // attachments are gone from the client; an unhydrated reference + // means they are transiently unreachable. Degrade both like a + // summary-only row so editing cannot silently discard them — a + // later hydrated snapshot upgrades the provisional case back. + ...(salvaged === undefined && + (contentHasDegradedMedia(message.content) || + contentHasUnhydratedMedia(message.content)) + ? { payloadCompleteness: 'summary-only' as const } + : {}), midTurnState: 'queued', midTurnMessageId: message.messageId, }); } if (restoredRows.length > 0) next = [...next, ...restoredRows]; + if (promotedImages.size > 0) { + next = next.map((prompt) => { + if ((prompt.images?.length ?? 0) > 0) return prompt; + const key = prompt.serverPromptId ?? prompt.midTurnMessageId; + const images = key ? promotedImages.get(key) : undefined; + return images ? { ...prompt, images } : prompt; + }); + } if (!areQueuedPromptsEqual(current, next)) { queuedPromptsRef.current = next; setQueuedPrompts(next); @@ -1101,10 +1264,24 @@ export function useQueuedPrompts({ const targetSessionId = latestSessionIdRef.current; const targetWorkspaceCwd = latestWorkspaceCwdRef.current; const ownerToken = ownerTokenRef.current; + const imageList = images ?? []; + // Mid-turn media needs the daemon-owned id surface AND the daemon's media + // capability; an image we can't type also keeps the whole message on the + // next-turn path so the daemon never drops part of the payload. + const canSendMidTurnMedia = + imageList.length > 0 && + canQueryMidTurn && + canInjectMidTurnMedia && + imageList.every( + (image) => + image.data.length > 0 && + image.media_type.startsWith('image/') && + image.media_type !== 'image/*', + ); const shouldInsertMidTurn = latestStreamingStateRef.current !== 'idle' && - (images?.length ?? 0) === 0 && (files?.length ?? 0) === 0 && + (imageList.length === 0 || canSendMidTurnMedia) && (inputAnnotations?.length ?? 0) === 0 && !isCommandPrompt(trimmed); const midTurnMessageId = @@ -1122,6 +1299,7 @@ export function useQueuedPrompts({ id: nextQueuedPromptIdRef.current++, sessionId: targetSessionId, text: trimmed, + ...(imageList.length > 0 ? { images: [...imageList] } : {}), midTurnMessageId, admissionOutcome: 'unknown', payloadCompleteness: 'complete', @@ -1131,22 +1309,95 @@ export function useQueuedPrompts({ prompt: pendingAdmission, workspaceCwd: targetWorkspaceCwd, }); + if (imageList.length > 0) { + queuedPromptsRef.current = [ + ...queuedPromptsRef.current, + { ...pendingAdmission, midTurnState: 'submitting' }, + ]; + setQueuedPrompts(queuedPromptsRef.current); + } if (onComplete) { settleCompletionCallback(midTurnMessageId, onComplete); } - void sessionActions - .enqueueMidTurnMessage(trimmed, { messageId: midTurnMessageId }) + const abort = midTurnEnqueueAbortRef.current ?? new AbortController(); + midTurnEnqueueAbortRef.current = abort; + let enqueueStarted = false; + let uploadedMediaReferences: DaemonSessionMediaReference[] = []; + const removeUploadedMedia = async () => { + if (!targetSessionId) return; + await Promise.allSettled( + uploadedMediaReferences.map( + async (reference) => + await sessionActions.removeMedia(reference.mediaId, { + sessionId: targetSessionId, + }), + ), + ); + }; + void Promise.allSettled( + imageList.map( + async (image) => + await sessionActions.uploadMedia( + { + data: image.data, + mimeType: image.media_type, + }, + { signal: abort.signal }, + ), + ), + ) + .then(async (results) => { + uploadedMediaReferences = results.flatMap((result) => + result.status === 'fulfilled' ? [result.value] : [], + ); + const failure = results.find( + (result): result is PromiseRejectedResult => + result.status === 'rejected', + ); + if (failure) { + await removeUploadedMedia(); + throw failure.reason; + } + if ( + abort.signal.aborted || + latestSessionIdRef.current !== targetSessionId || + latestWorkspaceCwdRef.current !== targetWorkspaceCwd + ) { + await removeUploadedMedia(); + throw new DOMException('Session changed', 'AbortError'); + } + enqueueStarted = true; + return await sessionActions.enqueueMidTurnMessage(trimmed, { + signal: abort.signal, + messageId: midTurnMessageId, + ...(uploadedMediaReferences.length > 0 + ? { content: uploadedMediaReferences } + : {}), + }); + }) .then(async (result) => { if (!result.accepted) { + await removeUploadedMedia(); completionCallbacksRef.current.delete(midTurnMessageId); + const pendingAdmissionStillOwned = + pendingMidTurnAdmissionsRef.current.delete(midTurnMessageId); if ( latestSessionIdRef.current !== targetSessionId || latestWorkspaceCwdRef.current !== targetWorkspaceCwd ) { + if (pendingAdmissionStillOwned) { + restoreQueuedPromptsToEditor( + [pendingAdmission], + undefined, + true, + ); + reportError( + new Error('Daemon rejected mid-turn message'), + t('queue.queueFailed'), + ); + } return; } - const pendingAdmissionStillOwned = - pendingMidTurnAdmissionsRef.current.delete(midTurnMessageId); if (!pendingAdmissionStillOwned) return; const next = queuedPromptsRef.current.filter( (prompt) => prompt.midTurnMessageId !== midTurnMessageId, @@ -1171,7 +1422,10 @@ export function useQueuedPrompts({ ) { await reconcileMidTurnMessages(targetSessionId); } - pendingMidTurnAdmissionsRef.current.delete(midTurnMessageId); + // Do NOT delete the pending admission here. `applyMidTurnSnapshot` + // deletes it atomically with salvaging its images; deleting early + // opens a race where a concurrent (e.g. streaming-driven) reconcile + // applies after this delete and rebuilds the row without the images. }) .catch(async (error: unknown) => { if ( @@ -1179,6 +1433,45 @@ export function useQueuedPrompts({ latestWorkspaceCwdRef.current !== targetWorkspaceCwd || !targetSessionId ) { + completionCallbacksRef.current.delete(midTurnMessageId); + const pendingAdmissionStillOwned = + pendingMidTurnAdmissionsRef.current.delete(midTurnMessageId); + if (!enqueueStarted) { + // Nothing reached the daemon, so the draft is still ours to + // return: restore it to the current editor instead of + // leaking it across the session switch. + if (pendingAdmissionStillOwned) { + restoreQueuedPromptsToEditor( + [pendingAdmission], + undefined, + true, + ); + reportError(error, t('queue.queueFailed')); + } + } + // An enqueue already dispatched when the session changed may + // have reached the daemon: keep the uploaded media (a queued + // message may reference it) and drop only the admission, so + // its base64 payload is not pinned until reload and no stale + // row materializes when returning to the old session. + return; + } + if (!enqueueStarted) { + completionCallbacksRef.current.delete(midTurnMessageId); + const pendingAdmissionStillOwned = + pendingMidTurnAdmissionsRef.current.delete(midTurnMessageId); + if (!pendingAdmissionStillOwned) return; + const next = queuedPromptsRef.current.filter( + (prompt) => prompt.midTurnMessageId !== midTurnMessageId, + ); + queuedPromptsRef.current = next; + setQueuedPrompts(next); + restoreQueuedPromptsToEditor( + [pendingAdmission], + targetSessionId, + true, + ); + reportError(error, t('queue.queueFailed')); return; } const snapshot = await reconcileMidTurnMessages(targetSessionId); @@ -1199,6 +1492,9 @@ export function useQueuedPrompts({ id: nextQueuedPromptIdRef.current++, sessionId: targetSessionId, text: trimmed, + // Keep the images so resolving this unknown row restores + // the full payload, not just the text. + ...(imageList.length > 0 ? { images: [...imageList] } : {}), midTurnMessageId, admissionOutcome: 'unknown' as const, payloadCompleteness: 'complete' as const, @@ -1211,29 +1507,28 @@ export function useQueuedPrompts({ reportError(error, t('queue.admissionUnknown')); return; } - const pendingAdmissionStillOwned = - pendingMidTurnAdmissionsRef.current.delete(midTurnMessageId); const known = snapshot.messages.some( (message) => message.messageId === midTurnMessageId, ) || snapshot.settledMessageIds.includes(midTurnMessageId) || snapshot.promotedMessageIds.includes(midTurnMessageId); - if (known || !pendingAdmissionStillOwned) return; - completionCallbacksRef.current.delete(midTurnMessageId); - restoreQueuedPromptsToEditor( - [ - { - id: nextQueuedPromptIdRef.current++, - sessionId: targetSessionId, - text: trimmed, - images: images ? [...images] : undefined, - payloadCompleteness: 'complete', - }, - ], - targetSessionId, + if (known) return; + if (!pendingMidTurnAdmissionsRef.current.has(midTurnMessageId)) { + return; + } + const next = queuedPromptsRef.current.map((prompt) => + prompt.midTurnMessageId === midTurnMessageId + ? { + ...prompt, + midTurnState: undefined, + admissionOutcome: 'unknown' as const, + } + : prompt, ); - reportError(error, t('queue.queueFailed')); + queuedPromptsRef.current = next; + setQueuedPrompts(next); + reportError(error, t('queue.admissionUnknown')); }); return true; } @@ -1293,6 +1588,7 @@ export function useQueuedPrompts({ return true; }, [ + canInjectMidTurnMedia, canQueryMidTurn, fallbackToPendingPrompt, reconcileMidTurnMessages, @@ -1311,8 +1607,11 @@ export function useQueuedPrompts({ // local rows still fall back to the ordinary queue at the turn boundary. useEffect(() => { if (!sessionId || midTurnInjectedBatches.length === 0) return; - for (const batch of midTurnInjectedBatches) { - if (batch.sessionId !== sessionId) continue; + const sessionBatches = midTurnInjectedBatches.filter( + (batch) => batch.sessionId === sessionId, + ); + if (sessionBatches.length === 0) return; + for (const batch of sessionBatches) { for (const messageId of batch.messageIds ?? []) { pendingMidTurnAdmissionsRef.current.delete(messageId); const callback = completionCallbacksRef.current.get(messageId); @@ -1323,7 +1622,7 @@ export function useQueuedPrompts({ const current = queuedPromptsRef.current; const next = removeInjectedFromQueue( current, - midTurnInjectedBatches, + sessionBatches, sessionId, clientId, canQueryMidTurn, @@ -1336,21 +1635,23 @@ export function useQueuedPrompts({ queuedPromptsRef.current = next; setQueuedPrompts(next); } - consumeMidTurnInjected( - midTurnInjectedBatches.filter((batch) => batch.sessionId === sessionId), - ); + consumeMidTurnInjected(sessionBatches); + // Fence an enqueue-time snapshot that may have captured the message before + // this injection, then confirm the local removal against daemon state. + if (canQueryMidTurn) void reconcileMidTurnMessages(sessionId); }, [ midTurnInjectedBatches, sessionId, clientId, canQueryMidTurn, consumeMidTurnInjected, + reconcileMidTurnMessages, ]); useEffect(() => { if (streamingState !== 'idle' || writeBlocked) return; const ctrl = midTurnEnqueueAbortRef.current; - if (ctrl) { + if (ctrl && !canQueryMidTurn) { ctrl.abort(); midTurnEnqueueAbortRef.current = null; } diff --git a/packages/web-shell/client/i18n.tsx b/packages/web-shell/client/i18n.tsx index d7a69ebdb41..a29fa8f5c8c 100644 --- a/packages/web-shell/client/i18n.tsx +++ b/packages/web-shell/client/i18n.tsx @@ -1426,7 +1426,6 @@ const EN: Messages = { 'queue.imageCount': (v) => `(+${v?.count ?? 0} images)`, 'queue.fileCount': (v) => `(+${v?.count ?? 0} files)`, 'queue.more': (v) => `... (+${v?.count ?? 0} more)`, - 'midTurn.inserted': (v) => `Inserted message: ${v?.message ?? ''}`, 'help.builtIn': 'Built-in commands', 'help.commandCount': (v) => `${v?.count ?? 0} commands`, 'help.commandMeta.builtIn': 'built-in', @@ -4353,7 +4352,6 @@ const ZH: Messages = { 'queue.imageCount': (v) => `（+${v?.count ?? 0} 张图片）`, 'queue.fileCount': (v) => `（+${v?.count ?? 0} 个文件）`, 'queue.more': (v) => `...（还有 ${v?.count ?? 0} 条）`, - 'midTurn.inserted': (v) => `已插入消息：${v?.message ?? ''}`, 'help.builtIn': '内置命令', 'help.commandCount': (v) => `${v?.count ?? 0} 个命令`, 'help.commandMeta.builtIn': '内置', diff --git a/packages/web-shell/client/midTurnDedup.test.ts b/packages/web-shell/client/midTurnDedup.test.ts index 5572ab631d7..7d3224db02b 100644 --- a/packages/web-shell/client/midTurnDedup.test.ts +++ b/packages/web-shell/client/midTurnDedup.test.ts @@ -17,6 +17,7 @@ interface Item { files?: unknown[]; midTurnState?: 'submitting' | 'queued'; midTurnMessageId?: string; + admissionOutcome?: 'unknown'; } let nextId = 1; @@ -89,19 +90,49 @@ describe('removeInjectedFromQueue', () => { ).toEqual(['other']); }); - it('never matches an image-bearing entry (images are not pushed mid-turn)', () => { + it('never matches an image-bearing entry on the TEXT fallback', () => { const prompts = [q('with image', [{ data: 'x' }]), q('with image')]; const next = removeInjectedFromQueue( prompts, [batch('s', 'with image')], 's', ); - // The text-only one is removed; the image-bearing one stays. + // The text-only one is removed; the image-bearing one stays — a text + // comparison can't verify its attachments. expect(next).not.toBeNull(); expect(next).toHaveLength(1); expect(next?.[0].images).toEqual([{ data: 'x' }]); }); + it('removes an image-bearing entry on a strict id match', () => { + const imageRow = q('with image', [{ data: 'x' }]); + const prompts = [q('keep'), imageRow]; + const next = removeInjectedFromQueue( + prompts, + [batchWithIds('s', ['with image'], [imageRow.midTurnMessageId!])], + 's', + ); + expect(next?.map((p) => p.text)).toEqual(['keep']); + }); + + it('removes an admission-unknown image row on a strict id match', () => { + // A row whose admission response was lost is re-added with + // `admissionOutcome: 'unknown'` and NO `midTurnState` (useQueuedPrompts + // catch path). When the daemon later drains it, the strict-id pass must + // still match — mirroring `applyMidTurnSnapshot`'s membership filter. + const unknown = { + ...q('lost ack', [{ data: 'x' }]), + midTurnState: undefined, + admissionOutcome: 'unknown' as const, + }; + const next = removeInjectedFromQueue( + [unknown, q('keep')], + [batchWithIds('s', ['lost ack'], [unknown.midTurnMessageId!])], + 's', + ); + expect(next?.map((p) => p.text)).toEqual(['keep']); + }); + it('never matches a file-bearing entry (files are not pushed mid-turn)', () => { const withFile = { ...q('with file'), files: [{ name: 'app.log' }] }; const prompts = [withFile, q('with file')]; diff --git a/packages/web-shell/client/midTurnDedup.ts b/packages/web-shell/client/midTurnDedup.ts index 253e1ab611c..c2b8a1829b4 100644 --- a/packages/web-shell/client/midTurnDedup.ts +++ b/packages/web-shell/client/midTurnDedup.ts @@ -10,6 +10,7 @@ export interface MidTurnQueueItem { files?: unknown[]; midTurnState?: 'submitting' | 'queued'; midTurnMessageId?: string; + admissionOutcome?: 'unknown'; } export interface MidTurnInjectedBatch { @@ -30,15 +31,16 @@ export interface MidTurnInjectedBatch { * `midTurnMessageId` match (the daemon mints an id at admission and echoes it on * injection); it wins regardless of array position, so two same-text sends can't * steal each other's removal when their admission responses arrive out of order. - * Only when no id match exists does the second pass fall back to the first - * text-only entry with matching text — any mid-turn row when the batch carries - * no ids (older daemon), or a still-`submitting` row that hasn't received its id - * yet. Matching stays count-based — one removal per injected message — so a - * queue that holds the same text twice loses one entry per matching injection. - * Entries carrying images or file attachments are never matched: attachment - * messages aren't pushed mid-turn (the drain channel carries plain strings), - * so they stay queued for the next turn. An entry that already fell back to - * the ordinary path (`midTurnState === undefined`) is never matched. + * Image rows are only matched here — the id proves the daemon owns them, while + * a text comparison can't verify their attachments. File rows are never + * matched — files are not pushed mid-turn, so they stay queued for the next + * turn. Only when no id match exists does the second pass fall back to the + * first text-only entry with matching text — any mid-turn row when the batch + * carries no ids (older daemon), or a still-`submitting` row that hasn't + * received its id yet. Matching stays count-based — one removal per injected + * message — so a queue that holds the same text twice loses one entry per + * matching injection. An entry that already fell back to the ordinary path + * (`midTurnState === undefined`) is never matched. * * Text fallback skips batches from another originator so coincidentally equal * messages are not removed. In strict-id mode, an exact id still wins because @@ -61,6 +63,7 @@ export function removeInjectedFromQueue( const isTextOnly = (prompt: T) => (!prompt.images || prompt.images.length === 0) && (!prompt.files || prompt.files.length === 0); + const hasNoFiles = (prompt: T) => !prompt.files || prompt.files.length === 0; let changed = false; for (const batch of batches) { if (batch.sessionId !== sessionId) continue; @@ -70,16 +73,23 @@ export function removeInjectedFromQueue( if (!originatorMatches && !strictMessageIds) continue; for (const [messageIndex, message] of batch.messages.entries()) { const messageId = batch.messageIds?.[messageIndex]; - // A strict id match wins regardless of position; the text fallback below - // only runs for rows the id can't reach (no ids in the batch, or a row - // still awaiting its admission id). + // A strict id match wins regardless of position — and is the only pass + // that may remove image rows; file rows are never matched because files + // are not pushed mid-turn. The text fallback below only runs for rows + // the id can't reach (no ids in the batch, or a row still awaiting its + // admission id). + // An admission-unknown row carries no `midTurnState` (re-added from + // the catch path when the admission response was lost), but the daemon + // owns it exactly like a queued one — accept it here, mirroring + // `applyMidTurnSnapshot`'s membership filter. let index = messageId !== undefined ? remaining.findIndex( (prompt) => - prompt.midTurnState !== undefined && + (prompt.midTurnState !== undefined || + prompt.admissionOutcome === 'unknown') && prompt.midTurnMessageId === messageId && - isTextOnly(prompt), + hasNoFiles(prompt), ) : -1; if (index < 0 && originatorMatches) { diff --git a/packages/webui/src/daemon/midTurnInjectedSidechannel.test.ts b/packages/webui/src/daemon/midTurnInjectedSidechannel.test.ts index 7a589278813..1a97aa46a4d 100644 --- a/packages/webui/src/daemon/midTurnInjectedSidechannel.test.ts +++ b/packages/webui/src/daemon/midTurnInjectedSidechannel.test.ts @@ -91,13 +91,105 @@ describe('parseSidechannelMidTurnInjected', () => { ).toEqual({ sessionId: 's-1', messages: ['hi'] }); }); - it('filters non-string and empty entries', () => { + it('preserves empty slots for image-only messages', () => { + // The renderability gate keeps the frame (image item), but the hydrated + // items themselves are never carried into the sidechannel payload. expect( parseSidechannelMidTurnInjected({ type: 'mid_turn_message_injected', - data: { sessionId: 's-1', messages: ['keep', '', 42, 'also'] }, + data: { + sessionId: 's-1', + messages: [''], + messageIds: ['mid-image'], + items: [ + { + content: [{ type: 'image', data: 'AQID', mimeType: 'image/png' }], + }, + ], + }, + }), + ).toEqual({ + sessionId: 's-1', + messages: [''], + messageIds: ['mid-image'], + }); + }); + + it('survives a degraded image-only echo (placeholder text block)', () => { + // The degraded-media drain publishes messages:[''] whose item holds only + // the '[Attached media is no longer available]' text block. The SDK + // normalizer renders that frame; the sidechannel must keep it too so + // dedup/callback settlement runs instead of waiting for the idle reconcile. + const placeholder = '[Attached media is no longer available]'; + expect( + parseSidechannelMidTurnInjected({ + type: 'mid_turn_message_injected', + data: { + sessionId: 's-1', + messages: [''], + messageIds: ['mid-gone'], + items: [{ content: [{ type: 'text', text: placeholder }] }], + }, + }), + ).toEqual({ + sessionId: 's-1', + messages: [''], + messageIds: ['mid-gone'], + }); + }); + + it('survives a misaligned renderable frame exactly like the SDK normalizer', () => { + // The SDK normalizer's hasRenderableItemContent evaluates the RAW items + // array with no index-alignment precondition. The parser must agree, or + // the two consumers of the same frame disagree: the normalizer renders + // the echo while the queued row's callback waits for the turn-end idle + // reconcile. + expect( + parseSidechannelMidTurnInjected({ + type: 'mid_turn_message_injected', + data: { + sessionId: 's-1', + messages: [''], + items: [ + { + content: [{ type: 'image', data: 'AQID', mimeType: 'image/png' }], + }, + { + content: [{ type: 'image', data: 'BAUG', mimeType: 'image/png' }], + }, + ], + }, + }), + ).toEqual({ sessionId: 's-1', messages: [''] }); + }); + + it('never carries hydrated items into the payload, even when aligned', () => { + // No consumer reads `items` from a sidechannel batch (dedup settles by + // messageIds and removes rows by messages); multi-MB hydrated base64 + // payloads must not ride the buffer unread. + const parsed = parseSidechannelMidTurnInjected({ + type: 'mid_turn_message_injected', + data: { + sessionId: 's-1', + messages: ['first'], + items: [ + { + content: [{ type: 'image', data: 'AQID', mimeType: 'image/png' }], + }, + ], + }, + }); + expect(parsed).toEqual({ sessionId: 's-1', messages: ['first'] }); + expect(parsed).not.toHaveProperty('items'); + }); + + it('rejects non-string message entries to preserve index alignment', () => { + expect( + parseSidechannelMidTurnInjected({ + type: 'mid_turn_message_injected', + data: { sessionId: 's-1', messages: ['keep', 42] }, }), - ).toEqual({ sessionId: 's-1', messages: ['keep', 'also'] }); + ).toBeUndefined(); }); it('returns undefined for wrong type, missing data, or no usable messages', () => { @@ -187,6 +279,24 @@ describe('mid-turn injected sidechannel pub/sub', () => { ]); }); + it('does not buffer hydrated items on publish', () => { + publishSidechannelMidTurnInjected({ + sessionId: 's-1', + messages: ['a'], + items: [ + { + content: [ + { type: 'image' as const, data: 'aW1n', mimeType: 'image/png' }, + ], + }, + ], + }); + expect(getSidechannelMidTurnInjected()).toEqual([ + { sessionId: 's-1', messages: ['a'] }, + ]); + expect(getSidechannelMidTurnInjected()[0]).not.toHaveProperty('items'); + }); + it('consume removes only the handled batches, leaving a later-arrived frame (race)', () => { // Models the render→effect race: the consumer reconciles `snapshot`, then a // new frame appends before its consume runs. Identity-removal must drop only diff --git a/packages/webui/src/daemon/midTurnInjectedSidechannel.ts b/packages/webui/src/daemon/midTurnInjectedSidechannel.ts index e3a06ddfdb8..e3f8174a71e 100644 --- a/packages/webui/src/daemon/midTurnInjectedSidechannel.ts +++ b/packages/webui/src/daemon/midTurnInjectedSidechannel.ts @@ -124,8 +124,8 @@ function notifyMidTurnInjectedListeners(): void { /** * Parse a raw daemon SSE frame into the injected-messages payload, or * `undefined` if the frame is not a well-formed `mid_turn_message_injected` - * event. Filters out non-string / empty entries; returns `undefined` when - * nothing usable remains. + * event. Empty text slots are preserved so `messages` and `messageIds` + * remain index-aligned for image-only messages. */ export function parseSidechannelMidTurnInjected( event: unknown, @@ -141,16 +141,43 @@ export function parseSidechannelMidTurnInjected( if (typeof sessionId !== 'string' || !Array.isArray(messages)) { return undefined; } - const stringMessages = messages.filter( - (message): message is string => - typeof message === 'string' && message.length > 0, - ); - if (stringMessages.length === 0) return undefined; + if (!messages.every((message) => typeof message === 'string')) { + return undefined; + } + const stringMessages = messages as string[]; + // Mirror the SDK normalizer's `hasRenderableItemContent`: a frame survives + // when any item carries an image block OR a non-empty text block. The + // degraded-media drain publishes `messages: ['']` whose items hold only the + // '[Attached media is no longer available]' text block — dropping it here + // while the normalizer renders it leaves the queued row unsettled until the + // turn-end idle reconcile. Evaluate over the RAW items array exactly like + // the normalizer (no index-alignment precondition), so the two consumers + // of the same frame can never disagree. + const items = dataRecord['items']; + const hasRenderableContent = + Array.isArray(items) && + items.some( + (item) => + !!item && + typeof item === 'object' && + Array.isArray((item as Record)['content']) && + ((item as Record)['content'] as unknown[]).some( + (block) => + !!block && + typeof block === 'object' && + ((block as Record)['type'] === 'image' || + ((block as Record)['type'] === 'text' && + typeof (block as Record)['text'] === + 'string' && + ((block as Record)['text'] as string).length > + 0)), + ), + ); + if (!stringMessages.some(Boolean) && !hasRenderableContent) return undefined; const messageIds = dataRecord['messageIds']; const stringMessageIds = Array.isArray(messageIds) && messageIds.length === messages.length && - stringMessages.length === messages.length && messageIds.every( (messageId) => typeof messageId === 'string' && messageId.length > 0, ) @@ -159,6 +186,11 @@ export function parseSidechannelMidTurnInjected( // Older daemons put `originatorClientId` on the SSE envelope. New clients // use it only in the capability fallback for those daemons. const originatorClientId = record['originatorClientId']; + // `items` (hydrated inline base64 image content) is deliberately NOT + // carried into the sidechannel: no consumer reads it (dedup settles by + // `messageIds` and removes rows by `messages`), and multi-MB payloads + // would otherwise ride parser → buffer → consume unread. The transcript + // renders from the hydrated event stream directly. return { sessionId, messages: stringMessages, diff --git a/packages/webui/src/daemon/session/DaemonSessionProvider.test.tsx b/packages/webui/src/daemon/session/DaemonSessionProvider.test.tsx index 6b93cabdefb..3506ef0632a 100644 --- a/packages/webui/src/daemon/session/DaemonSessionProvider.test.tsx +++ b/packages/webui/src/daemon/session/DaemonSessionProvider.test.tsx @@ -96,6 +96,13 @@ interface MockSession { updateMetadata: (metadata: { displayName?: string; }) => Promise<{ displayName?: string }>; + getTranscriptPage: (opts: unknown) => Promise<{ + events: DaemonEvent[]; + hasMore: boolean; + nextCursor?: string; + partial?: true; + replayError?: string; + }>; replaySnapshot: { compactedReplay: DaemonEvent[]; liveJournal: DaemonEvent[]; @@ -1554,7 +1561,7 @@ describe('DaemonSessionProvider', () => { expect(blocks).toMatchObject([ { kind: 'status', - text: 'Inserted message: also check the tests', + text: 'also check the tests', source: 'mid_turn_message_injected', data: { sessionId: 'mt-session', @@ -12373,7 +12380,7 @@ function createTextReplaySnapshot(text: string): MockSession['replaySnapshot'] { } function createMockSession(opts: Partial = {}): MockSession { - const session = { + const session: MockSession = { sessionId: opts.sessionId ?? 'session-1', workspaceCwd: opts.workspaceCwd ?? '/mock-workspace', clientId: opts.clientId ?? 'client-1', @@ -12438,6 +12445,21 @@ function createMockSession(opts: Partial = {}): MockSession { updateMetadata: opts.updateMetadata ?? vi.fn(async (metadata: { displayName?: string }) => metadata), + getTranscriptPage: + opts.getTranscriptPage ?? + vi.fn(async (pageOpts: unknown) => { + if (!session.client) throw new Error('Session client is unavailable'); + return (await session.client.getSessionTranscriptPage( + session.sessionId, + pageOpts, + )) as { + events: DaemonEvent[]; + hasMore: boolean; + nextCursor?: string; + partial?: true; + replayError?: string; + }; + }), replaySnapshot: opts.replaySnapshot ?? { compactedReplay: [], liveJournal: [], diff --git a/packages/webui/src/daemon/session/DaemonSessionProvider.tsx b/packages/webui/src/daemon/session/DaemonSessionProvider.tsx index a3df97d06a9..8dd1402f3e8 100644 --- a/packages/webui/src/daemon/session/DaemonSessionProvider.tsx +++ b/packages/webui/src/daemon/session/DaemonSessionProvider.tsx @@ -3067,18 +3067,15 @@ export function DaemonSessionProvider(props: DaemonSessionProviderProps) { }); let terminalFailure = false; try { - const page = await activeSession.client.getSessionTranscriptPage( - activeSession.sessionId, - { - ...(history.cursor !== undefined - ? { cursor: history.cursor } - : history.beforeRecordId !== undefined - ? { beforeRecordId: history.beforeRecordId } - : {}), - limit: historyPageSizeRef.current ?? 100, - clientId: activeSession.clientId, - }, - ); + const page = await activeSession.getTranscriptPage({ + ...(history.cursor !== undefined + ? { cursor: history.cursor } + : history.beforeRecordId !== undefined + ? { beforeRecordId: history.beforeRecordId } + : {}), + limit: historyPageSizeRef.current ?? 100, + clientId: activeSession.clientId, + }); if ( sessionRef.current !== activeSession || transcriptHistoryRef.current !== history diff --git a/packages/webui/src/daemon/session/actions.test.ts b/packages/webui/src/daemon/session/actions.test.ts index 272fdd50a5c..9e587e39cb3 100644 --- a/packages/webui/src/daemon/session/actions.test.ts +++ b/packages/webui/src/daemon/session/actions.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { DaemonHttpError, + DaemonPendingPromptLimitError, type DaemonCapabilities, type DaemonSessionClient, } from '@qwen-code/sdk/daemon'; @@ -1037,6 +1038,449 @@ describe('createDaemonSessionActions', () => { expect(session.submitPrompt).not.toHaveBeenCalled(); }); + it('uploads prompt images and submits media references instead of base64', async () => { + const session = createMockSession('session-a'); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await actions.submitPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/png' }], + }); + + expect(session.uploadMedia).toHaveBeenCalledWith( + expect.any(Blob), + 'image/png', + undefined, + ); + expect(session.submitPrompt).toHaveBeenCalledWith({ + prompt: [ + { type: 'text', text: 'look' }, + { + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }, + ], + }); + }); + + it('keeps images without a concrete mime type inline instead of uploading', async () => { + const session = createMockSession('session-a'); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await actions.submitPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/*' }], + }); + + // The media route matches concrete image types only; uploading a literal + // image/* Content-Type 400s, so such images must stay inline (untyped, + // matching the legacy shape). + expect(session.uploadMedia).not.toHaveBeenCalled(); + expect(session.submitPrompt).toHaveBeenCalledWith({ + prompt: [ + { type: 'text', text: 'look' }, + { type: 'image', data: 'AQID' }, + ], + }); + }); + + it('does not mark admission started when media upload fails', async () => { + const onAdmissionStarted = vi.fn(); + const session = createMockSession('session-a'); + session.uploadMedia.mockRejectedValueOnce(new Error('upload failed')); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await expect( + actions.sendPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/png' }], + onAdmissionStarted, + }), + ).rejects.toThrow('upload failed'); + + expect(onAdmissionStarted).not.toHaveBeenCalled(); + expect(session.submitPrompt).not.toHaveBeenCalled(); + }); + + it('falls back to inline image data when the media route is unavailable', async () => { + const session = createMockSession('session-a'); + session.uploadMedia.mockRejectedValueOnce( + new DaemonHttpError(404, undefined, 'Not found'), + ); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await actions.submitPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/png' }], + }); + + expect(session.submitPrompt).toHaveBeenCalledWith({ + prompt: [ + { type: 'text', text: 'look' }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + }); + }); + + it('removes successful media uploads when another upload fails', async () => { + const session = createMockSession('session-a'); + session.uploadMedia + .mockResolvedValueOnce({ + type: 'image', + mediaId: 'uploaded-before-failure', + mimeType: 'image/png', + size: 3, + }) + .mockRejectedValueOnce(new Error('second upload failed')); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await expect( + actions.submitPrompt('look', { + images: [ + { data: 'AQID', mimeType: 'image/png' }, + { data: 'BAUG', mimeType: 'image/png' }, + ], + }), + ).rejects.toThrow('second upload failed'); + + expect(session.removeMedia).toHaveBeenCalledWith('uploaded-before-failure'); + expect(session.submitPrompt).not.toHaveBeenCalled(); + }); + + it('removes uploaded media when prompt admission is rejected', async () => { + const session = createMockSession('session-a'); + session.submitPrompt.mockRejectedValueOnce( + new DaemonPendingPromptLimitError('session-a', 20, 20), + ); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await expect( + actions.sendPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/png' }], + }), + ).rejects.toThrow('Pending prompts full'); + + expect(session.removeMedia).toHaveBeenCalledWith('media-1'); + }); + + it('keeps uploaded media when prompt admission is uncertain', async () => { + const session = createMockSession('session-a'); + session.submitPrompt.mockRejectedValueOnce(new TypeError('fetch failed')); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await expect( + actions.submitPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/png' }], + }), + ).rejects.toThrow('fetch failed'); + + expect(session.removeMedia).not.toHaveBeenCalled(); + }); + + it('removes uploaded media when cancelled before prompt admission', async () => { + const upload = createDeferred<{ + type: 'image'; + mediaId: string; + mimeType: string; + size: number; + }>(); + const session = createMockSession('session-a'); + session.uploadMedia.mockReturnValueOnce(upload.promise); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + const prompt = actions.sendPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/png' }], + }); + await vi.waitFor(() => expect(session.uploadMedia).toHaveBeenCalled()); + await actions.cancel(); + upload.resolve({ + type: 'image', + mediaId: 'media-1', + mimeType: 'image/png', + size: 3, + }); + + await expect(prompt).resolves.toEqual({ stopReason: 'cancelled' }); + expect(session.removeMedia).toHaveBeenCalledWith('media-1'); + expect(session.submitPrompt).not.toHaveBeenCalled(); + }); + + it('removes uploaded media when an admitted pending prompt is removed', async () => { + const controller = new AbortController(); + const session = createMockSession('session-a'); + session.submitPrompt.mockImplementationOnce(async () => { + controller.abort(); + return { promptId: 'prompt-1' }; + }); + session.removePendingPrompt.mockResolvedValueOnce({ removed: true }); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await expect( + actions.submitPrompt('look', { + signal: controller.signal, + images: [{ data: 'AQID', mimeType: 'image/png' }], + }), + ).resolves.toEqual({ promptId: 'prompt-1', removedAfterAbort: true }); + + expect(session.removeMedia).toHaveBeenCalledWith('media-1'); + }); + + it('keeps uploaded media when the admitted prompt already started', async () => { + const controller = new AbortController(); + const session = createMockSession('session-a'); + session.submitPrompt.mockImplementationOnce(async () => { + controller.abort(); + return { promptId: 'prompt-1' }; + }); + session.removePendingPrompt.mockResolvedValueOnce({ removed: false }); + const { actions } = createActionsHarness({ + session, + connection: { + status: 'connected', + workspaceCwd: '/workspace', + capabilities: { + v: 1, + mode: 'http-bridge', + features: ['session_media'], + modelServices: [], + }, + }, + }); + + await expect( + actions.submitPrompt('look', { + signal: controller.signal, + images: [{ data: 'AQID', mimeType: 'image/png' }], + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + + expect(session.removeMedia).not.toHaveBeenCalled(); + }); + + it('keeps prompt images inline for an older daemon', async () => { + const session = createMockSession('session-a'); + const { actions } = createActionsHarness({ session }); + + await actions.submitPrompt('look', { + images: [{ data: 'AQID', mimeType: 'image/png' }], + }); + + expect(session.uploadMedia).not.toHaveBeenCalled(); + expect(session.submitPrompt).toHaveBeenCalledWith({ + prompt: [ + { type: 'text', text: 'look' }, + { + type: 'image', + data: 'AQID', + mimeType: 'image/png', + }, + ], + }); + }); + + it('removes an orphaned upload from its original session after a switch', async () => { + const session = createMockSession('session-current', 'client-current'); + const { actions } = createActionsHarness({ session }); + + await expect( + actions.removeMedia('media-old', { sessionId: 'session-old' }), + ).resolves.toBe(true); + + expect(session.removeMedia).not.toHaveBeenCalled(); + expect(session.client.removeSessionMedia).toHaveBeenCalledWith( + 'session-old', + 'media-old', + ); + }); + + it('removes an orphaned upload after the active session is cleared', async () => { + const session = createMockSession('session-old', 'client-old'); + const { actions, sessionRef } = createActionsHarness({ session }); + await actions.uploadMedia({ data: 'AQID', mimeType: 'image/png' }); + sessionRef.current = undefined; + + await expect( + actions.removeMedia('media-old', { sessionId: 'session-old' }), + ).resolves.toBe(true); + + expect(session.client.removeSessionMedia).toHaveBeenCalledWith( + 'session-old', + 'media-old', + { clientId: 'client-old' }, + ); + }); + + it('uses the target session client id when removing old media', async () => { + vi.stubGlobal('window', { + sessionStorage: { + getItem: vi.fn(() => 'client-old'), + }, + }); + try { + const session = createMockSession('session-current', 'client-current'); + const { actions } = createActionsHarness({ session }); + + await actions.removeMedia('media-old', { sessionId: 'session-old' }); + + expect(session.client.removeSessionMedia).toHaveBeenCalledWith( + 'session-old', + 'media-old', + { clientId: 'client-old' }, + ); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('retries cross-session media removal without the client id when it is stale', async () => { + // Detach unregisters the persisted client id on the daemon; the cleanup + // must degrade to a no-clientId removal instead of orphaning the media. + vi.stubGlobal('window', { + sessionStorage: { + getItem: vi.fn(() => 'client-old'), + }, + }); + try { + const session = createMockSession('session-current', 'client-current'); + session.client.removeSessionMedia = vi + .fn() + .mockRejectedValueOnce( + new DaemonHttpError( + 400, + { code: 'invalid_client_id' }, + 'invalid client id', + ), + ) + .mockResolvedValueOnce(true); + const { actions } = createActionsHarness({ session }); + + await expect( + actions.removeMedia('media-old', { sessionId: 'session-old' }), + ).resolves.toBe(true); + + expect(session.client.removeSessionMedia).toHaveBeenCalledTimes(2); + expect(session.client.removeSessionMedia).toHaveBeenNthCalledWith( + 1, + 'session-old', + 'media-old', + { clientId: 'client-old' }, + ); + expect(session.client.removeSessionMedia).toHaveBeenNthCalledWith( + 2, + 'session-old', + 'media-old', + ); + } finally { + vi.unstubAllGlobals(); + } + }); + it('does not restart the event stream when the admitted prompt is stale', async () => { const restartEventStream = vi.fn(); const session = createMockSession('session-a'); @@ -1487,11 +1931,20 @@ function createMockSession( })), listWorkspaceSessions: vi.fn(), closeSession: vi.fn(), + removeSessionMedia: vi.fn(async () => true), }, cancel: vi.fn(async () => undefined), context: vi.fn(async () => contextStatus(sessionId)), detach: vi.fn(async () => undefined), setModel: vi.fn(async () => ({})), + uploadMedia: vi.fn(async (_data: Blob, mimeType: string) => ({ + type: 'image' as const, + mediaId: 'media-1', + mimeType, + size: 3, + })), + removeMedia: vi.fn(async () => true), + removePendingPrompt: vi.fn(async () => ({ removed: true })), submitPrompt: vi.fn(async () => ({ promptId: 'prompt-1' })), supportedCommands: vi.fn(async () => supportedCommandsStatus(sessionId)), tasks: vi.fn(async () => ({ v: 1 as const, sessionId, tasks: [] })), diff --git a/packages/webui/src/daemon/session/actions.ts b/packages/webui/src/daemon/session/actions.ts index 142ff895799..16850f3fb9e 100644 --- a/packages/webui/src/daemon/session/actions.ts +++ b/packages/webui/src/daemon/session/actions.ts @@ -26,20 +26,27 @@ import type { DaemonCapabilities, DaemonBranchSessionResult, DaemonBranchedSession, + DaemonSessionMediaReference, PermissionResponse, + PromptContentBlock, } from '@qwen-code/sdk/daemon'; import { + DaemonHttpError, + DaemonPendingPromptLimitError, isDaemonTurnError, isStaleBranchPointError, type PromptResult, } from '@qwen-code/sdk/daemon'; -import { extractHttpStatus } from './httpErrors.js'; +import { extractHttpStatus, isInvalidClientIdError } from './httpErrors.js'; import { mapReasoningControls, mapSessionContextReasoning, mapSupportedCommands, } from './mappers.js'; -import { toDaemonPromptContent } from './promptContent.js'; +import { + daemonPromptImageToBlob, + toDaemonPromptContent, +} from './promptContent.js'; import { clearPassiveAssistantDoneTimer, withActionTimeout, @@ -213,6 +220,9 @@ export function createDaemonSessionActions({ clearLiveJournalRepair = () => undefined, }: CreateDaemonSessionActionsArgs): DaemonSessionActions { const silentHardFailureNoticeKeys = new Set(); + let mediaClient = sessionRef.current?.client; + let mediaClientId = sessionRef.current?.clientId; + let mediaClientSessionId = sessionRef.current?.sessionId; let noticeOwner = sessionRef.current; let reasoningActionToken = 0; let appliedReasoningActionToken = 0; @@ -241,6 +251,78 @@ export function createDaemonSessionActions({ ); } + async function promptContentWithUploadedMedia( + session: DaemonSessionClient, + text: string, + images: ReadonlyArray<{ data: string; mimeType: string }>, + files: readonly DaemonPromptFile[], + signal?: AbortSignal, + ): Promise<{ + content: PromptContentBlock[]; + references: DaemonSessionMediaReference[]; + }> { + const supportsMediaUpload = + getConnection().capabilities?.features.includes('session_media') === true; + // The media route matches concrete image types only; a literal image/* + // Content-Type 400s, so untyped images stay inline as before the upload + // path existed. + if ( + images.length === 0 || + !supportsMediaUpload || + images.some((image) => image.mimeType === 'image/*') || + typeof session.uploadMedia !== 'function' + ) { + return { + content: toDaemonPromptContent(text, images, files), + references: [], + }; + } + const results = await Promise.allSettled( + images.map( + async (image) => + await session.uploadMedia( + daemonPromptImageToBlob(image), + image.mimeType, + signal, + ), + ), + ); + const references = results.flatMap((result) => + result.status === 'fulfilled' ? [result.value] : [], + ); + const failure = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected', + ); + if (!failure) { + const content = toDaemonPromptContent(text, [], files); + content.splice(1, 0, ...references); + return { content, references }; + } + await removeUploadedMedia(session, references); + if (extractHttpStatus(failure.reason) === 404) { + return { + content: toDaemonPromptContent(text, images, files), + references: [], + }; + } + throw failure.reason; + } + + async function removeUploadedMedia( + session: DaemonSessionClient, + references: readonly DaemonSessionMediaReference[], + ): Promise { + await Promise.allSettled( + references.map( + async (reference) => await session.removeMedia(reference.mediaId), + ), + ); + } + + const isDefinitePromptAdmissionRejection = (error: unknown) => + error instanceof DaemonHttpError || + error instanceof DaemonPendingPromptLimitError; + const ignoreStaleNotice: AddDaemonSessionNotice = (notice) => ({ ...notice, id: notice.id ?? 'stale-session-notice', @@ -515,24 +597,39 @@ export function createDaemonSessionActions({ normalizedFiles, ); } + const uploaded = await promptContentWithUploadedMedia( + session, + text, + normalizedImages, + normalizedFiles, + ctrl.signal, + ); + if (ctrl.signal.aborted) { + await removeUploadedMedia(session, uploaded.references); + ctrl.signal.throwIfAborted(); + } const promptRequest: Record = { - prompt: toDaemonPromptContent( - text, - normalizedImages, - normalizedFiles, - ), + prompt: uploaded.content, }; + options?.onAdmissionStarted?.(); if (inputAnnotations) { promptRequest['_meta'] = { inputAnnotations }; } if (options?.retry) { promptRequest['retry'] = true; } - options?.onAdmissionStarted?.(); - const accepted = await session.submitPrompt( - promptRequest as Parameters[0], - ctrl.signal, - ); + let accepted: Awaited>; + try { + accepted = await session.submitPrompt( + promptRequest as Parameters[0], + ctrl.signal, + ); + } catch (error) { + if (isDefinitePromptAdmissionRejection(error)) { + await removeUploadedMedia(session, uploaded.references); + } + throw error; + } if (activePromptsRef.current.get(sessionId)?.controller === ctrl) { restartEventStream(sessionId); } @@ -608,8 +705,15 @@ export function createDaemonSessionActions({ normalizedFiles, ); } + const uploaded = await promptContentWithUploadedMedia( + session, + text, + normalizedImages, + normalizedFiles, + options?.signal, + ); const promptRequest: Record = { - prompt: toDaemonPromptContent(text, normalizedImages, normalizedFiles), + prompt: uploaded.content, }; if (inputAnnotations) { promptRequest['_meta'] = { inputAnnotations }; @@ -617,13 +721,22 @@ export function createDaemonSessionActions({ if (options?.retry) { promptRequest['retry'] = true; } - const accepted = await session.submitPrompt( - promptRequest as Parameters[0], - ); + let accepted: Awaited>; + try { + accepted = await session.submitPrompt( + promptRequest as Parameters[0], + ); + } catch (error) { + if (isDefinitePromptAdmissionRejection(error)) { + await removeUploadedMedia(session, uploaded.references); + } + throw error; + } if (options?.signal?.aborted) { try { const removal = await session.removePendingPrompt(accepted.promptId); if (removal.removed) { + await removeUploadedMedia(session, uploaded.references); return { promptId: accepted.promptId, removedAfterAbort: true }; } } catch (err) { @@ -1396,9 +1509,62 @@ export function createDaemonSessionActions({ } }, + async uploadMedia(image, opts) { + const session = requireSessionForAction( + addNotice, + sessionRef.current, + 'Media upload failed', + 'send_prompt', + ); + const mimeType = + image.mimeType ?? image.mediaType ?? image.media_type ?? 'image/*'; + mediaClient = session.client; + mediaClientId = session.clientId; + mediaClientSessionId = session.sessionId; + return await session.uploadMedia( + daemonPromptImageToBlob(image), + mimeType, + opts?.signal, + ); + }, + + async removeMedia(mediaId, opts) { + const session = sessionRef.current; + if (opts?.sessionId && session?.sessionId !== opts.sessionId) { + const client = session?.client ?? mediaClient; + if (!client) return false; + const targetClientId = + getPersistedClientId(opts.sessionId) ?? + (mediaClientSessionId === opts.sessionId ? mediaClientId : undefined); + if (!targetClientId) { + return await client.removeSessionMedia(opts.sessionId, mediaId); + } + try { + return await client.removeSessionMedia(opts.sessionId, mediaId, { + clientId: targetClientId, + }); + } catch (error) { + // Detach unregisters the persisted client id on the daemon, so a + // session switch can stale it before this cleanup lands. The daemon + // accepts an absent id — retry without it instead of orphaning the + // media behind a swallowed 400. + if (isInvalidClientIdError(error)) { + return await client.removeSessionMedia(opts.sessionId, mediaId); + } + throw error; + } + } + if (!session) return false; + return await session.removeMedia(mediaId); + }, + async enqueueMidTurnMessage( message: string, - opts?: { signal?: AbortSignal; messageId?: string }, + opts?: { + signal?: AbortSignal; + messageId?: string; + content?: PromptContentBlock[]; + }, ): Promise { // Calls without an id are the old-daemon compatibility path and fall back // locally. With a stable id, transport failure is ambiguous (the POST may diff --git a/packages/webui/src/daemon/session/httpErrors.ts b/packages/webui/src/daemon/session/httpErrors.ts index ffbdfa0dde5..bee385edaaa 100644 --- a/packages/webui/src/daemon/session/httpErrors.ts +++ b/packages/webui/src/daemon/session/httpErrors.ts @@ -14,6 +14,15 @@ export function extractHttpStatus(error: unknown): number | undefined { return undefined; } +export function isInvalidClientIdError(error: unknown): boolean { + return ( + error instanceof DaemonHttpError && + error.status === 400 && + isRecord(error.body) && + error.body['code'] === 'invalid_client_id' + ); +} + export function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/packages/webui/src/daemon/session/promptContent.test.ts b/packages/webui/src/daemon/session/promptContent.test.ts index 6f50aced072..c09e039e5ef 100644 --- a/packages/webui/src/daemon/session/promptContent.test.ts +++ b/packages/webui/src/daemon/session/promptContent.test.ts @@ -5,7 +5,36 @@ */ import { describe, expect, it } from 'vitest'; -import { toDaemonPromptContent } from './promptContent.js'; +import { + daemonPromptImageToBlob, + toDaemonPromptContent, +} from './promptContent.js'; + +describe('daemonPromptImageToBlob', () => { + it('decodes raw base64 image data', async () => { + const blob = daemonPromptImageToBlob({ + data: 'AQID', + mimeType: 'image/png', + }); + + expect(blob.type).toBe('image/png'); + expect(new Uint8Array(await blob.arrayBuffer())).toEqual( + Uint8Array.of(1, 2, 3), + ); + }); + + it('strips a data URI prefix before decoding', async () => { + const blob = daemonPromptImageToBlob({ + data: 'data:image/jpeg;base64,BAUG', + media_type: 'image/jpeg', + }); + + expect(blob.type).toBe('image/jpeg'); + expect(new Uint8Array(await blob.arrayBuffer())).toEqual( + Uint8Array.of(4, 5, 6), + ); + }); +}); describe('toDaemonPromptContent', () => { it('keeps text prompts as the first daemon content block', () => { diff --git a/packages/webui/src/daemon/session/promptContent.ts b/packages/webui/src/daemon/session/promptContent.ts index 36d6f5dec82..9bdff0f69e7 100644 --- a/packages/webui/src/daemon/session/promptContent.ts +++ b/packages/webui/src/daemon/session/promptContent.ts @@ -11,6 +11,20 @@ export function attachmentUriForName(name: string): string { return `attachment:///${name}`; } +export function daemonPromptImageToBlob(image: DaemonPromptImage): Blob { + const comma = image.data.indexOf(','); + const encoded = + image.data.startsWith('data:') && comma >= 0 + ? image.data.slice(comma + 1) + : image.data; + const binary = atob(encoded); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i); + return new Blob([bytes], { + type: image.mimeType ?? image.mediaType ?? image.media_type ?? 'image/*', + }); +} + export function toDaemonPromptContent( text: string, images: readonly DaemonPromptImage[] = [], diff --git a/packages/webui/src/daemon/session/types.ts b/packages/webui/src/daemon/session/types.ts index 709c41aabb9..225c7484d79 100644 --- a/packages/webui/src/daemon/session/types.ts +++ b/packages/webui/src/daemon/session/types.ts @@ -15,6 +15,7 @@ import type { DaemonInputAnnotation, DaemonSessionBtwResult, DaemonSessionGenerationEvent, + DaemonSessionMediaReference, DaemonMidTurnMessageResult, DaemonMidTurnMessagesResult, DaemonRemoveMidTurnMessageResult, @@ -39,6 +40,7 @@ import type { DaemonWorkspaceProvidersStatus, HeartbeatResult, PermissionResponse, + PromptContentBlock, PromptResult, SessionMetadataResult, SetModelResult, @@ -440,14 +442,25 @@ export interface DaemonSessionActions { question: string, opts?: { signal?: AbortSignal }, ): Promise; + uploadMedia( + image: DaemonPromptImage, + opts?: { signal?: AbortSignal }, + ): Promise; + removeMedia(mediaId: string, opts?: { sessionId?: string }): Promise; /** * Queue a message typed while a turn is running. Calls without an id support * old daemons and are best-effort; calls with a stable `messageId` may reject - * on an ambiguous transport failure so the caller can reconcile. + * on an ambiguous transport failure so the caller can reconcile. `content` + * carries image blocks — pre-flight the daemon's + * `session_media` capability before attaching them. */ enqueueMidTurnMessage( message: string, - opts?: { signal?: AbortSignal; messageId?: string }, + opts?: { + signal?: AbortSignal; + messageId?: string; + content?: PromptContentBlock[]; + }, ): Promise; removeMidTurnMessage( messageId: string,