fix(core): isolate image payload eviction state

packages/core/src/core/geminiChat.test.ts

@@ -2584,6 +2584,47 @@ describe('GeminiChat', async () => {
       });
     });
 
+    it('reattaches stored image markers on later below-threshold requests', async () => {
+      vi.mocked(mockConfig.getChatCompression).mockReturnValue({
+        maxRecentImagesToRetain: 1,
+        imagePayloadThreshold: 1,
+      });
+      chat.setHistory([
+        {
+          role: 'user',
+          parts: [{ inlineData: { mimeType: 'image/png', data: 'old-shot' } }],
+        },
+        { role: 'model', parts: [{ text: 'I see the image' }] },
+      ]);
+      vi.mocked(mockContentGenerator.generateContentStream).mockImplementation(
+        async () => streamResponse(stopResponse([{ text: 'response' }])),
+      );
+
+      for (const [message, promptId] of [
+        ['first question', 'prompt-id-image-refs-first'],
+        ['second question', 'prompt-id-image-refs-second'],
+      ] as const) {
+        const stream = await chat.sendMessageStream(
+          'test-model',
+          { message },
+          promptId,
+        );
+        for await (const _ of stream) {
+          // consume stream
+        }
+      }
+
+      const durable = JSON.stringify(chat.getHistory());
+      expect(durable).toMatch(/Image #[a-f0-9]{12}/);
+      expect(durable).not.toContain('"data":"old-shot"');
+      const secondRequest = vi.mocked(
+        mockContentGenerator.generateContentStream,
+      ).mock.calls[1]?.[0];
+      expect(JSON.stringify(secondRequest?.contents)).toContain(
+        '"data":"old-shot"',
+      );
+    });
+
     it('coalesces startup reminders with the first user prompt for provider requests', async () => {
       chat.setHistory([
         {
@@ -5687,19 +5728,29 @@ describe('GeminiChat', async () => {
   });
 
   describe('getHistoryShallow', () => {
-    it('copies containers without structured-cloning large part payloads', () => {
+    it('copies Part containers without cloning large leaf payloads', () => {
       const payload = { output: 'x'.repeat(128 * 1024) };
+      const topLevelInlineData = {
+        mimeType: 'image/png',
+        data: 'top-level-image',
+      };
+      const nestedInlineData = {
+        mimeType: 'image/png',
+        data: 'nested-image',
+      };
+      const topLevelPart: Part = { inlineData: topLevelInlineData };
+      const nestedPart: Part = { inlineData: nestedInlineData };
+      const functionResponsePart: Part = {
+        functionResponse: {
+          id: 'call-1',
+          name: 'read_file',
+          response: payload,
+          parts: [nestedPart],
+        },
+      };
       const content: Content = {
         role: 'user',
-        parts: [
-          {
-            functionResponse: {
-              id: 'call-1',
-              name: 'read_file',
-              response: payload,
-            },
-          },
-        ],
+        parts: [topLevelPart, functionResponsePart],
       };
       chat.addHistory(content);
       const structuredCloneSpy = vi
@@ -5714,7 +5765,25 @@ describe('GeminiChat', async () => {
       expect(history).toEqual([content]);
       expect(history[0]).not.toBe(content);
       expect(history[0]!.parts).not.toBe(content.parts);
-      const response = history[0]!.parts![0] as {
+      expect(history[0]!.parts![0]).not.toBe(topLevelPart);
+      expect(history[0]!.parts![0]!.inlineData).toBe(topLevelInlineData);
+      const copiedFunctionResponsePart = history[0]!.parts![1]!;
+      expect(copiedFunctionResponsePart).not.toBe(functionResponsePart);
+      expect(copiedFunctionResponsePart.functionResponse).not.toBe(
+        functionResponsePart.functionResponse,
+      );
+      const copiedNested = copiedFunctionResponsePart.functionResponse
+        ?.parts as Part[];
+      expect(copiedNested).not.toBe(
+        functionResponsePart.functionResponse?.parts,
+      );
+      expect(copiedNested[0]).not.toBe(nestedPart);
+      expect(copiedNested[0]!.inlineData).toBe(nestedInlineData);
+      delete history[0]!.parts![0]!.inlineData;
+      delete copiedNested[0]!.inlineData;
+      expect(topLevelPart.inlineData).toBe(topLevelInlineData);
+      expect(nestedPart.inlineData).toBe(nestedInlineData);
+      const response = copiedFunctionResponsePart as {
         functionResponse: { response: typeof payload };
       };
       expect(response.functionResponse.response).toBe(payload);

packages/core/src/core/geminiChat.ts

@@ -77,6 +77,7 @@ import {
 } from '../services/chatCompressionService.js';
 import { acquireSleepInhibitor } from '../services/sleepInhibitor.js';
 import {
+  getFunctionResponseParts,
   resolveCompactionTuning,
   resolveSlimmingConfig,
   slimCompactionInput,
@@ -1233,7 +1234,19 @@ function appendCuratedContent(
 function copyContentContainer(content: Content): Content {
   return {
     ...content,
-    ...(content.parts ? { parts: [...content.parts] } : {}),
+    ...(content.parts ? { parts: content.parts.map(copyPartContainer) } : {}),
+  };
+}
+
+function copyPartContainer(part: Part): Part {
+  const nested = getFunctionResponseParts(part);
+  if (!nested) return { ...part };
+  return {
+    ...part,
+    functionResponse: {
+      ...part.functionResponse,
+      parts: nested.map((inner) => ({ ...inner })),
+    },
   };
 }
 
@@ -1876,6 +1889,7 @@ export class GeminiChat {
     const { maxRecentImages, imagePayloadThreshold } = resolveCompactionTuning(
       this.config.getChatCompression(),
     );
+    let replaced: ReturnType<typeof replaceImagePayloadsInPlace> = [];
     if (countAllInlineImages(curatedHistory) >= imagePayloadThreshold) {
       const skipEntry = currentUserContent
         ? curatedHistory.find(
@@ -1885,24 +1899,28 @@ export class GeminiChat {
                 currentUserContent.parts?.some((p) => c.parts?.includes(p))),
           )
         : undefined;
-      const replaced = replaceImagePayloadsInPlace(
+      replaced = replaceImagePayloadsInPlace(
         curatedHistory,
         this.imagePayloadStore,
         skipEntry,
       );
-      const requestHistory = curatedHistory.map(copyContentContainer);
-      const reattachParts = buildReattachParts(replaced, maxRecentImages);
-      if (reattachParts.length > 0) {
-        const last = requestHistory.at(-1);
-        if (last?.role === 'user') {
-          last.parts = [...(last.parts ?? []), ...reattachParts];
-        } else {
-          requestHistory.push({ role: 'user', parts: reattachParts });
-        }
+    }
+    const requestHistory = curatedHistory.map(copyContentContainer);
+    const reattachParts = buildReattachParts(
+      replaced,
+      maxRecentImages,
+      requestHistory,
+      this.imagePayloadStore,
+    );
+    if (reattachParts.length > 0) {
+      const last = requestHistory.at(-1);
+      if (last?.role === 'user') {
+        last.parts = [...(last.parts ?? []), ...reattachParts];
+      } else {
+        requestHistory.push({ role: 'user', parts: reattachParts });
       }
-      return requestHistory;
     }
-    return curatedHistory.map(copyContentContainer);
+    return requestHistory;
   }
 
   private getRequestHistoryForRoute(
@@ -4047,9 +4065,9 @@ export class GeminiChat {
   }
 
   /**
-   * Returns a shallow copy of the history and each entry's parts array without
-   * cloning large part payloads. Use only for read-only consumers or consumers
-   * that replace touched entries before mutating them.
+   * Copies history containers, Part objects, and nested functionResponse parts
+   * without cloning large leaf payloads. Consumers must not mutate leaf
+   * payload objects.
    */
   getHistoryShallow(curated: boolean = false): Content[] {
     const history = curated

packages/core/src/services/image-payload-references.test.ts

@@ -88,14 +88,16 @@ describe('prepareImagePayloadsForRequest', () => {
         store,
       },
     );
-    const id = JSON.stringify(firstPass).match(/Image #([a-f0-9]{12})/)?.[1];
-    expect(id).toBeDefined();
+    const marker = JSON.stringify(firstPass).match(
+      /\[Image #[a-f0-9]{12}: [^\]]+\]/,
+    )?.[0];
+    expect(marker).toBeDefined();
 
     const prepared = prepareImagePayloadsForRequest(
       [
         oldImage,
         toolImageTurn('new-shot'),
-        { role: 'user', parts: [{ text: `inspect Image #${id}` }] },
+        { role: 'user', parts: [{ text: `inspect ${marker}` }] },
       ],
       {
         maxRecentImages: 0,
@@ -117,11 +119,13 @@ describe('prepareImagePayloadsForRequest', () => {
         store,
       },
     );
-    const id = JSON.stringify(firstPass).match(/Image #([a-f0-9]{12})/)?.[1];
-    expect(id).toBeDefined();
+    const marker = JSON.stringify(firstPass).match(
+      /\[Image #[a-f0-9]{12}: [^\]]+\]/,
+    )?.[0];
+    expect(marker).toBeDefined();
 
     const prepared = prepareImagePayloadsForRequest(
-      [{ role: 'user', parts: [{ text: `inspect Image #${id}` }] }],
+      [{ role: 'user', parts: [{ text: `inspect ${marker}` }] }],
       {
         maxRecentImages: 0,
         store,
@@ -133,6 +137,25 @@ describe('prepareImagePayloadsForRequest', () => {
     ]);
   });
 
+  it('does not resurrect a stored image from a bare Image #id echo', () => {
+    const store = new InMemoryImagePayloadStore();
+    const firstPass = prepareImagePayloadsForRequest(
+      [toolImageTurn('old-shot'), { role: 'model', parts: [{ text: 'ok' }] }],
+      { maxRecentImages: 0, store },
+    );
+    const id = JSON.stringify(firstPass).match(/Image #([a-f0-9]{12})/)?.[1];
+    expect(id).toBeDefined();
+
+    // A model reply echoing just the id (not the full eviction marker) must
+    // not re-inject the stored payload.
+    const prepared = prepareImagePayloadsForRequest(
+      [{ role: 'user', parts: [{ text: `I saw Image #${id} earlier` }] }],
+      { maxRecentImages: 0, store },
+    );
+
+    expect(imageParts(prepared)).toEqual([]);
+  });
+
   it('reattaches the most recent unique historical images', () => {
     const store = new InMemoryImagePayloadStore();
     const prepared = prepareImagePayloadsForRequest(
@@ -270,6 +293,43 @@ describe('replaceImagePayloadsInPlace', () => {
     expect(JSON.stringify(contents)).toContain('"data":"current-shot"');
     expect(JSON.stringify(contents)).not.toContain('"data":"old-shot"');
   });
+
+  it('rewrites shared top-level and nested Part objects', () => {
+    const store = new InMemoryImagePayloadStore();
+    const topLevel: Part = {
+      inlineData: { mimeType: 'image/png', data: 'top-level' },
+    };
+    const nested: Part = {
+      inlineData: { mimeType: 'image/png', data: 'nested' },
+    };
+    const durable: Content[] = [
+      { role: 'user', parts: [topLevel] },
+      {
+        role: 'user',
+        parts: [
+          {
+            functionResponse: {
+              id: 'call-1',
+              name: 'screenshot',
+              response: {},
+              parts: [nested],
+            },
+          },
+        ],
+      },
+    ];
+    const curated: Content[] = durable.map((content) => ({
+      ...content,
+      parts: [...(content.parts ?? [])],
+    }));
+
+    replaceImagePayloadsInPlace(curated, store);
+
+    expect(JSON.stringify(durable)).not.toContain('"data":');
+    expect(JSON.stringify(durable).match(/Image #[a-f0-9]{12}/g)).toHaveLength(
+      2,
+    );
+  });
 });
 
 describe('buildReattachParts', () => {
@@ -296,4 +356,84 @@ describe('buildReattachParts', () => {
     const replaced = replaceImagePayloadsInPlace([toolImageTurn('a')], store);
     expect(buildReattachParts(replaced, 0)).toEqual([]);
   });
+
+  it('resolves stored markers even when the current replacement pass is empty', () => {
+    const store = new InMemoryImagePayloadStore();
+    const contents = [
+      toolImageTurn('a'),
+      toolImageTurn('b'),
+      toolImageTurn('c'),
+      { role: 'user', parts: [{ text: 'continue' }] },
+    ];
+    replaceImagePayloadsInPlace(contents, store);
+
+    const parts = buildReattachParts([], 2, contents, store);
+
+    expect(
+      parts
+        .filter((part) => part.inlineData)
+        .map((part) => part.inlineData?.data),
+    ).toEqual(['b', 'c']);
+  });
+
+  it('reattaches a marker in the current user turn outside the recency cap', () => {
+    const store = new InMemoryImagePayloadStore();
+    const contents = [toolImageTurn('current')];
+    replaceImagePayloadsInPlace(contents, store);
+
+    const parts = buildReattachParts([], 0, contents, store);
+
+    expect(parts.at(-1)?.inlineData?.data).toBe('current');
+  });
+
+  it('bounds current-turn marker reattachment to the recency cap', () => {
+    const store = new InMemoryImagePayloadStore();
+    const contents: Content[] = [
+      {
+        role: 'user',
+        parts: ['a', 'b', 'c'].map((data) => ({
+          inlineData: { mimeType: 'image/png', data },
+        })),
+      },
+    ];
+    replaceImagePayloadsInPlace(contents, store);
+
+    const parts = buildReattachParts([], 1, contents, store);
+
+    expect(
+      parts
+        .filter((part) => part.inlineData)
+        .map((part) => part.inlineData?.data),
+    ).toEqual(['c']);
+  });
+
+  it('does not reattach an image that is already inline', () => {
+    const store = new InMemoryImagePayloadStore();
+    const markerContents = [toolImageTurn('same')];
+    replaceImagePayloadsInPlace(markerContents, store);
+    const marker = markerContents[0]!.parts![0]!;
+    const referencedContents: Content[] = [
+      {
+        role: 'user',
+        parts: [
+          marker,
+          { inlineData: { mimeType: 'image/png', data: 'same' } },
+        ],
+      },
+    ];
+
+    expect(buildReattachParts([], 1, referencedContents, store)).toEqual([]);
+  });
+
+  it('does not reattach an image already inline in a tool response', () => {
+    const store = new InMemoryImagePayloadStore();
+    const markerContents = [toolImageTurn('same')];
+    replaceImagePayloadsInPlace(markerContents, store);
+    const referencedContents: Content[] = [
+      markerContents[0]!,
+      toolImageTurn('same'),
+    ];
+
+    expect(buildReattachParts([], 1, referencedContents, store)).toEqual([]);
+  });
 });