fix(core): Reduce multimodal history payload size

packages/core/src/core/geminiChat.test.ts

@@ -1520,6 +1520,76 @@ describe('GeminiChat', async () => {
       );
     });
 
+    it('keeps historical image refs stable and reattaches only recent image bytes', async () => {
+      vi.mocked(mockConfig.getChatCompression).mockReturnValue({
+        maxRecentImagesToRetain: 1,
+      });
+      chat.setHistory([
+        {
+          role: 'user',
+          parts: [{ inlineData: { mimeType: 'image/png', data: 'old-shot' } }],
+        },
+        {
+          role: 'user',
+          parts: [{ inlineData: { mimeType: 'image/png', data: 'new-shot' } }],
+        },
+      ]);
+      const response = (async function* () {
+        yield {
+          candidates: [
+            {
+              content: {
+                parts: [{ text: 'response' }],
+                role: 'model',
+              },
+              finishReason: 'STOP',
+              index: 0,
+              safetyRatings: [],
+            },
+          ],
+          text: () => 'response',
+        } as unknown as GenerateContentResponse;
+      })();
+      vi.mocked(mockContentGenerator.generateContentStream).mockResolvedValue(
+        response,
+      );
+
+      const stream = await chat.sendMessageStream(
+        'test-model',
+        { message: 'continue' },
+        'prompt-id-image-refs',
+      );
+      for await (const _ of stream) {
+        // consume stream
+      }
+
+      const request = vi.mocked(mockContentGenerator.generateContentStream).mock
+        .calls[0]?.[0];
+      const contents = request?.contents as Content[];
+      const serialized = JSON.stringify(contents);
+      expect(serialized).toMatch(
+        /\[Image #[a-f0-9]{12}: image\/png, \d+ bytes\]/,
+      );
+      expect(serialized).not.toContain('"data":"old-shot"');
+      expect(serialized?.match(/"data":"new-shot"/g)).toHaveLength(1);
+      expect(contents.at(-1)).toEqual({
+        role: 'user',
+        parts: expect.arrayContaining([
+          { text: 'continue' },
+          {
+            text: expect.stringContaining('Recent images reattached'),
+          },
+          {
+            inlineData: {
+              mimeType: 'image/png',
+              data: 'new-shot',
+              displayName: undefined,
+            },
+          },
+        ]),
+      });
+    });
+
     it('coalesces startup reminders with the first user prompt for provider requests', async () => {
       chat.setHistory([
         {

packages/core/src/core/geminiChat.ts

@@ -56,7 +56,14 @@ import {
   type CompactTrigger,
 } from '../services/chatCompressionService.js';
 import { acquireSleepInhibitor } from '../services/sleepInhibitor.js';
-import { resolveSlimmingConfig } from '../services/compactionInputSlimming.js';
+import {
+  resolveCompactionTuning,
+  resolveSlimmingConfig,
+} from '../services/compactionInputSlimming.js';
+import {
+  InMemoryImagePayloadStore,
+  prepareImagePayloadsForRequest,
+} from '../services/image-payload-references.js';
 import {
   estimateContentTokens,
   estimatePromptTokens,
@@ -1491,8 +1498,19 @@ export class GeminiChat {
    * Public history readers still use {@link getHistory}, which returns a
    * defensive deep copy for caller mutation safety.
    */
-  private getRequestHistory(): Content[] {
-    return extractCuratedHistory(this.history).map(copyContentContainer);
+  private getRequestHistory(currentUserPartCount = 0): Content[] {
+    const requestHistory = extractCuratedHistory(this.history).map(
+      copyContentContainer,
+    );
+    const { maxRecentImages } = resolveCompactionTuning(
+      this.config.getChatCompression(),
+    );
+    const imageStore = new InMemoryImagePayloadStore();
+    return prepareImagePayloadsForRequest(requestHistory, {
+      maxRecentImages,
+      preserveLastUserImagePartCount: currentUserPartCount,
+      store: imageStore,
+    });
   }
 
   /**
@@ -1779,7 +1797,7 @@ export class GeminiChat {
           parsedEnvMaxTokensForThreshold ??
           0)
         : Math.max(ESCALATED_MAX_TOKENS, tokenLimit(model, 'output')));
-
+    let currentUserPartCount = 0;
     try {
       // The send-lock above is held but the generator's `finally` (which
       // resolves it) has not run yet. Any setup error before returning the
@@ -1984,7 +2002,8 @@ export class GeminiChat {
               .join(', '),
         );
       }
-      requestContents = this.getRequestHistory();
+      currentUserPartCount = userContent.parts?.length ?? 0;
+      requestContents = this.getRequestHistory(currentUserPartCount);
     } catch (error) {
       if (userContentAdded) {
         this.history.pop();
@@ -2287,7 +2306,8 @@ export class GeminiChat {
                     // other retry branches in case a future in-place
                     // tryCompress stops resetting it.
                     popPartialIfPushed();
-                    requestContents = self.getRequestHistory();
+                    requestContents =
+                      self.getRequestHistory(currentUserPartCount);
                     debugLogger.info(
                       `Reactive compression succeeded: ` +
                         `${reactiveInfo.originalTokenCount} -> ` +
@@ -2530,7 +2550,8 @@ export class GeminiChat {
             // model's continuation appends to the previous partial output.
             yield { type: StreamEventType.RETRY, isContinuation: true };
             // Re-send with the updated history (includes partial + recovery)
-            const recoveryContents = self.getRequestHistory();
+            const recoveryContents =
+              self.getRequestHistory(currentUserPartCount);
             escalatedFinishReason = undefined;
             try {
               const recoveryStream = await self.makeApiCallAndProcessStream(

packages/core/src/services/image-payload-references.test.ts

@@ -0,0 +1,134 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import type { Content, Part } from '@google/genai';
+import { describe, expect, it } from 'vitest';
+import {
+  InMemoryImagePayloadStore,
+  prepareImagePayloadsForRequest,
+} from './image-payload-references.js';
+
+function toolImageTurn(data: string): Content {
+  return {
+    role: 'user',
+    parts: [
+      {
+        functionResponse: {
+          id: `call-${data}`,
+          name: 'screenshot',
+          response: { output: `captured ${data}` },
+          parts: [{ inlineData: { mimeType: 'image/png', data } }],
+        },
+      },
+    ],
+  };
+}
+
+function imageParts(contents: Content[]): Part[] {
+  const result: Part[] = [];
+  for (const content of contents) {
+    for (const part of content.parts ?? []) {
+      if (part.inlineData?.mimeType?.startsWith('image/')) {
+        result.push(part);
+      }
+      const nested = part.functionResponse?.parts as Part[] | undefined;
+      for (const inner of nested ?? []) {
+        if (inner.inlineData?.mimeType?.startsWith('image/')) {
+          result.push(inner);
+        }
+      }
+    }
+  }
+  return result;
+}
+
+describe('prepareImagePayloadsForRequest', () => {
+  it('replaces historical image positions with stable refs and reattaches only the most recent images', () => {
+    const store = new InMemoryImagePayloadStore();
+    const history: Content[] = [
+      toolImageTurn('old-shot'),
+      toolImageTurn('new-shot'),
+      { role: 'user', parts: [{ text: 'continue' }] },
+    ];
+
+    const prepared = prepareImagePayloadsForRequest(history, {
+      maxRecentImages: 1,
+      store,
+    });
+
+    const serialized = JSON.stringify(prepared);
+    expect(serialized).toMatch(
+      /\[Image #[a-f0-9]{12}: image\/png, \d+ bytes\]/,
+    );
+    expect(serialized).not.toContain('"data":"old-shot"');
+    expect(imageParts(prepared).map((part) => part.inlineData?.data)).toEqual([
+      'new-shot',
+    ]);
+    expect(prepared).toHaveLength(history.length);
+    expect(prepared.at(-1)?.role).toBe('user');
+    expect(prepared.at(-1)?.parts?.[0]?.text).toBe('continue');
+    expect(prepared.at(-1)?.parts?.[1]?.text).toContain(
+      'Recent images reattached',
+    );
+  });
+
+  it('reattaches an older image when the current request explicitly references its stable id', () => {
+    const store = new InMemoryImagePayloadStore();
+    const oldImage = toolImageTurn('old-shot');
+    const firstPass = prepareImagePayloadsForRequest(
+      [oldImage, { role: 'model', parts: [{ text: 'ok' }] }],
+      {
+        maxRecentImages: 0,
+        store,
+      },
+    );
+    const id = JSON.stringify(firstPass).match(/Image #([a-f0-9]{12})/)?.[1];
+    expect(id).toBeDefined();
+
+    const prepared = prepareImagePayloadsForRequest(
+      [
+        oldImage,
+        toolImageTurn('new-shot'),
+        { role: 'user', parts: [{ text: `inspect Image #${id}` }] },
+      ],
+      {
+        maxRecentImages: 0,
+        store,
+      },
+    );
+
+    expect(imageParts(prepared).map((part) => part.inlineData?.data)).toEqual([
+      'old-shot',
+    ]);
+  });
+
+  it('preserves images in the current user request when maxRecentImages is zero', () => {
+    const store = new InMemoryImagePayloadStore();
+    const prepared = prepareImagePayloadsForRequest(
+      [
+        toolImageTurn('old-shot'),
+        {
+          role: 'user',
+          parts: [
+            { text: 'inspect this' },
+            { inlineData: { mimeType: 'image/png', data: 'current-shot' } },
+          ],
+        },
+      ],
+      {
+        maxRecentImages: 0,
+        preserveLastUserImagePartCount: 2,
+        store,
+      },
+    );
+
+    const serialized = JSON.stringify(prepared);
+    expect(serialized).not.toContain('"data":"old-shot"');
+    expect(imageParts(prepared).map((part) => part.inlineData?.data)).toEqual([
+      'current-shot',
+    ]);
+  });
+});