Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions packages/core/src/services/chatCompressionService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -835,4 +835,67 @@ describe('ChatCompressionService', () => {
);
});
});

it('should strip function_response.parts (Gemini 3 multimodal data) before sending to summarizer', async () => {
const history: Content[] = [
{ role: 'user', parts: [{ text: 'msg1' }] },
{
role: 'user',
parts: [
{
functionResponse: {
name: 'screenshot_tool',
response: { output: 'Screenshot taken.' },
// Gemini 3 nests inlineData inside function_response.parts at runtime
parts: [
{ inlineData: { mimeType: 'image/png', data: 'base64data' } },
],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any,
},
],
},
{ role: 'model', parts: [{ text: 'resp2' }] },
{ role: 'user', parts: [{ text: 'msg3' }] },
{ role: 'model', parts: [{ text: 'resp4' }] },
];

vi.mocked(mockChat.getHistory).mockReturnValue(history);
vi.mocked(mockChat.getLastPromptTokenCount).mockReturnValue(600000);

await service.compress(
mockChat,
mockPromptId,
true,
mockModel,
mockConfig,
false,
);

// Verify that both generateContent calls had the nested .parts stripped
const generateContentMock = vi.mocked(
mockConfig.getBaseLlmClient().generateContent,
);
expect(generateContentMock).toHaveBeenCalledTimes(2);

for (const call of generateContentMock.mock.calls) {
const contents = call[0].contents;
for (const content of contents) {
for (const part of content.parts ?? []) {
if (part.functionResponse) {
// The nested .parts field should have been removed
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(part.functionResponse as any).parts,
).toBeUndefined();
// But the rest of functionResponse should be preserved
expect(part.functionResponse.name).toBe('screenshot_tool');
expect(part.functionResponse.response).toEqual({
output: 'Screenshot taken.',
});
}
}
}
}
});
});
27 changes: 25 additions & 2 deletions packages/core/src/services/chatCompressionService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,27 @@ async function truncateHistoryToBudget(
return truncatedHistory;
}

/**
* Strips the Gemini-3-specific `function_response.parts` (nested multimodal
* data such as inlineData) from history entries. The compression summarizer
* model may not support this feature, so we sanitize before sending.
*/
function sanitizeHistoryForSummarizer(history: Content[]): Content[] {
return history.map((content) => ({
...content,
parts: (content.parts ?? []).map((part) => {
if (part.functionResponse) {
const { parts: _nestedParts, ...cleanFR } =
part.functionResponse as // not part of the official FunctionResponse type. // The `parts` field is added at runtime for Gemini 3 models and is
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Record<string, any>;
return { ...part, functionResponse: cleanFR };
}
return part;
}),
}));
}

export class ChatCompressionService {
async compress(
chat: GeminiChat,
Expand Down Expand Up @@ -350,10 +371,12 @@ export class ChatCompressionService {
? 'A previous <state_snapshot> exists in the history. You MUST integrate all still-relevant information from that snapshot into the new one, updating it with the more recent events. Do not lose established constraints or critical knowledge.'
: 'Generate a new <state_snapshot> based on the provided history.';

const sanitizedHistory = sanitizeHistoryForSummarizer(historyForSummarizer);

const summaryResponse = await config.getBaseLlmClient().generateContent({
modelConfigKey: { model: modelStringToModelConfigAlias(model) },
contents: [
...historyForSummarizer,
...sanitizedHistory,
{
role: 'user',
parts: [
Expand All @@ -378,7 +401,7 @@ export class ChatCompressionService {
.generateContent({
modelConfigKey: { model: modelStringToModelConfigAlias(model) },
contents: [
...historyForSummarizer,
...sanitizedHistory,
{
role: 'model',
parts: [{ text: summary }],
Expand Down