Skip to content
328 changes: 323 additions & 5 deletions packages/core/src/core/geminiChat.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,25 @@ describe('GeminiChat', async () => {
return collecting;
}

function streamResponse(
response: GenerateContentResponse,
): AsyncGenerator<GenerateContentResponse> {
return (async function* () {
yield response;
})();
}

function stopResponse(parts: Part[]): GenerateContentResponse {
return {
candidates: [
{
content: { parts },
finishReason: 'STOP',
},
],
} as unknown as GenerateContentResponse;
}

describe('system instruction helpers', () => {
it('replaces prior session-start context instead of appending indefinitely', () => {
const isolatedChat = new GeminiChat(
Expand Down Expand Up @@ -4607,12 +4626,20 @@ describe('GeminiChat', async () => {
);
await expectStreamExhaustion(stream);

// Should be called 3 times (1 initial + 2 transient retries)
// Should be called 5 times (1 initial + 4 transient retries)
expect(
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(3);
expect(mockLogContentRetry).toHaveBeenCalledTimes(2);
).toHaveBeenCalledTimes(5);
expect(mockLogContentRetry).toHaveBeenCalledTimes(4);
expect(mockLogContentRetryFailure).toHaveBeenCalledTimes(1);
expect(mockLogContentRetryFailure).toHaveBeenCalledWith(
mockConfig,
expect.objectContaining({
total_attempts: 5,
final_error_type: 'NO_FINISH_REASON',
model: 'test-model',
}),
);

// History should still contain the user message.
const history = chat.getHistory();
Expand All @@ -4626,9 +4653,237 @@ describe('GeminiChat', async () => {
}
});

Comment thread
yiliang114 marked this conversation as resolved.
Resolved
it('should retry usage-only empty streams and succeed on a later attempt', async () => {
it('should recover after four consecutive invalid streams', async () => {
vi.useFakeTimers();
try {
let callCount = 0;
vi.mocked(
mockContentGenerator.generateContentStream,
).mockImplementation(async () => {
callCount++;
if (callCount <= 4) {
return (async function* () {
yield {
candidates: [
{
content: { parts: [] },
finishReason: 'STOP',
},
],
} as unknown as GenerateContentResponse;
})();
}

return (async function* () {
yield {
candidates: [
{
content: { parts: [{ text: 'Recovered response' }] },
finishReason: 'STOP',
},
],
} as unknown as GenerateContentResponse;
})();
});

const stream = await chat.sendMessageStream(
'test-model',
{ message: 'test' },
'prompt-id-four-invalid-streams',
);
const events = await collectStreamWithFakeTimers(stream, 25_000);

expect(
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(5);
expect(mockLogContentRetry).toHaveBeenCalledTimes(4);
for (const [index, retryDelayMs] of [
2000, 4000, 6000, 8000,
].entries()) {
expect(mockLogContentRetry).toHaveBeenNthCalledWith(
index + 1,
mockConfig,
expect.objectContaining({
attempt_number: index,
error_type: 'NO_RESPONSE_TEXT',
retry_delay_ms: retryDelayMs,
model: 'test-model',
}),
);
}
expect(mockLogContentRetryFailure).not.toHaveBeenCalled();
expect(
events.some(
(event) =>
event.type === StreamEventType.CHUNK &&
event.value.candidates?.[0]?.content?.parts?.[0]?.text ===
'Recovered response',
),
).toBe(true);
expect(chat.getHistory()).toEqual([
{ role: 'user', parts: [{ text: 'test' }] },
{ role: 'model', parts: [{ text: 'Recovered response' }] },
]);
} finally {
vi.useRealTimers();
}
});

it('should keep protocol tag leak retries at the existing budget', async () => {
vi.useFakeTimers();
try {
vi.mocked(
mockContentGenerator.generateContentStream,
).mockImplementation(async () =>
(async function* () {
yield {
candidates: [
{
content: {
parts: [
{
text: '<analysis>hidden</analysis><summary>leaked</summary>',
},
],
},
finishReason: 'STOP',
},
],
} as unknown as GenerateContentResponse;
})(),
);

const stream = await chat.sendMessageStream(
'test-model',
{ message: 'test' },
'prompt-id-protocol-leak-budget',
Comment thread
yiliang114 marked this conversation as resolved.
Resolved
);
await expectStreamExhaustion(stream);

expect(
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(3);
expect(mockLogContentRetry).toHaveBeenCalledTimes(2);
expect(mockLogContentRetryFailure).toHaveBeenCalledWith(
mockConfig,
expect.objectContaining({
total_attempts: 3,
final_error_type: 'PROTOCOL_TAG_LEAK',
model: 'test-model',
}),
);
} finally {
vi.useRealTimers();
}
});

it('keeps invalid stream retry budgets independent across error types', async () => {
vi.useFakeTimers();
try {
let callCount = 0;
vi.mocked(
mockContentGenerator.generateContentStream,
).mockImplementation(async () => {
callCount++;
if (callCount <= 2) {
return streamResponse(stopResponse([]));
}
if (callCount === 3) {
return streamResponse(
stopResponse([
{
text: '<analysis>hidden</analysis><summary>leaked</summary>',
},
]),
);
}

return streamResponse(stopResponse([{ text: 'Recovered response' }]));
});

const stream = await chat.sendMessageStream(
'test-model',
{ message: 'test' },
'prompt-id-mixed-invalid-streams',
);
const events = await collectStreamWithFakeTimers(stream, 15_000);

expect(
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(4);
expect(mockLogContentRetry).toHaveBeenCalledTimes(3);
expect(mockLogContentRetry).toHaveBeenLastCalledWith(
mockConfig,
expect.objectContaining({
attempt_number: 0,
error_type: 'PROTOCOL_TAG_LEAK',
retry_delay_ms: 2000,
model: 'test-model',
}),
);
expect(
events.some(
(event) =>
event.type === StreamEventType.CHUNK &&
event.value.candidates?.[0]?.content?.parts?.[0]?.text ===
'Recovered response',
),
).toBe(true);
} finally {
vi.useRealTimers();
}
});

it('surfaces an abort fired during the invalid-stream retry delay without retrying again', async () => {
vi.useFakeTimers();
try {
const abortController = new AbortController();
vi.mocked(
mockContentGenerator.generateContentStream,
).mockImplementation(async () => streamResponse(stopResponse([])));

const stream = await chat.sendMessageStream(
'test-model',
{ message: 'test', config: { abortSignal: abortController.signal } },
'prompt-id-invalid-stream-abort-delay',
);

const iterator = stream[Symbol.asyncIterator]();
let next = await iterator.next();
while (!next.done && next.value.type !== StreamEventType.RETRY) {
next = await iterator.next();
}
if (next.done) {
throw new Error('Expected invalid stream retry event.');
}
expect(next.value.type).toBe(StreamEventType.RETRY);

const nextPromise = iterator.next();
abortController.abort();
await expect(nextPromise).rejects.toThrow();

expect(
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});

it('should retry usage-only empty streams without recording failed attempts', async () => {
vi.useFakeTimers();
try {
const recordAssistantTurn = vi.fn();
const chatWithRecording = new GeminiChat(
mockConfig,
config,
[],
{
recordAssistantTurn,
recordChatCompression: vi.fn(),
} as unknown as ConstructorParameters<typeof GeminiChat>[3],
uiTelemetryService,
);
vi.mocked(mockContentGenerator.generateContentStream)
.mockImplementationOnce(async () =>
(async function* () {
Expand Down Expand Up @@ -4656,7 +4911,7 @@ describe('GeminiChat', async () => {
})(),
);

const stream = await chat.sendMessageStream(
const stream = await chatWithRecording.sendMessageStream(
'test-model',
{ message: 'test' },
'prompt-id-empty-usage-retry',
Expand All @@ -4667,6 +4922,10 @@ describe('GeminiChat', async () => {
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(2);
expect(mockLogContentRetry).toHaveBeenCalledTimes(1);
expect(recordAssistantTurn).toHaveBeenCalledTimes(1);
expect(recordAssistantTurn.mock.calls[0]?.[0].message).toEqual([
{ text: 'Recovered after empty stream' },
]);
expect(
events.some(
(e) =>
Expand Down Expand Up @@ -9620,6 +9879,65 @@ describe('GeminiChat', async () => {
).toBe(true);
});

it('keeps protocol tag leak budget during output continuation', async () => {
vi.useFakeTimers();
try {
const protocolLeak = () =>
({
[Symbol.asyncIterator]() {
return this;
},
async next() {
throw new InvalidStreamError(
'Protocol tag leaked during continuation.',
'PROTOCOL_TAG_LEAK',
);
},
async return() {
return { done: true, value: undefined };
},
async throw(error?: unknown) {
throw error;
},
}) as AsyncGenerator<GenerateContentResponse>;
const streams = [
makeStream([makeChunk([{ text: 'initial' }], 'MAX_TOKENS')]),
makeStream([makeChunk([{ text: 'escalated' }], 'MAX_TOKENS')]),
protocolLeak(),
protocolLeak(),
protocolLeak(),
protocolLeak(),
protocolLeak(),
];
let callIndex = 0;
vi.mocked(
mockContentGenerator.generateContentStream,
).mockImplementation(async () => streams[callIndex++]!);

const stream = await chat.sendMessageStream(
'gemini-pro',
{ message: 'essay' },
'prompt-recovery-protocol-leak-budget',
);

await collectStreamWithFakeTimers(stream, 35_000);

expect(
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(5);
expect(mockLogContentRetry).toHaveBeenCalledTimes(2);
expect(mockLogContentRetry).toHaveBeenLastCalledWith(
mockConfig,
expect.objectContaining({
attempt_number: 1,
error_type: 'PROTOCOL_TAG_LEAK',
}),
);
} finally {
vi.useRealTimers();
}
});

it('should cap recovery attempts at MAX_OUTPUT_RECOVERY_ATTEMPTS (3)', async () => {
// Every stream returns MAX_TOKENS with text (no functionCall).
vi.mocked(mockContentGenerator.generateContentStream).mockImplementation(
Expand Down
Loading
Loading