diff --git a/.ai-sdk-factory/reproduction-replay.json b/.ai-sdk-factory/reproduction-replay.json new file mode 100644 index 000000000000..5eafed8943b3 --- /dev/null +++ b/.ai-sdk-factory/reproduction-replay.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "issueId": 18872, + "command": "pnpm -C examples/ai-functions exec tsx src/reproduction/open-responses-truncated-reasoning.ts", + "artifactPaths": [ + "examples/ai-functions/src/reproduction/open-responses-truncated-reasoning.ts", + "packages/open-responses/src/responses/__fixtures__/issue-18872-truncated-reasoning.chunks.txt", + "packages/open-responses/src/responses/__fixtures__/issue-18872-unknown-incomplete-reason.chunks.txt" + ], + "expectedFailure": { + "exitCode": 1, + "signal": "ISSUE #18872: Open Responses truncated-stream handling is incorrect" + } +} diff --git a/examples/ai-functions/src/reproduction/open-responses-truncated-reasoning.ts b/examples/ai-functions/src/reproduction/open-responses-truncated-reasoning.ts new file mode 100644 index 000000000000..28b8291d67c6 --- /dev/null +++ b/examples/ai-functions/src/reproduction/open-responses-truncated-reasoning.ts @@ -0,0 +1,119 @@ +import { createOpenResponses } from '@ai-sdk/open-responses'; +import { streamText, tool } from 'ai'; +import fs from 'node:fs'; +import { z } from 'zod/v4'; + +function createSseFetch(fixtureName: string): typeof fetch { + return async () => + new Response( + [ + ...fs + .readFileSync( + `../../packages/open-responses/src/responses/__fixtures__/${fixtureName}.chunks.txt`, + 'utf8', + ) + .trim() + .split('\n') + .map(event => `data: ${event}\n\n`), + 'data: [DONE]\n\n', + ].join(''), + { + headers: { + 'content-type': 'text/event-stream', + }, + }, + ); +} + +async function reproduceMismatchedReasoningEnd() { + const openResponses = createOpenResponses({ + name: 'issue-18872-reasoning', + url: 'https://example.test/v1/responses', + fetch: createSseFetch('issue-18872-truncated-reasoning'), + }); + + const result = streamText({ + model: openResponses('reasoning-model'), + prompt: 'Think until the output token limit is reached.', + }); + + const errors: unknown[] = []; + + for await (const part of result.fullStream) { + if (part.type === 'error') { + errors.push(part.error); + } + } + + return { + errors, + finishReason: await result.finishReason, + }; +} + +async function reproduceUnknownIncompleteReason() { + const openResponses = createOpenResponses({ + name: 'issue-18872-finish-reason', + url: 'https://example.test/v1/responses', + fetch: createSseFetch('issue-18872-unknown-incomplete-reason'), + }); + + const result = streamText({ + model: openResponses('tool-model'), + prompt: 'Check the weather.', + tools: { + get_weather: tool({ + inputSchema: z.object({ location: z.string() }), + execute: async ({ location }) => ({ location, temperature: 20 }), + }), + }, + }); + + for await (const _part of result.fullStream) { + // Consume the stream so the terminal finish reason is available. + } + + return { + finishReason: await result.finishReason, + rawFinishReason: await result.rawFinishReason, + }; +} + +async function main() { + const reasoning = await reproduceMismatchedReasoningEnd(); + const unknownReason = await reproduceUnknownIncompleteReason(); + const failures: string[] = []; + + if (reasoning.errors.length > 0) { + failures.push( + `truncated reasoning emitted stream errors: ${reasoning.errors.map(String).join(', ')}`, + ); + } + + if (reasoning.finishReason !== 'length') { + failures.push( + `max_output_tokens mapped to ${JSON.stringify(reasoning.finishReason)} instead of "length"`, + ); + } + + if (unknownReason.finishReason !== 'other') { + failures.push( + `unknown incomplete reason ${JSON.stringify(unknownReason.rawFinishReason)} mapped to ${JSON.stringify(unknownReason.finishReason)} instead of "other"`, + ); + } + + if (failures.length > 0) { + console.error( + `ISSUE #18872: Open Responses truncated-stream handling is incorrect\n${failures.map(failure => `- ${failure}`).join('\n')}`, + ); + process.exitCode = 1; + return; + } + + console.log('Issue #18872 is not reproduced.'); +} + +main().catch(error => { + console.error(error); + process.exitCode = 1; +}); diff --git a/packages/open-responses/src/responses/__fixtures__/issue-18872-truncated-reasoning.chunks.txt b/packages/open-responses/src/responses/__fixtures__/issue-18872-truncated-reasoning.chunks.txt new file mode 100644 index 000000000000..16738d306f02 --- /dev/null +++ b/packages/open-responses/src/responses/__fixtures__/issue-18872-truncated-reasoning.chunks.txt @@ -0,0 +1,2 @@ +{"type":"response.output_item.added","sequence_number":0,"output_index":0,"item":{"id":"rs_actual_reasoning_id","type":"reasoning","status":"in_progress","summary":[]}} +{"type":"response.incomplete","sequence_number":1,"response":{"status":"incomplete","incomplete_details":{"reason":"max_output_tokens"},"usage":{"input_tokens":1,"input_tokens_details":{"cached_tokens":0},"output_tokens":1,"output_tokens_details":{"reasoning_tokens":1},"total_tokens":2}}} diff --git a/packages/open-responses/src/responses/__fixtures__/issue-18872-unknown-incomplete-reason.chunks.txt b/packages/open-responses/src/responses/__fixtures__/issue-18872-unknown-incomplete-reason.chunks.txt new file mode 100644 index 000000000000..a6defd12a4d3 --- /dev/null +++ b/packages/open-responses/src/responses/__fixtures__/issue-18872-unknown-incomplete-reason.chunks.txt @@ -0,0 +1,4 @@ +{"type":"response.output_item.added","sequence_number":0,"output_index":0,"item":{"id":"fc_weather","type":"function_call","status":"in_progress","call_id":"call_weather","name":"get_weather","arguments":""}} +{"type":"response.function_call_arguments.done","sequence_number":1,"item_id":"fc_weather","output_index":0,"arguments":"{\"location\":\"San Francisco\"}"} +{"type":"response.output_item.done","sequence_number":2,"output_index":0,"item":{"id":"fc_weather","type":"function_call","status":"completed","call_id":"call_weather","name":"get_weather","arguments":"{\"location\":\"San Francisco\"}"}} +{"type":"response.incomplete","sequence_number":3,"response":{"status":"incomplete","incomplete_details":{"reason":"server_timeout"},"usage":{"input_tokens":1,"input_tokens_details":{"cached_tokens":0},"output_tokens":1,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":2}}} diff --git a/packages/open-responses/src/responses/open-responses-language-model.test.ts b/packages/open-responses/src/responses/open-responses-language-model.test.ts index 960e83c839f1..5657573a2a5d 100644 --- a/packages/open-responses/src/responses/open-responses-language-model.test.ts +++ b/packages/open-responses/src/responses/open-responses-language-model.test.ts @@ -706,6 +706,54 @@ describe('OpenResponsesLanguageModel', () => { }; } + describe('truncated responses', () => { + it('should close active reasoning with its actual item id', async () => { + prepareChunksFixtureResponse('issue-18872-truncated-reasoning'); + + const result = await createModel().doStream({ + prompt: TEST_PROMPT, + }); + const parts = await convertReadableStreamToArray(result.stream); + const reasoningStart = parts.find( + part => part.type === 'reasoning-start', + ); + const reasoningEnd = parts.find(part => part.type === 'reasoning-end'); + + expect(reasoningStart).toMatchObject({ + type: 'reasoning-start', + id: 'rs_actual_reasoning_id', + }); + expect(reasoningEnd).toMatchObject({ + type: 'reasoning-end', + id: 'rs_actual_reasoning_id', + }); + expect(parts.at(-1)).toMatchObject({ + type: 'finish', + finishReason: { + unified: 'length', + raw: 'max_output_tokens', + }, + }); + }); + + it('should map an unknown incomplete reason to other even when tool calls exist', async () => { + prepareChunksFixtureResponse('issue-18872-unknown-incomplete-reason'); + + const result = await createModel().doStream({ + prompt: TEST_PROMPT, + }); + const parts = await convertReadableStreamToArray(result.stream); + + expect(parts.at(-1)).toMatchObject({ + type: 'finish', + finishReason: { + unified: 'other', + raw: 'server_timeout', + }, + }); + }); + }); + describe('basic generation', () => { it('should stream content', async () => { prepareChunksFixtureResponse('lmstudio-basic.1');