From 0df12e075a96beedd95ba5a4dc0f3aff8502455b Mon Sep 17 00:00:00 2001 From: Parth Dogra Date: Thu, 6 Aug 2026 10:06:15 +0530 Subject: [PATCH] fix(anthropic): honor schema alongside tools via final_result tool --- package.json | 3 +- src/lib/core/modules/GenerationHandler.ts | 26 + .../core/modules/structuredOutputPolicy.ts | 8 + src/lib/providers/anthropic/client.ts | 133 ++- .../providers/anthropic/structuredOutput.ts | 118 +++ src/lib/types/generate.ts | 11 + ...s-test-suite-anthropic-structured-tools.ts | 841 ++++++++++++++++++ 7 files changed, 1136 insertions(+), 4 deletions(-) create mode 100644 src/lib/providers/anthropic/structuredOutput.ts create mode 100644 test/continuous-test-suite-anthropic-structured-tools.ts diff --git a/package.json b/package.json index 2860e6097..5fc35ea9c 100644 --- a/package.json +++ b/package.json @@ -134,6 +134,7 @@ "test:autoresearch": "npx tsx test/continuous-test-suite-autoresearch.ts", "test:autoresearch:redis": "npx tsx test/continuous-test-suite-autoresearch-redis.ts", "test:anthropic-tools-policy": "npx tsx test/continuous-test-suite-anthropic-tools-policy.ts", + "test:anthropic-structured": "npx tsx test/continuous-test-suite-anthropic-structured-tools.ts", "test:sagemaker-tools": "npx tsx test/continuous-test-suite-sagemaker-tools.ts", "test:anthropic-multimodal": "npx tsx test/continuous-test-suite-anthropic-multimodal.ts", "test:excel-interop": "npx tsx test/continuous-test-suite-excel-interop.ts", @@ -153,7 +154,7 @@ "test:system-messages": "npx tsx test/continuous-test-suite-system-messages.ts", "test:test-stubs": "npx tsx test/continuous-test-suite-test-stubs.ts", "test:tool-routing-semantic": "npx tsx test/continuous-test-suite-tool-routing-semantic.ts", - "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:file-detector-extension && pnpm run test:file-detector-magic-bytes && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:litellm-context && pnpm run test:dedup-execute-map && pnpm run test:step-budget-guard && pnpm run test:agent-plumbing && pnpm run test:tool-execution-recorder && pnpm run test:proxy-terminal-errors && pnpm run test:system-messages && pnpm run test:tool-routing-semantic && pnpm run test:anthropic-tools-policy && pnpm run test:sagemaker-tools && pnpm run test:anthropic-multimodal && pnpm run test:excel-interop && pnpm run test:model-capabilities && pnpm run test:agent-runtime:vitest && pnpm run test:agent-delegation && pnpm run test:retry-after:vitest && pnpm run test:sampling-params && pnpm run test:structured-recovery && pnpm run test:prompt-redaction && pnpm run test:mcp-result-cache && pnpm run test:test-stubs && pnpm run test:model-not-found-retryable && pnpm run test:websearch-grounding", + "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:file-detector-extension && pnpm run test:file-detector-magic-bytes && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:litellm-context && pnpm run test:dedup-execute-map && pnpm run test:step-budget-guard && pnpm run test:agent-plumbing && pnpm run test:tool-execution-recorder && pnpm run test:proxy-terminal-errors && pnpm run test:system-messages && pnpm run test:tool-routing-semantic && pnpm run test:anthropic-tools-policy && pnpm run test:anthropic-structured && pnpm run test:sagemaker-tools && pnpm run test:anthropic-multimodal && pnpm run test:excel-interop && pnpm run test:model-capabilities && pnpm run test:agent-runtime:vitest && pnpm run test:agent-delegation && pnpm run test:retry-after:vitest && pnpm run test:sampling-params && pnpm run test:structured-recovery && pnpm run test:prompt-redaction && pnpm run test:mcp-result-cache && pnpm run test:test-stubs && pnpm run test:model-not-found-retryable && pnpm run test:websearch-grounding", "// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit)": "", "test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic", "// CI tier — product output (image/video/TTS/PPT) — costs $$ per run": "", diff --git a/src/lib/core/modules/GenerationHandler.ts b/src/lib/core/modules/GenerationHandler.ts index c3867097a..40f46c9d4 100644 --- a/src/lib/core/modules/GenerationHandler.ts +++ b/src/lib/core/modules/GenerationHandler.ts @@ -58,11 +58,13 @@ import { isToolsSchemaExclusionInForce, } from "./structuredOutputPolicy.js"; import { coerceJsonToSchema } from "../../utils/json/coerce.js"; +import { convertZodToJsonSchema } from "../../utils/schemaConversion.js"; import type { LanguageModel, ModelMessage, PrepareStepFunction, Tool, + ZodUnknownSchema, } from "../../types/index.js"; import { NoObjectGeneratedError } from "../../utils/generationErrors.js"; import { Output, stepCountIs } from "../../utils/tool.js"; @@ -165,11 +167,15 @@ function buildProviderOptions( options: TextGenerationOptions, isGoogleProvider: boolean, callerTimeoutMs: number | undefined, + finalResultSchema: Record | undefined, ): Parameters[0]["providerOptions"] { const providerOptions: Record> = {}; if (callerTimeoutMs !== undefined) { providerOptions.neurolink = { timeoutMs: callerTimeoutMs }; } + if (finalResultSchema) { + providerOptions.anthropic = { finalResultSchema }; + } if (options.thinkingConfig?.enabled && isGoogleProvider) { // Gemini 3 uses thinkingLevel; Gemini 2.5 uses thinkingBudget. providerOptions.google = { @@ -349,10 +355,30 @@ export class GenerationHandler { const { callerTimeoutMs, turnBudgetMs, wrapupLeadMs, turnDeadline } = resolveTurnBudget(options, turnStartMs); let wrapupForced = false; + // The native Anthropic Messages surface cannot combine AI-SDK structured + // output with tools (see structuredOutputPolicy — experimental_output + // replaces the tools array), so `useStructuredOutput` is false above and + // the schema would simply be dropped for every agent/MCP turn. Hand the + // JSON Schema to the provider instead: it appends an additive + // `final_result` tool and returns the answer as that tool's arguments, + // keeping the real tools callable. Bedrock is deliberately excluded — it + // runs on the third-party @ai-sdk/amazon-bedrock model, which has no such + // handling. + const finalResultSchema = + this.providerName === "anthropic" && + !!options.schema && + shouldUseTools && + Object.keys(tools).length > 0 + ? (convertZodToJsonSchema(options.schema as ZodUnknownSchema) as Record< + string, + unknown + >) + : undefined; const providerOptions = buildProviderOptions( options, isGoogleProvider, callerTimeoutMs, + finalResultSchema, ); // Hoist system-role messages into generateText's top-level `system` option diff --git a/src/lib/core/modules/structuredOutputPolicy.ts b/src/lib/core/modules/structuredOutputPolicy.ts index 09ec70f49..1d2308673 100644 --- a/src/lib/core/modules/structuredOutputPolicy.ts +++ b/src/lib/core/modules/structuredOutputPolicy.ts @@ -38,6 +38,14 @@ export function isGeminiProvider( * experimental_output + tools silently drops tool_use blocks on this surface, so * structured output must be disabled when tools are active. Vertex+Claude is NOT * matched here (different transport, no conflict). + * + * Being excluded here no longer means the schema is LOST for provider + * "anthropic": GenerationHandler forwards the JSON Schema to the provider via + * `providerOptions.anthropic.finalResultSchema`, and the provider appends an + * additive `final_result` tool (see providers/anthropic/structuredOutput.ts) — + * schema enforcement without giving up tool calling. "bedrock" has no such + * handling (it runs on the third-party @ai-sdk/amazon-bedrock model) and still + * falls back to text-mode coercion. */ export function isNativeAnthropicProvider(providerName: string): boolean { return providerName === "anthropic" || providerName === "bedrock"; diff --git a/src/lib/providers/anthropic/client.ts b/src/lib/providers/anthropic/client.ts index 9dfc64569..cd712a7c2 100644 --- a/src/lib/providers/anthropic/client.ts +++ b/src/lib/providers/anthropic/client.ts @@ -55,6 +55,7 @@ import type { ClaudeSubscriptionTier, ClaudeUsageInfo, OAuthToken, + ZodUnknownSchema, } from "../../types/index.js"; import { AuthenticationError, @@ -108,6 +109,12 @@ import { } from "../openaiChatCompletionsClient.js"; import { ANTHROPIC_BETA_HEADERS } from "./constants.js"; +import { + appendFinalResultInstruction, + appendFinalResultTool, + FINAL_RESULT_TOOL_NAME, + stringifyFinalResultInput, +} from "./structuredOutput.js"; // AnthropicProviderConfig is imported from types/providers.ts // Re-export for backward compatibility @@ -1382,9 +1389,13 @@ export class AnthropicProvider extends BaseProvider { } & Record, ) => { await refreshAuth(); - const { system, messages } = messagesToAnthropic( + const built = messagesToAnthropic( options.prompt as Array<{ role: string; content: unknown }>, ); + const messages = built.messages; + // `let`: the additive structured-output path below appends the + // final_result instruction to the system prompt. + let system = built.system; let tools: Anthropic.Messages.Tool[] | undefined = (options.tools ?? []) .filter((t) => t.type === "function") @@ -1434,6 +1445,25 @@ export class AnthropicProvider extends BaseProvider { toolChoice = { type: "tool", name: jsonTool }; } + // Additive structured output: when the caller wants a schema AND real + // tools, the forced-json path above cannot be used (it replaces the + // tools array), and the AI-SDK experimental_output path is excluded + // for this surface by structuredOutputPolicy. GenerationHandler hands + // the JSON Schema down here instead, and we APPEND a `final_result` + // tool — tool_choice stays auto, so every real tool keeps working and + // the model self-selects final_result when it is ready to answer. + const finalResultSchema = options.providerOptions?.anthropic + ?.finalResultSchema as Record | undefined; + let finalResultActive = false; + if (!jsonTool && finalResultSchema) { + const appended = appendFinalResultTool(tools, finalResultSchema); + tools = appended.tools; + finalResultActive = appended.applied; + if (appended.applied) { + system = appendFinalResultInstruction(system); + } + } + // Extended thinking passthrough (providerOptions.anthropic.thinking). const thinking = options.providerOptions?.anthropic?.thinking as | { type: "enabled"; budget_tokens: number } @@ -1528,6 +1558,7 @@ export class AnthropicProvider extends BaseProvider { } const content: Array<{ type: string } & Record> = []; + let finalResultText: string | undefined; for (const block of response.content) { if (block.type === "thinking") { content.push({ type: "reasoning", text: block.thinking }); @@ -1544,6 +1575,13 @@ export class AnthropicProvider extends BaseProvider { type: "text", text: stringifyToolInput(block.input), }); + } else if ( + finalResultActive && + block.name === FINAL_RESULT_TOOL_NAME + ) { + // Internal pattern: never surfaced as a tool call. Its arguments + // ARE the structured answer. + finalResultText = stringifyToolInput(block.input); } else { content.push({ type: "tool-call", @@ -1555,12 +1593,34 @@ export class AnthropicProvider extends BaseProvider { } } + // final_result is terminal — parity with the native Claude-on-Vertex + // and Gemini loops, which break out of the tool loop the moment it + // arrives. Reasoning blocks are kept; any prose preamble and any tool + // calls issued alongside it are dropped so `text` is exactly the + // structured payload and the AI-SDK loop stops here. + if (finalResultText !== undefined) { + const reasoning = content.filter((part) => part.type === "reasoning"); + content.length = 0; + content.push(...reasoning, { type: "text", text: finalResultText }); + logger.debug( + "[Anthropic] Extracted structured output from final_result tool (generate)", + { chars: finalResultText.length }, + ); + } + const cacheRead = response.usage.cache_read_input_tokens ?? 0; const cacheWrite = response.usage.cache_creation_input_tokens ?? 0; return { content, finishReason: { - unified: mapAnthropicStopReason(response.stop_reason), + // A final_result call ends the turn: the provider reports + // stop_reason "tool_use", but no tool call is surfaced, so + // reporting "tool-calls" would misread as a step-capped turn. + // `raw` still carries the provider's verbatim stop_reason. + unified: + finalResultText !== undefined + ? ("stop" as const) + : mapAnthropicStopReason(response.stop_reason), raw: response.stop_reason ?? "stop", }, usage: { @@ -1745,6 +1805,9 @@ export class AnthropicProvider extends BaseProvider { messages: Anthropic.Messages.MessageParam[]; }; let shouldUseTools: boolean; + // True once the additive `final_result` tool is in the request — the + // streaming twin of the doGenerate path above. + let finalResultActive = false; try { // options.tools is pre-merged by BaseProvider.stream() with base tools // (MCP/built-in) + user-provided tools (RAG, etc.) @@ -1761,6 +1824,24 @@ export class AnthropicProvider extends BaseProvider { payload = messagesToAnthropic( built as Array<{ role: string; content: unknown }>, ); + // Schema + tools: append final_result rather than pinning tool_choice to + // a json tool, so the real tools stay callable for the whole turn. + // Unlike generate, no plumbing is needed — this is a native loop, so the + // caller's Zod/JSON schema is right here on the options. + if (options.schema && anthropicTools && anthropicTools.length > 0) { + const appended = appendFinalResultTool( + anthropicTools, + convertZodToJsonSchema(options.schema as ZodUnknownSchema) as Record< + string, + unknown + >, + ); + anthropicTools = appended.tools; + finalResultActive = appended.applied; + if (appended.applied) { + payload.system = appendFinalResultInstruction(payload.system); + } + } } catch (setupErr) { timeoutController?.cleanup(); throw this.handleProviderError(setupErr); @@ -1869,6 +1950,15 @@ export class AnthropicProvider extends BaseProvider { ...(totalCacheWrite > 0 ? { cacheCreationTokens: totalCacheWrite } : {}), }); + // Structured-output turns are delivered as ONE chunk, not incrementally: + // a caller that passed a schema needs parseable JSON, and text deltas + // emitted before the model calls final_result would prefix the payload + // with prose and break every JSON.parse on the consumer side. Same + // contract as the native Vertex loops. Non-schema streams are untouched + // and stay fully incremental. + let bufferedText = ""; + let finalResultText: string | undefined; + const runLoop = async (): Promise => { const conversation = payload.messages.slice(); @@ -1984,7 +2074,11 @@ export class AnthropicProvider extends BaseProvider { event.index, (textAcc.get(event.index) ?? "") + delta.text, ); - pushChunk({ content: delta.text }); + if (finalResultActive) { + bufferedText += delta.text; + } else { + pushChunk({ content: delta.text }); + } } else if (delta.type === "thinking_delta") { const acc = thinkingAcc.get(event.index) ?? { text: "", @@ -2018,6 +2112,26 @@ export class AnthropicProvider extends BaseProvider { } lastStop = stopReason; + // final_result is terminal: its arguments ARE the answer, so the turn + // ends here and any tool calls issued alongside it are not executed + // (parity with the native Vertex loops). It is never executed as a + // tool, never recorded in toolsUsed, and never stored as a tool + // execution — the pattern stays invisible to callers. + if (finalResultActive) { + const finalCall = [...toolAcc.values()].find( + (acc) => acc.name === FINAL_RESULT_TOOL_NAME, + ); + if (finalCall) { + finalResultText = stringifyFinalResultInput(finalCall.inputJson); + lastStop = "end_turn"; + logger.debug( + "[Anthropic] Extracted structured output from final_result tool (stream)", + { chars: finalResultText.length }, + ); + break; + } + } + if (stopReason !== "tool_use" || toolAcc.size === 0) { break; } @@ -2191,6 +2305,19 @@ export class AnthropicProvider extends BaseProvider { throw this.formatProviderError(error); }) .finally(() => { + // Deliver the buffered structured-output turn: `finalResultText` when + // the model called final_result, otherwise the prose it produced + // instead — never nothing, so a model that ignores the instruction + // degrades to today's plain-text behaviour rather than an empty + // stream. In `finally` so a turn that dies mid-loop still surfaces + // the text it had already buffered, exactly as the unbuffered path + // surfaces its partial deltas. + if (finalResultActive) { + const output = finalResultText ?? bufferedText; + if (output.length > 0) { + pushChunk({ content: output }); + } + } timeoutController?.cleanup(); pushChunk({ done: true }); }); diff --git a/src/lib/providers/anthropic/structuredOutput.ts b/src/lib/providers/anthropic/structuredOutput.ts new file mode 100644 index 000000000..72f8ba895 --- /dev/null +++ b/src/lib/providers/anthropic/structuredOutput.ts @@ -0,0 +1,118 @@ +import type Anthropic from "@anthropic-ai/sdk"; +import { logger } from "../../utils/logger.js"; +import { inlineJsonSchema } from "../../utils/schemaConversion.js"; + +/** + * Additive structured output for the native Anthropic Messages API. + * + * Anthropic has no `response_format`, so a schema has to be expressed as a + * tool. The provider's pre-existing `responseFormat` path does that by + * REPLACING the tools array with a single json tool and pinning `tool_choice` + * to it — correct for a schema-only call, but mutually exclusive with real + * tools, so agent/MCP turns that pass both silently lost the schema. + * + * The additive pattern here APPENDS a `final_result` tool to the caller's + * tools and leaves `tool_choice` on auto: the model keeps calling real tools + * for as long as it needs, then emits its answer as `final_result` arguments + * that already conform to the schema. This mirrors the native + * Claude-on-Vertex loop in `googleVertex/client.ts`, which has used the same + * tool name, description, and instruction wording since it shipped. + */ + +/** Internal tool name — filtered out of every returned tool call / execution. */ +export const FINAL_RESULT_TOOL_NAME = "final_result"; + +const FINAL_RESULT_TOOL_DESCRIPTION = + "Return the final structured result. You MUST call this tool when you have gathered all information and are ready to provide the final answer. The arguments should contain the structured data matching the expected schema."; + +/** Appended to the system prompt whenever the final_result tool is in play. */ +export const FINAL_RESULT_INSTRUCTION = + "\n\nIMPORTANT: You MUST call the 'final_result' tool to return your response in the required structured format. Do not respond with plain text - always use the final_result tool."; + +/** + * Build the `final_result` tool definition from a JSON Schema. + * + * `$ref`s are inlined and `$schema` dropped — Anthropic's `input_schema` must + * be a self-contained object schema. Schemas that are not object-rooted (a + * bare array/string schema) are wrapped so `input_schema.type` is always + * "object", which the Messages API requires. + */ +export function buildFinalResultTool( + jsonSchema: Record, +): Anthropic.Messages.Tool { + const inlined = inlineJsonSchema({ ...jsonSchema }); + delete inlined.$schema; + + const properties = inlined.properties as Record | undefined; + const input_schema = { + type: "object", + properties: properties ?? inlined, + required: Array.isArray(inlined.required) ? inlined.required : [], + } as Anthropic.Messages.Tool.InputSchema; + + return { + name: FINAL_RESULT_TOOL_NAME, + description: FINAL_RESULT_TOOL_DESCRIPTION, + input_schema, + }; +} + +/** + * Append `final_result` to an Anthropic tool list. + * + * Returns a NEW array so the caller's tool list is never mutated, and reports + * `applied: false` (with the list unchanged) when the pattern must not run: + * there are no real tools to preserve, or the caller already exposes a tool of + * that name — shadowing a caller's tool would break their turn. + */ +export function appendFinalResultTool( + tools: Anthropic.Messages.Tool[] | undefined, + jsonSchema: Record, +): { tools: Anthropic.Messages.Tool[] | undefined; applied: boolean } { + if (!tools || tools.length === 0) { + return { tools, applied: false }; + } + if (tools.some((tool) => tool.name === FINAL_RESULT_TOOL_NAME)) { + logger.warn( + "[Anthropic] A caller tool is already named 'final_result'; skipping the additive structured-output tool", + ); + return { tools, applied: false }; + } + // Appended LAST so any cache_control breakpoint an upstream layer placed on + // the previously-last tool keeps marking the same prefix boundary. + return { tools: [...tools, buildFinalResultTool(jsonSchema)], applied: true }; +} + +/** + * Append the final_result instruction to an Anthropic `system` value. + * + * The block-array form gets a NEW trailing block rather than an edit to the + * existing one: rewriting a block that carries a `cache_control` marker would + * change the cached prefix and invalidate the prompt cache on every turn. + */ +export function appendFinalResultInstruction( + system: string | Anthropic.Messages.TextBlockParam[] | undefined, +): string | Anthropic.Messages.TextBlockParam[] { + if (system === undefined) { + return FINAL_RESULT_INSTRUCTION.trim(); + } + if (typeof system === "string") { + return system + FINAL_RESULT_INSTRUCTION; + } + return [...system, { type: "text", text: FINAL_RESULT_INSTRUCTION.trim() }]; +} + +/** + * Canonical JSON text for a `final_result` payload. + * + * Accepts the raw accumulated `input_json` from a stream so a payload + * truncated by the token cap is still returned verbatim — the caller's + * coercion layer can repair it, whereas dropping it loses the whole answer. + */ +export function stringifyFinalResultInput(inputJson: string): string { + try { + return JSON.stringify(JSON.parse(inputJson || "{}")); + } catch { + return inputJson; + } +} diff --git a/src/lib/types/generate.ts b/src/lib/types/generate.ts index e83ea1712..f06b83b11 100644 --- a/src/lib/types/generate.ts +++ b/src/lib/types/generate.ts @@ -295,6 +295,11 @@ export type GenerateOptions = { * (see `coerceJsonToSchema`), and `disableTools: true` remains available as * an explicit override. * + * On the native Anthropic Messages surface (provider "anthropic", including + * via a proxy) tools + schema are honored through an internal `final_result` + * tool the model calls with the structured answer — invisible to callers: it + * never appears in `toolCalls` / `toolExecutions`. + * * @example * ```typescript * // ✅ Vertex + Claude: tools AND schema together are fully supported @@ -304,6 +309,12 @@ export type GenerateOptions = { * model: "claude-sonnet-4-6", * }); * + * // ✅ Direct Anthropic + tools: schema honored via the final_result tool + * const result = await neurolink.generate({ + * schema: MySchema, + * provider: "anthropic", + * }); + * * // ✅ Gemini + tools: SDK auto-falls back to coerced text-mode JSON * const result = await neurolink.generate({ * schema: MySchema, diff --git a/test/continuous-test-suite-anthropic-structured-tools.ts b/test/continuous-test-suite-anthropic-structured-tools.ts new file mode 100644 index 000000000..5285f8c5c --- /dev/null +++ b/test/continuous-test-suite-anthropic-structured-tools.ts @@ -0,0 +1,841 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: additive structured output on the native Anthropic + * Messages surface (pure, no API). + * + * The provider's only structured path used to be `responseFormat.type === + * "json"`, which REPLACES the tools array with a single json tool and pins + * `tool_choice` to it — mutually exclusive with real tools, so any caller + * passing `schema` alongside tools (every agent/MCP turn) silently lost the + * schema and got prose back. + * + * The fix appends a `final_result` tool instead, leaving `tool_choice` on + * auto. Covered here: + * - the pure helpers (tool shape, append guards, system instruction, + * payload stringification) + * - doGenerate: schema + tools (final_result appended LAST, real tools + * intact, tool_choice still auto), the final_result response unwrap + * (single text part, zero tool calls, finishReason "stop"), real tool + * calls still passing through, schema WITHOUT tools (the pre-existing + * forced-json path, unchanged), and no schema at all + * - executeStream: the same three cases, plus the single-chunk delivery + * contract and the prose fallback when the model ignores final_result + * - GenerationHandler: the providerOptions channel that carries the JSON + * Schema to the provider, and the providers it must NOT fire for + * + * Driven through a mock-injected `client.messages.create` (the private + * `client` field is overridden with a cast, matching the precedent in + * continuous-test-suite-anthropic-cap.ts). + * + * Runner: `npx tsx test/continuous-test-suite-anthropic-structured-tools.ts` + * (package.json: `pnpm run test:anthropic-structured`). + */ +import "dotenv/config"; + +import { z } from "zod"; + +import { defineSuite, assert, assertEqual } from "./helpers/harness.js"; +import { + appendFinalResultInstruction, + appendFinalResultTool, + buildFinalResultTool, + FINAL_RESULT_INSTRUCTION, + FINAL_RESULT_TOOL_NAME, + stringifyFinalResultInput, +} from "../src/lib/providers/anthropic/structuredOutput.js"; +import { AnthropicProvider } from "../src/lib/providers/anthropic/client.js"; +import { GenerationHandler } from "../src/lib/core/modules/GenerationHandler.js"; +import { convertZodToJsonSchema } from "../src/lib/utils/schemaConversion.js"; + +const { test, runSuite } = defineSuite("Anthropic additive structured output"); + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +/** The envelope shape production callers pass on every turn. */ +const ENVELOPE_SCHEMA = z.object({ + summary: z.string(), + blocks: z.array(z.string()), + attachment: z.string().optional(), +}); + +const ENVELOPE_JSON_SCHEMA = convertZodToJsonSchema( + ENVELOPE_SCHEMA as never, +) as Record; + +const ENVELOPE_PAYLOAD = { + summary: "deploy finished", + blocks: ["section-1", "section-2"], +}; + +/** A caller tool that must survive the additive pattern untouched. */ +const SEARCH_TOOL = { + type: "function" as const, + name: "search_docs", + description: "Search the docs", + inputSchema: { type: "object", properties: { q: { type: "string" } } }, +}; + +type MessagesCreateParams = { + tools?: Array<{ name: string; input_schema?: unknown }>; + tool_choice?: { type: string; name?: string }; + system?: string | Array<{ type: string; text: string }>; + stream?: boolean; +}; + +/** Records every request the provider issues and replays canned responses. */ +function mockClient(responses: unknown[]): { + client: { messages: { create: (params: unknown) => Promise } }; + requests: MessagesCreateParams[]; +} { + const requests: MessagesCreateParams[] = []; + let index = 0; + return { + requests, + client: { + messages: { + create: async (params: unknown) => { + requests.push(params as MessagesCreateParams); + const next = responses[Math.min(index, responses.length - 1)]; + index++; + return next; + }, + }, + }, + }; +} + +/** Build a provider with the network client replaced by `client`. */ +function providerWith(client: unknown): AnthropicProvider { + const provider = new AnthropicProvider( + "claude-sonnet-4-6", + undefined, + undefined, + { apiKey: "test-key-not-used" }, + ); + (provider as unknown as { client: unknown }).client = client; + return provider; +} + +/** A non-streaming Messages response. */ +function messagesResponse( + content: Array>, + stopReason: string, +): Record { + return { + id: "msg_test", + model: "claude-sonnet-4-6", + content, + stop_reason: stopReason, + usage: { input_tokens: 10, output_tokens: 5 }, + }; +} + +/** An SSE event sequence for one streamed step. */ +function streamEvents( + blocks: Array< + | { kind: "text"; text: string } + | { kind: "tool"; id: string; name: string; input: string } + >, + stopReason: string, +): AsyncIterable { + const events: unknown[] = [ + { + type: "message_start", + message: { usage: { input_tokens: 10, output_tokens: 0 } }, + }, + ]; + blocks.forEach((block, index) => { + if (block.kind === "text") { + events.push({ + type: "content_block_start", + index, + content_block: { type: "text" }, + }); + events.push({ + type: "content_block_delta", + index, + delta: { type: "text_delta", text: block.text }, + }); + } else { + events.push({ + type: "content_block_start", + index, + content_block: { type: "tool_use", id: block.id, name: block.name }, + }); + events.push({ + type: "content_block_delta", + index, + delta: { type: "input_json_delta", partial_json: block.input }, + }); + } + }); + events.push({ + type: "message_delta", + delta: { stop_reason: stopReason }, + usage: { output_tokens: 5 }, + }); + return { + async *[Symbol.asyncIterator]() { + for (const event of events) { + yield event; + } + }, + }; +} + +/** Drain a StreamResult into the list of non-empty content chunks. */ +async function drain(stream: AsyncIterable): Promise { + const chunks: string[] = []; + for await (const chunk of stream) { + const content = (chunk as { content?: string }).content; + if (typeof content === "string" && content.length > 0) { + chunks.push(content); + } + } + return chunks; +} + +/** doGenerate call options shared by the generate-path tests. */ +function generateOptions( + extra: Record, +): Record { + return { + prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }], + ...extra, + }; +} + +// --------------------------------------------------------------------------- +// Pure helpers +// --------------------------------------------------------------------------- + +await test("buildFinalResultTool produces an object-rooted input_schema", () => { + const tool = buildFinalResultTool(ENVELOPE_JSON_SCHEMA); + assertEqual(tool.name, FINAL_RESULT_TOOL_NAME, "tool name"); + assert( + (tool.description ?? "").includes("final structured result"), + "description states the contract", + ); + const schema = tool.input_schema as unknown as { + type: string; + properties: Record; + required: string[]; + }; + assertEqual(schema.type, "object", "input_schema is object-rooted"); + assertEqual( + Object.keys(schema.properties).sort().join(","), + "attachment,blocks,summary", + "every schema property is carried through", + ); + assertEqual( + schema.required.sort().join(","), + "blocks,summary", + "required list is carried through (optional field excluded)", + ); + assert(!("$schema" in schema), "the $schema keyword is dropped"); +}); + +await test("appendFinalResultTool appends LAST and leaves the input untouched", () => { + const original = [ + { name: "a", input_schema: { type: "object" as const } }, + { name: "b", input_schema: { type: "object" as const } }, + ]; + const result = appendFinalResultTool(original, ENVELOPE_JSON_SCHEMA); + assertEqual(result.applied, true, "applied"); + assertEqual( + (result.tools ?? []).map((t) => t.name).join(","), + `a,b,${FINAL_RESULT_TOOL_NAME}`, + "real tools preserved in order, final_result last", + ); + assertEqual(original.length, 2, "caller's array is not mutated"); +}); + +await test("appendFinalResultTool declines when there is nothing to preserve", () => { + assertEqual( + appendFinalResultTool([], ENVELOPE_JSON_SCHEMA).applied, + false, + "empty tool list → not applied", + ); + assertEqual( + appendFinalResultTool(undefined, ENVELOPE_JSON_SCHEMA).applied, + false, + "no tool list → not applied", + ); +}); + +await test("appendFinalResultTool never shadows a caller tool of the same name", () => { + const callerTools = [ + { name: FINAL_RESULT_TOOL_NAME, input_schema: { type: "object" as const } }, + ]; + const result = appendFinalResultTool(callerTools, ENVELOPE_JSON_SCHEMA); + assertEqual(result.applied, false, "not applied"); + assertEqual((result.tools ?? []).length, 1, "caller's tool list unchanged"); +}); + +await test("appendFinalResultInstruction handles both system shapes", () => { + assertEqual( + appendFinalResultInstruction("base"), + "base" + FINAL_RESULT_INSTRUCTION, + "string system is concatenated", + ); + const cached = [ + { + type: "text" as const, + text: "cached prefix", + cache_control: { type: "ephemeral" as const }, + }, + ]; + const blocks = appendFinalResultInstruction(cached) as Array<{ + text: string; + cache_control?: unknown; + }>; + assertEqual(blocks.length, 2, "a NEW block is appended"); + assertEqual(blocks[0].text, "cached prefix", "cached block text unchanged"); + assert( + blocks[0].cache_control !== undefined, + "cached block keeps its breakpoint", + ); + assert( + blocks[1].text.includes(FINAL_RESULT_TOOL_NAME), + "new block carries the instruction", + ); + assertEqual(cached.length, 1, "caller's block array is not mutated"); +}); + +await test("stringifyFinalResultInput canonicalizes, and keeps truncated JSON", () => { + assertEqual( + stringifyFinalResultInput('{ "a" : 1 }'), + '{"a":1}', + "well-formed input is canonicalized", + ); + assertEqual( + stringifyFinalResultInput(""), + "{}", + "empty input → empty object", + ); + // A payload cut off by the token cap must still reach the caller's + // coercion/repair layer rather than being dropped. + assertEqual( + stringifyFinalResultInput('{"summary":"tru'), + '{"summary":"tru', + "truncated input is returned verbatim", + ); +}); + +// --------------------------------------------------------------------------- +// doGenerate +// --------------------------------------------------------------------------- + +await test("doGenerate: schema + tools appends final_result and keeps tool_choice auto", async () => { + const { client, requests } = mockClient([ + messagesResponse( + [ + { + type: "tool_use", + id: "tu_1", + name: FINAL_RESULT_TOOL_NAME, + input: ENVELOPE_PAYLOAD, + }, + ], + "tool_use", + ), + ]); + const model = providerWith(client).getAISDKModel(); + await ( + model as unknown as { + doGenerate: (o: unknown) => Promise; + } + ).doGenerate( + generateOptions({ + tools: [SEARCH_TOOL], + providerOptions: { + anthropic: { finalResultSchema: ENVELOPE_JSON_SCHEMA }, + }, + }), + ); + + const sent = requests[0]; + assertEqual( + (sent.tools ?? []).map((t) => t.name).join(","), + `search_docs,${FINAL_RESULT_TOOL_NAME}`, + "real tool intact, final_result appended last", + ); + assertEqual( + sent.tool_choice, + undefined, + "tool_choice stays auto so real tools remain callable", + ); + assert( + typeof sent.system === "string" && + sent.system.includes(FINAL_RESULT_TOOL_NAME), + "system prompt carries the final_result instruction", + ); +}); + +await test("doGenerate: final_result is unwrapped to text and never surfaced as a tool call", async () => { + const { client } = mockClient([ + messagesResponse( + [ + { type: "thinking", thinking: "considering" }, + { type: "text", text: "Here you go:" }, + { + type: "tool_use", + id: "tu_1", + name: FINAL_RESULT_TOOL_NAME, + input: ENVELOPE_PAYLOAD, + }, + ], + "tool_use", + ), + ]); + const model = providerWith(client).getAISDKModel(); + const result = (await ( + model as unknown as { doGenerate: (o: unknown) => Promise } + ).doGenerate( + generateOptions({ + tools: [SEARCH_TOOL], + providerOptions: { + anthropic: { finalResultSchema: ENVELOPE_JSON_SCHEMA }, + }, + }), + )) as { + content: Array<{ type: string; text?: string; toolName?: string }>; + finishReason: { unified: string; raw: string }; + }; + + assertEqual( + result.content.filter((p) => p.type === "tool-call").length, + 0, + "final_result is filtered out of the returned tool calls", + ); + const textParts = result.content.filter((p) => p.type === "text"); + assertEqual(textParts.length, 1, "exactly one text part"); + assertEqual( + textParts[0].text, + JSON.stringify(ENVELOPE_PAYLOAD), + "text part is exactly the structured payload (prose preamble dropped)", + ); + assertEqual( + result.content.filter((p) => p.type === "reasoning").length, + 1, + "reasoning blocks are preserved", + ); + assertEqual( + result.finishReason.unified, + "stop", + "a final_result turn reports a completed turn, not tool-calls", + ); + assertEqual( + result.finishReason.raw, + "tool_use", + "raw stop_reason stays verbatim", + ); +}); + +await test("doGenerate: real tool calls still pass through while a schema is active", async () => { + const { client } = mockClient([ + messagesResponse( + [ + { + type: "tool_use", + id: "tu_1", + name: "search_docs", + input: { q: "deploys" }, + }, + ], + "tool_use", + ), + ]); + const model = providerWith(client).getAISDKModel(); + const result = (await ( + model as unknown as { doGenerate: (o: unknown) => Promise } + ).doGenerate( + generateOptions({ + tools: [SEARCH_TOOL], + providerOptions: { + anthropic: { finalResultSchema: ENVELOPE_JSON_SCHEMA }, + }, + }), + )) as { + content: Array<{ type: string; toolName?: string }>; + finishReason: { unified: string }; + }; + + const toolCalls = result.content.filter((p) => p.type === "tool-call"); + assertEqual(toolCalls.length, 1, "the real tool call survives"); + assertEqual(toolCalls[0].toolName, "search_docs", "tool name preserved"); + assertEqual( + result.finishReason.unified, + "tool-calls", + "the agent loop is allowed to continue", + ); +}); + +await test("doGenerate: schema WITHOUT tools keeps the forced-json path unchanged", async () => { + const { client, requests } = mockClient([ + messagesResponse( + [{ type: "tool_use", id: "tu_1", name: "json", input: ENVELOPE_PAYLOAD }], + "tool_use", + ), + ]); + const model = providerWith(client).getAISDKModel(); + const result = (await ( + model as unknown as { doGenerate: (o: unknown) => Promise } + ).doGenerate( + generateOptions({ + responseFormat: { type: "json", schema: ENVELOPE_JSON_SCHEMA }, + }), + )) as { content: Array<{ type: string; text?: string }> }; + + const sent = requests[0]; + assertEqual( + (sent.tools ?? []).map((t) => t.name).join(","), + "json", + "still a single forced json tool — no final_result", + ); + assertEqual( + sent.tool_choice?.name, + "json", + "tool_choice is still pinned to the json tool", + ); + assertEqual( + result.content[0]?.text, + JSON.stringify(ENVELOPE_PAYLOAD), + "payload still unwrapped into text", + ); +}); + +await test("doGenerate: no schema leaves the request and the tool calls alone", async () => { + const { client, requests } = mockClient([ + messagesResponse( + [ + { + type: "tool_use", + id: "tu_1", + name: "search_docs", + input: { q: "x" }, + }, + ], + "tool_use", + ), + ]); + const model = providerWith(client).getAISDKModel(); + const result = (await ( + model as unknown as { doGenerate: (o: unknown) => Promise } + ).doGenerate(generateOptions({ tools: [SEARCH_TOOL] }))) as { + content: Array<{ type: string; toolName?: string }>; + finishReason: { unified: string }; + }; + + assertEqual( + (requests[0].tools ?? []).map((t) => t.name).join(","), + "search_docs", + "no final_result tool is added", + ); + assert( + !String(requests[0].system ?? "").includes(FINAL_RESULT_TOOL_NAME), + "no final_result instruction is added", + ); + assertEqual( + result.content.filter((p) => p.type === "tool-call").length, + 1, + "tool call passes through", + ); + assertEqual( + result.finishReason.unified, + "tool-calls", + "finish reason untouched", + ); +}); + +// --------------------------------------------------------------------------- +// executeStream +// --------------------------------------------------------------------------- + +await test("executeStream: schema + tools appends final_result and yields the payload", async () => { + const { client, requests } = mockClient([ + streamEvents( + [ + { + kind: "tool", + id: "tu_1", + name: FINAL_RESULT_TOOL_NAME, + input: JSON.stringify(ENVELOPE_PAYLOAD), + }, + ], + "tool_use", + ), + ]); + const provider = providerWith(client); + const result = await provider.stream({ + input: { text: "summarize" }, + schema: ENVELOPE_SCHEMA, + tools: { search_docs: { description: "Search", inputSchema: {} } }, + disableTools: false, + } as never); + + const chunks = await drain(result.stream); + const sent = requests[0]; + // BaseProvider.stream() merges built-in/MCP tools in, so assert on + // membership and position rather than on an exact tool list. + const names = (sent.tools ?? []).map((t) => t.name); + assertEqual(sent.stream, true, "streaming request"); + assert(names.includes("search_docs"), "the caller's tool is still declared"); + assertEqual( + names[names.length - 1], + FINAL_RESULT_TOOL_NAME, + "final_result is appended LAST", + ); + assertEqual( + names.filter((n) => n === FINAL_RESULT_TOOL_NAME).length, + 1, + "final_result is declared exactly once", + ); + assertEqual(sent.tool_choice, undefined, "tool_choice stays auto"); + assertEqual(chunks.length, 1, "structured turns deliver exactly one chunk"); + assertEqual( + chunks[0], + JSON.stringify(ENVELOPE_PAYLOAD), + "the chunk is exactly the structured payload", + ); +}); + +await test("executeStream: prose is still delivered when the model skips final_result", async () => { + const { client } = mockClient([ + streamEvents( + [ + { kind: "text", text: "sorry, " }, + { kind: "text", text: "no JSON today" }, + ], + "end_turn", + ), + ]); + const provider = providerWith(client); + const result = await provider.stream({ + input: { text: "summarize" }, + schema: ENVELOPE_SCHEMA, + tools: { search_docs: { description: "Search", inputSchema: {} } }, + } as never); + + const chunks = await drain(result.stream); + assertEqual(chunks.length, 1, "buffered into a single chunk"); + assertEqual( + chunks[0], + "sorry, no JSON today", + "no text is lost when the model ignores the instruction", + ); +}); + +await test("executeStream: no schema keeps incremental delivery and adds no tool", async () => { + const { client, requests } = mockClient([ + streamEvents( + [ + { kind: "text", text: "chunk one " }, + { kind: "text", text: "chunk two" }, + ], + "end_turn", + ), + ]); + const provider = providerWith(client); + const result = await provider.stream({ + input: { text: "hi" }, + tools: { search_docs: { description: "Search", inputSchema: {} } }, + } as never); + + const chunks = await drain(result.stream); + const names = (requests[0].tools ?? []).map((t) => t.name); + assert(names.includes("search_docs"), "the caller's tool is still declared"); + assert( + !names.includes(FINAL_RESULT_TOOL_NAME), + "no final_result tool is added", + ); + assert( + !String(requests[0].system ?? "").includes(FINAL_RESULT_TOOL_NAME), + "no final_result instruction is added", + ); + assertEqual(chunks.length, 2, "text deltas still stream incrementally"); +}); + +// --------------------------------------------------------------------------- +// End-to-end: the contract callers actually observe +// --------------------------------------------------------------------------- + +await test("generate(): a multi-step schema+tools turn returns parsed structuredData", async () => { + // Step 1 calls a real tool; step 2 answers with final_result. This is the + // production shape: before the fix the schema was dropped entirely and the + // turn came back as prose. + const { client, requests } = mockClient([ + messagesResponse( + [{ type: "tool_use", id: "t1", name: "lookup", input: { q: "x" } }], + "tool_use", + ), + messagesResponse( + [ + { + type: "tool_use", + id: "t2", + name: FINAL_RESULT_TOOL_NAME, + input: ENVELOPE_PAYLOAD, + }, + ], + "tool_use", + ), + ]); + const result = await providerWith(client).generate({ + input: { text: "summarize the deploy" }, + schema: ENVELOPE_SCHEMA, + tools: { + lookup: { + description: "Look something up", + inputSchema: z.object({ q: z.string() }), + execute: async () => ({ found: true }), + }, + }, + maxSteps: 5, + } as never); + + assertEqual(requests.length, 2, "the agent loop ran both steps"); + const step2Tools = (requests[1].tools ?? []).map((t) => t.name); + assert(step2Tools.includes("lookup"), "the real tool stayed callable"); + assertEqual( + step2Tools[step2Tools.length - 1], + FINAL_RESULT_TOOL_NAME, + "final_result declared last on every step", + ); + assertEqual(requests[1].tool_choice, undefined, "tool_choice stays auto"); + + assertEqual( + result?.content, + JSON.stringify(ENVELOPE_PAYLOAD), + "content is the structured payload", + ); + assertEqual( + JSON.stringify(result?.structuredData), + JSON.stringify(ENVELOPE_PAYLOAD), + "structuredData is the parsed object — the whole point of the fix", + ); + assertEqual( + (result?.toolsUsed ?? []).join(","), + "lookup", + "final_result is filtered out of toolsUsed; the real tool is reported", + ); + assertEqual( + (result?.toolCalls ?? []).filter( + (tc) => tc.toolName === FINAL_RESULT_TOOL_NAME, + ).length, + 0, + "final_result never appears in toolCalls", + ); + assertEqual( + (result?.toolExecutions ?? []).filter( + (te) => te.toolName === FINAL_RESULT_TOOL_NAME, + ).length, + 0, + "final_result never appears in toolExecutions", + ); + assertEqual(result?.finishReason, "stop", "turn reports a clean finish"); +}); + +// --------------------------------------------------------------------------- +// GenerationHandler → providerOptions channel +// --------------------------------------------------------------------------- + +/** + * Run one generation through GenerationHandler with `generateText` stubbed, + * and return the providerOptions the handler built. + */ +async function capturedProviderOptions( + providerName: string, + options: Record, + tools: Record, +): Promise> | undefined> { + let captured: Record> | undefined; + const handler = new GenerationHandler( + providerName as never, + "claude-sonnet-4-6", + () => true, + () => undefined, + async () => {}, + { + generateTextFn: (async (args: Record) => { + captured = args.providerOptions as + | Record> + | undefined; + return { + text: "{}", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1 }, + steps: [], + toolCalls: [], + }; + }) as never, + }, + ); + await handler.executeGeneration( + { modelId: "claude-sonnet-4-6", provider: providerName } as never, + [{ role: "user", content: "hi" }] as never, + tools as never, + options as never, + ); + return captured; +} + +await test("GenerationHandler hands the JSON Schema to the anthropic provider", async () => { + const captured = await capturedProviderOptions( + "anthropic", + { schema: ENVELOPE_SCHEMA }, + { search_docs: { description: "Search", inputSchema: {} } }, + ); + const schema = captured?.anthropic?.finalResultSchema as + | Record + | undefined; + assert(schema !== undefined, "finalResultSchema is forwarded"); + assertEqual( + Object.keys((schema?.properties ?? {}) as Record) + .sort() + .join(","), + "attachment,blocks,summary", + "the forwarded schema is the caller's schema", + ); +}); + +await test("GenerationHandler withholds the schema where the pattern must not run", async () => { + const noTools = await capturedProviderOptions( + "anthropic", + { schema: ENVELOPE_SCHEMA }, + {}, + ); + assertEqual( + noTools?.anthropic, + undefined, + "no tools → the forced-json path handles it, no final_result", + ); + + const noSchema = await capturedProviderOptions( + "anthropic", + {}, + { + search_docs: { description: "Search", inputSchema: {} }, + }, + ); + assertEqual(noSchema?.anthropic, undefined, "no schema → nothing forwarded"); + + // Vertex+Claude and Gemini have their own native handling; bedrock runs on + // the third-party @ai-sdk/amazon-bedrock model, which cannot read this. + for (const provider of ["vertex", "bedrock", "google-ai"]) { + const other = await capturedProviderOptions( + provider, + { schema: ENVELOPE_SCHEMA }, + { search_docs: { description: "Search", inputSchema: {} } }, + ); + assertEqual( + other?.anthropic, + undefined, + `${provider} → nothing forwarded on the anthropic channel`, + ); + } +}); + +await runSuite();