diff --git a/CLAUDE.md b/CLAUDE.md index f5ed35b28..16f12e2a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ These are non-negotiable. Violating them breaks the build or introduces bugs. 1. **Dynamic imports only in registry** — All providers must use dynamic imports inside factory functions in `providerRegistry.ts`. Static imports create circular dependencies. 2. **Types in canonical location** — All type definitions go in `src/lib/types/`. Never create type files inside feature subdirectories. 3. **Gemini tools + JSON schema are mutually exclusive** — Google AI Studio and Vertex **Gemini** models cannot use tools and `structuredOutput` with a JSON schema simultaneously (a Gemini API limitation). This does **not** apply to Vertex **Claude** models, which support both at once — the exclusion is gated on `isGeminiProvider` in `structuredOutputPolicy.ts`, not on the Vertex provider as a whole. Providers that reject the combination at runtime (e.g. Groq) are detected via `isToolsSchemaConflictError` and transparently retried without structured output. Regardless of provider, `generate({ schema })` is guaranteed to return valid JSON in `content` plus a parsed `structuredData` object (see `coerceJsonToSchema`). - - **Huge-text / truncation:** the native Claude paths (Vertex+Claude, direct Anthropic) must default `max_tokens` to the model's real output ceiling via `resolveClaudeMaxTokens` (Sonnet 4.x → 64K, Opus 4.x → 32K), **never** the legacy hard-coded 4096 that silently truncated large structured responses mid-JSON. The direct Anthropic non-streaming path also passes an explicit request `timeout` so the SDK's "streaming is required for long requests" pre-flight guard doesn't reject a large `max_tokens`. When output still hits the cap, truncation is surfaced — not silent: `coerceJsonToSchema` returns `{ repaired, truncated }`, and `GenerateResult` exposes `jsonRepaired` / `jsonTruncated` (set when `finishReason==="length"` or the recovered JSON came from an unclosed span) plus a WARN log. + - **Huge-text / truncation:** the native Claude paths (Vertex+Claude, direct Anthropic) must default `max_tokens` to the model's real output ceiling via `resolveClaudeMaxTokens` (Sonnet 4.x → 64K, Opus 4.x → 32K), **never** the legacy hard-coded 4096 that silently truncated large structured responses mid-JSON. The direct Anthropic non-streaming path also passes an explicit request `timeout` so the SDK's "streaming is required for long requests" pre-flight guard doesn't reject a large `max_tokens`. When output still hits the cap, truncation is surfaced — not silent: `coerceJsonToSchema` returns `{ repaired, truncated }`, and `GenerateResult` exposes `jsonRepaired` / `jsonTruncated` (set when `finishReason==="length"` or the recovered JSON came from an unclosed span) plus a WARN log. A truncated response must still yield a **partial object** — never a raw string: `coerceJsonToSchema` prefers the candidate starting at the document's real root (so a bracket pair scraped from inside a string value can't win), and backs off to the last completed field when jsonrepair can't close the span. That recovered `structuredData` is a **plain object, not necessarily a schema-valid one** — when the response was cut short it may be partial — and `jsonTruncated` is set in exactly that case (`jsonRepaired` when the JSON had to be recovered), so a caller can distinguish a salvaged object from a complete one. A caller that needs schema-valid data must check `jsonTruncated` before trusting the object; a caller that wants best-effort data can use it as is. Only schema-rejected **scalar** roots (e.g. a raw string under an object schema) are suppressed via `schemaAccepts`, since they carry no recoverable structure. 4. **CLI ≠ SDK** — CLI can use manual MCP connections; the SDK cannot. Keep concerns separate. 5. **Backward compatibility** — Public SDK API must not break existing callers. 6. **`formatProviderError` must return, never throw** — Any provider error formatter must return the error object, not throw it. diff --git a/package.json b/package.json index 8bda9f05e..009754b9b 100644 --- a/package.json +++ b/package.json @@ -135,6 +135,7 @@ "test:file-detector-magic-bytes": "npx tsx test/continuous-test-suite-file-detector-magic-bytes.ts", "test:json": "npx tsx test/continuous-test-suite-json.ts", "test:json-e2e": "npx tsx test/continuous-test-suite-json-e2e.ts", + "test:coerce-truncation": "npx tsx test/continuous-test-suite-coerce-truncation.ts", "test:workflow": "npx tsx test/continuous-test-suite-workflow.ts", "test:hitl": "npx tsx test/continuous-test-suite-hitl.ts", "test:analytics": "npx tsx test/continuous-test-suite-analytics.ts", diff --git a/src/lib/core/modules/GenerationHandler.ts b/src/lib/core/modules/GenerationHandler.ts index 40f46c9d4..396ee3d10 100644 --- a/src/lib/core/modules/GenerationHandler.ts +++ b/src/lib/core/modules/GenerationHandler.ts @@ -57,7 +57,11 @@ import { isToolsSchemaConflictError, isToolsSchemaExclusionInForce, } from "./structuredOutputPolicy.js"; -import { coerceJsonToSchema } from "../../utils/json/coerce.js"; +import { + coerceJsonToSchema, + recoverScalarRoot, + schemaAccepts, +} from "../../utils/json/coerce.js"; import { convertZodToJsonSchema } from "../../utils/schemaConversion.js"; import type { LanguageModel, @@ -1133,9 +1137,9 @@ export class GenerationHandler { } return coerced.content; } - try { - const scalar: unknown = JSON.parse(strippedText); - if (scalar === "") { + const scalar = recoverScalarRoot(strippedText, options.schema); + switch (scalar.kind) { + case "empty": // A JSON-encoded empty string is an EMPTY completion, not a // recovered scalar — normalize to a true empty ('' content, no // structuredData) so callers' empty-response handling fires @@ -1145,19 +1149,32 @@ export class GenerationHandler { { provider: this.providerName, model: this.modelName }, ); return ""; - } - if (scalar !== null && scalar !== undefined) { - structuredData = scalar; + case "accepted": + // A JSON scalar root is only real structured data when the caller's + // schema actually accepts it. Under an OBJECT schema a recovered + // string/number is the raw completion in disguise (the shape a + // truncated response degrades to) — publishing it would hand the + // caller a `structuredData` that violates the schema they passed. + structuredData = scalar.value; + return strippedText; + case "rejected": + logger.warn( + "[GenerationHandler] recovered a JSON scalar the requested schema rejects; leaving structuredData unset", + { + provider: this.providerName, + model: this.modelName, + scalarType: typeof scalar.value, + }, + ); + return strippedText; + case "nullish": + case "not-json": + logger.warn( + "[GenerationHandler] schema requested but no JSON could be recovered from model text; returning raw text", + { provider: this.providerName, model: this.modelName }, + ); return strippedText; - } - } catch { - // not JSON at all — fall through to raw text + WARN } - logger.warn( - "[GenerationHandler] schema requested but no JSON could be recovered from model text; returning raw text", - { provider: this.providerName, model: this.modelName }, - ); - return strippedText; }; if (useStructuredOutput) { try { @@ -1173,7 +1190,23 @@ export class GenerationHandler { const rawTextEcho = typeof experimentalOutput === "string" && experimentalOutput === (generateResult.text ?? ""); - if (experimentalOutput !== undefined && !rawTextEcho) { + // The equality check above only catches an EXACT echo. On a multi-step + // or truncated turn the echo can differ from `text` (a different step's + // text, a fence, trailing whitespace), and a raw string would then be + // published as `structuredData` under an object schema — the "returned + // a string instead of the schema object" failure. A string is trusted + // as structured output ONLY when the caller's schema accepts it + // (string-root schemas keep working); otherwise it is coerced like any + // other raw model text. + const untrustedStringOutput = + typeof experimentalOutput === "string" && + !!options.schema && + !schemaAccepts(options.schema, experimentalOutput); + if ( + experimentalOutput !== undefined && + !rawTextEcho && + !untrustedStringOutput + ) { // AI-SDK already parsed + schema-validated the object. Expose it // directly and serialise canonically — no hand-parsing needed. structuredData = experimentalOutput; diff --git a/src/lib/neurolink.ts b/src/lib/neurolink.ts index 9c879f507..1cb125467 100644 --- a/src/lib/neurolink.ts +++ b/src/lib/neurolink.ts @@ -94,6 +94,7 @@ import type { AnalyticsData, EvaluationData, NeurolinkCredentials, + OptionalValidationSchema, ProviderStatus, TextGenerationOptions, TextGenerationResult, @@ -264,7 +265,11 @@ import { } from "./utils/lifecycleCallbacks.js"; import { resolveLifecycleTimeoutMs } from "./utils/lifecycleTimeout.js"; import { cloneOptionsForCallIsolation } from "./utils/cloneOptions.js"; -import { coerceJsonToSchema } from "./utils/json/coerce.js"; +import { + coerceJsonToSchema, + recoverScalarRoot, + schemaAccepts, +} from "./utils/json/coerce.js"; // Factory processing imports import { createCleanStreamOptions, @@ -5453,6 +5458,98 @@ Current user's request: ${currentInput}`; return textOptions; } + /** + * Provider-agnostic JSON recovery for schema requests. Structured-output + * enforcement makes valid JSON the overwhelming case; for every other + * provider path — including generate() overrides (Vertex, Anthropic, + * Bedrock, Google AI Studio) — object/array roots are recovered here via + * balanced-scan + jsonrepair and scalar JSON roots via plain JSON.parse, + * with the parsed value exposed as `structuredData`. If nothing JSON-shaped + * is recoverable (pure prose), the raw text is returned, `structuredData` + * stays undefined, and a WARN makes the case observable. + * + * Mutates `textResult` in place, and must run BEFORE the end-of-generation + * emits so event consumers see the same content/structuredData the caller + * receives. + */ + private recoverStructuredData( + textResult: TextGenerationResult, + schema: OptionalValidationSchema, + ): void { + // A provider path that produced its own `structuredData` normally owns it. + // The one exception is a STRING the caller's schema rejects: that is the + // raw completion leaking through as structured output (the shape a + // truncated response degrades to), so re-run recovery over the text rather + // than handing back a value the declared schema forbids. + const structuredIsRejectedString = + typeof textResult.structuredData === "string" && + !schemaAccepts(schema, textResult.structuredData); + if ( + !schema || + (textResult.structuredData !== undefined && + !structuredIsRejectedString) || + typeof textResult.content !== "string" + ) { + return; + } + if (structuredIsRejectedString) { + textResult.structuredData = undefined; + } + const coerced = coerceJsonToSchema(textResult.content, schema); + if (coerced) { + textResult.content = coerced.content; + textResult.structuredData = coerced.structuredData; + if (coerced.repaired) { + textResult.jsonRepaired = true; + } + if (coerced.truncated) { + textResult.jsonTruncated = true; + } + return; + } + const scalar = recoverScalarRoot(textResult.content, schema); + switch (scalar.kind) { + case "empty": + // A JSON-encoded empty string is an EMPTY completion, not a recovered + // scalar — normalize to a true empty so callers' empty-response + // handling fires instead of a literal '""' reaching the user. + // `structuredData` stays undefined. + textResult.content = ""; + logger.warn( + "[NeuroLink] schema requested but the model returned an empty JSON string; normalizing to empty content", + { provider: textResult.provider, model: textResult.model }, + ); + break; + case "accepted": + // Only publish a scalar root the caller's schema actually accepts. + // Under an OBJECT schema a recovered string is the raw completion in + // disguise — the shape a truncated response degrades to — and exposing + // it hands the caller a `structuredData` that violates the schema they + // passed. + textResult.structuredData = scalar.value; + break; + case "rejected": + logger.warn( + "[NeuroLink] recovered a JSON scalar the requested schema rejects; leaving structuredData unset", + { + provider: textResult.provider, + model: textResult.model, + scalarType: typeof scalar.value, + }, + ); + break; + case "nullish": + // JSON null/undefined — no structured value to publish. + break; + case "not-json": + logger.warn( + "[NeuroLink] schema requested but no JSON could be recovered from model output; returning raw text", + { provider: textResult.provider, model: textResult.model }, + ); + break; + } + } + private finalizeGenerateRequestResult(params: { generateSpan: ReturnType; options: GenerateOptions; @@ -5472,58 +5569,7 @@ Current user's request: ${currentInput}`; startTime, } = params; - // Provider-agnostic JSON coercion for schema requests. Structured-output - // enforcement makes valid JSON the overwhelming case; for every other - // provider path — including generate() overrides (Vertex, Anthropic, - // Bedrock, Google AI Studio) — object/array roots are recovered here via - // balanced-scan + jsonrepair and scalar JSON roots via plain JSON.parse, - // with the parsed value exposed as `structuredData`. If nothing - // JSON-shaped is recoverable (pure prose), the raw text is returned, - // `structuredData` stays undefined, and a WARN makes the case observable. - // Runs BEFORE the end-of-generation emits below so event consumers see - // the same coerced content/structuredData the caller receives. - if ( - textOptions.schema && - textResult.structuredData === undefined && - typeof textResult.content === "string" - ) { - const coerced = coerceJsonToSchema( - textResult.content, - textOptions.schema, - ); - if (coerced) { - textResult.content = coerced.content; - textResult.structuredData = coerced.structuredData; - if (coerced.repaired) { - textResult.jsonRepaired = true; - } - if (coerced.truncated) { - textResult.jsonTruncated = true; - } - } else { - try { - const scalar: unknown = JSON.parse(textResult.content); - if (scalar === "") { - // A JSON-encoded empty string is an EMPTY completion, not a - // recovered scalar — normalize to a true empty so callers' - // empty-response handling fires instead of a literal '""' - // reaching the user. `structuredData` stays undefined. - textResult.content = ""; - logger.warn( - "[NeuroLink] schema requested but the model returned an empty JSON string; normalizing to empty content", - { provider: textResult.provider, model: textResult.model }, - ); - } else if (scalar !== null && scalar !== undefined) { - textResult.structuredData = scalar; - } - } catch { - logger.warn( - "[NeuroLink] schema requested but no JSON could be recovered from model output; returning raw text", - { provider: textResult.provider, model: textResult.model }, - ); - } - } - } + this.recoverStructuredData(textResult, textOptions.schema); // Surface truncation when a schema was requested: either the provider // reported finishReason="length" or the recovered JSON came from an diff --git a/src/lib/providers/anthropic/client.ts b/src/lib/providers/anthropic/client.ts index 74d3cd7cd..75f097b2d 100644 --- a/src/lib/providers/anthropic/client.ts +++ b/src/lib/providers/anthropic/client.ts @@ -1566,18 +1566,24 @@ export class AnthropicProvider extends BaseProvider { const content: Array<{ type: string } & Record> = []; let finalResultText: string | undefined; + // Text emitted in forced-json mode, kept only as a fallback (see below). + const jsonModeText: string[] = []; + let jsonToolAnswered = false; for (const block of response.content) { if (block.type === "thinking") { content.push({ type: "reasoning", text: block.thinking }); } else if (block.type === "text") { // In forced-json mode the payload arrives via the tool input, not // text — pass text through only in normal mode. - if (!jsonTool) { + if (jsonTool) { + jsonModeText.push(block.text); + } else { content.push({ type: "text", text: block.text }); } } else if (block.type === "tool_use") { if (jsonTool && block.name === jsonTool) { // Unwrap the synthetic tool call back into text JSON. + jsonToolAnswered = true; content.push({ type: "text", text: stringifyToolInput(block.input), @@ -1599,6 +1605,15 @@ export class AnthropicProvider extends BaseProvider { } } } + // Forced-json mode normally drops text blocks because the payload rides + // in the synthetic tool's input. But when the response is cut short + // (stop_reason "max_tokens") the tool call can be missing entirely, and + // dropping the text would leave an EMPTY completion with nothing for + // coerceJsonToSchema to recover. Fall back to the text so a partial + // object can still be salvaged and flagged truncated. + if (jsonTool && !jsonToolAnswered && jsonModeText.length > 0) { + content.push({ type: "text", text: jsonModeText.join("") }); + } // final_result is terminal — parity with the native Claude-on-Vertex // and Gemini loops, which break out of the tool loop the moment it diff --git a/src/lib/types/utilities.ts b/src/lib/types/utilities.ts index a0b102897..7be78b578 100644 --- a/src/lib/types/utilities.ts +++ b/src/lib/types/utilities.ts @@ -334,3 +334,26 @@ export type JsonCoercionResult = { */ truncated: boolean; }; + +/** + * Decision returned by `recoverScalarRoot`. Each caller applies it to its own + * result shape and logger prefix, preserving its existing warning behaviour: + * + * - `empty` — the text is a JSON-encoded empty string (an EMPTY + * completion, not a recovered scalar). Callers normalize to a + * true empty. + * - `accepted` — a scalar root the caller's schema accepts; safe to publish + * as `structuredData`. + * - `rejected` — a scalar root the caller's schema rejects (e.g. a raw + * string under an object schema — the shape a truncated + * response degrades to). Do NOT publish it. + * - `nullish` — the text is the JSON literals `null`/`undefined`; there is + * no structured value to publish. + * - `not-json` — the text is not JSON at all. + */ +export type ScalarRecoveryDecision = + | { kind: "empty" } + | { kind: "accepted"; value: unknown } + | { kind: "rejected"; value: unknown } + | { kind: "nullish" } + | { kind: "not-json" }; diff --git a/src/lib/utils/json/coerce.ts b/src/lib/utils/json/coerce.ts index a1d9846ce..ad5c94da1 100644 --- a/src/lib/utils/json/coerce.ts +++ b/src/lib/utils/json/coerce.ts @@ -19,6 +19,7 @@ import { jsonrepair } from "jsonrepair"; import type { JsonCoercionResult, + ScalarRecoveryDecision, ValidationSchema, } from "../../types/index.js"; import { logger } from "../logger.js"; @@ -29,6 +30,57 @@ function hasSafeParse(schema: ValidationSchema): boolean { return typeof (schema as { safeParse?: unknown }).safeParse === "function"; } +/** + * Does `value` satisfy `schema`? A schema we cannot validate with (absent, or + * without a Zod-style `safeParse`) accepts everything — callers use this as a + * gate, not as proof, so an unknown schema must never block a value. + * + * Exists so the consumers of coerceJsonToSchema can refuse to publish a + * recovered value as `structuredData` when the caller's schema rejects it — + * e.g. a raw *string* under an object schema, which is the shape a truncated + * response degrades to. + */ +export function schemaAccepts( + schema: ValidationSchema | undefined, + value: unknown, +): boolean { + if (!schema || !hasSafeParse(schema)) { + return true; + } + return ( + schema as { safeParse: (v: unknown) => { success: boolean } } + ).safeParse(value).success; +} + +/** + * Recover a JSON *scalar* root (string/number/boolean) from model text. The + * object/array case is handled by `coerceJsonToSchema`; this covers the + * residual scalar root after that path returns null. Encapsulates the JSON + * parsing, the empty-string normalization, and the `schemaAccepts` gate so the + * same policy cannot drift between consumers (`neurolink.recoverStructuredData` + * and `GenerationHandler.coerceTextMode`). + */ +export function recoverScalarRoot( + text: string, + schema: ValidationSchema | undefined, +): ScalarRecoveryDecision { + try { + const scalar: unknown = JSON.parse(text); + if (scalar === "") { + return { kind: "empty" }; + } + if (scalar === null || scalar === undefined) { + return { kind: "nullish" }; + } + if (schemaAccepts(schema, scalar)) { + return { kind: "accepted", value: scalar }; + } + return { kind: "rejected", value: scalar }; + } catch { + return { kind: "not-json" }; + } +} + /** * Parse `candidate` as JSON, repairing common escaping mistakes on failure. * Returns the parsed value plus whether jsonrepair had to alter the text. @@ -59,6 +111,42 @@ function parseOrRepair( /** Bounds the recursive nested-string unwrap against pathological inputs. */ const MAX_NESTED_UNWRAP_DEPTH = 6; +/** Bounds the structural back-off when salvaging a truncated root object. */ +const MAX_SALVAGE_TRIMS = 8; + +/** + * Last-resort recovery for output cut off mid-JSON: walk the unclosed root span + * backwards to successively earlier structural boundaries (`,` / `}` / `]`) and + * re-attempt parse+repair at each one. Truncation can land somewhere jsonrepair + * cannot close on its own (inside an escape, a half-written key, a dangling + * separator); dropping back to the last completed field gives it a repairable + * prefix. Returns a PARTIAL object — the fields the model did finish — so a + * truncated response degrades to an incomplete object rather than to raw text. + */ +function salvageTruncatedRoot(span: string): unknown | undefined { + let candidate = span; + for (let i = 0; i < MAX_SALVAGE_TRIMS && candidate.length > 1; i++) { + const cut = Math.max( + candidate.lastIndexOf(","), + candidate.lastIndexOf("}"), + candidate.lastIndexOf("]"), + ); + if (cut <= 0) { + break; + } + candidate = candidate.slice(0, cut); + const outcome = parseOrRepair(candidate); + if ( + outcome !== undefined && + outcome.value !== null && + typeof outcome.value === "object" + ) { + return outcome.value; + } + } + return undefined; +} + /** * Recursively replace any string-valued field whose content is itself a JSON * object/array with the parsed value. Models sometimes double-encode a NESTED @@ -151,32 +239,60 @@ export function coerceJsonToSchema( // 3. first "{" or "[" to end of text (TRUNCATED output — // finishReason=length — where the closing bracket was cut off; // jsonrepair closes it) - // `truncated` marks the first-open-to-end candidate: it is only reachable - // when no balanced span and no first-to-last span matched, i.e. there was no - // closing bracket at all — the signature of token-truncated output. - const candidates: Array<{ text: string; truncated: boolean }> = []; + // `truncated` marks a candidate that could only come from output cut short: + // the first-open-to-end span, and — when the ROOT bracket never closes — + // every other candidate too, since all of them are then partial views of an + // unfinished document. + // + // `rootAligned` marks candidates that start at the document's first opening + // bracket, i.e. at the real root. Candidates are ordered root-aligned FIRST. + // On truncated output the root brace never balances, so the balanced scan + // walks on and matches brackets that live INSIDE a string value (`[step 1]` + // in a shell script, say) — a syntactically fine but semantically bogus + // array. Preferring the root keeps the partial real object ahead of that. + const candidates: Array<{ + text: string; + truncated: boolean; + rootAligned: boolean; + }> = []; + const openIndexes = [text.indexOf("{"), text.indexOf("[")].filter( + (i) => i >= 0, + ); + const firstOpen = openIndexes.length > 0 ? Math.min(...openIndexes) : -1; + let rootClosed = false; let searchFrom = 0; for (;;) { const found = nextBalancedJsonSpan(text, searchFrom); if (!found) { break; } - candidates.push({ text: found.span, truncated: false }); + const start = found.end - found.span.length; + if (start === firstOpen) { + rootClosed = true; + } + candidates.push({ + text: found.span, + truncated: false, + rootAligned: start === firstOpen, + }); searchFrom = found.end; } - const openIndexes = [text.indexOf("{"), text.indexOf("[")].filter( - (i) => i >= 0, - ); - const firstOpen = openIndexes.length > 0 ? Math.min(...openIndexes) : -1; + // No balanced span starts at the root bracket → the document is unclosed. + const rootUnclosed = firstOpen >= 0 && !rootClosed; const lastClose = Math.max(text.lastIndexOf("}"), text.lastIndexOf("]")); if (firstOpen >= 0 && lastClose > firstOpen) { candidates.push({ text: text.slice(firstOpen, lastClose + 1), - truncated: false, + truncated: rootUnclosed, + rootAligned: true, }); } if (firstOpen >= 0) { - candidates.push({ text: text.slice(firstOpen), truncated: true }); + candidates.push({ + text: text.slice(firstOpen), + truncated: true, + rootAligned: true, + }); } // JSON-string-literal wrapper: some providers double-encode and return the @@ -193,7 +309,11 @@ export function coerceJsonToSchema( if (!innerSpan) { break; } - candidates.push({ text: innerSpan.span, truncated: false }); + candidates.push({ + text: innerSpan.span, + truncated: false, + rootAligned: false, + }); innerFrom = innerSpan.end; } } @@ -202,17 +322,33 @@ export function coerceJsonToSchema( } } + // Stable partition: root-aligned candidates first. Only the ORDER changes — + // no candidate is dropped — so the multi-object "prefer the most complete + // schema-valid candidate" selection below is unaffected (it is order + // independent), while the unschema'd first-parseable-wins path and the + // `firstValid` fallback now favour the document's real root. + const ordered = [ + ...candidates.filter((c) => c.rootAligned), + ...candidates.filter((c) => !c.rootAligned), + ]; + let firstValid: - | { value: unknown; repaired: boolean; truncated: boolean } + | { + value: unknown; + repaired: boolean; + truncated: boolean; + rootAligned: boolean; + } | undefined; const schemaValid: Array<{ value: unknown; repaired: boolean; truncated: boolean; + rootAligned: boolean; }> = []; const hasSchema = !!(schema && hasSafeParse(schema)); const seen = new Set(); - for (const candidate of candidates) { + for (const candidate of ordered) { if (seen.has(candidate.text)) { continue; } @@ -229,6 +365,7 @@ export function coerceJsonToSchema( value: outcome.value, repaired: outcome.repaired, truncated: candidate.truncated, + rootAligned: candidate.rootAligned, }; if (firstValid === undefined) { firstValid = record; @@ -259,6 +396,7 @@ export function coerceJsonToSchema( value: unwrapped.value, repaired: true, truncated: candidate.truncated, + rootAligned: candidate.rootAligned, }); } else if ( // Single-element array wrapper: for an OBJECT schema, models sometimes @@ -277,6 +415,7 @@ export function coerceJsonToSchema( value: outcome.value[0], repaired: true, truncated: candidate.truncated, + rootAligned: candidate.rootAligned, }); } } @@ -296,7 +435,35 @@ export function coerceJsonToSchema( ) : undefined; - const chosen = schemaMatch ?? firstValid; + let chosen = schemaMatch ?? firstValid; + // Salvage the root when the document is unclosed AND nothing trustworthy came + // out of it: either nothing parsed at all, or the only parseable candidate is + // a bracket pair scraped from INSIDE a string value (not root-aligned, not + // schema-valid) — the `["step 1"]`-from-a-shell-script case. Backing off to + // the last completed field yields a PARTIAL OBJECT flagged `truncated`, + // instead of a bogus value or raw text degrading to a string. + if ( + rootUnclosed && + schemaMatch === undefined && + (chosen === undefined || !chosen.rootAligned) + ) { + const salvaged = salvageTruncatedRoot(text.slice(firstOpen)); + if (salvaged !== undefined) { + logger.debug("[coerceJsonToSchema] salvaged a partial truncated object", { + textLength: text.length, + }); + chosen = { + value: salvaged, + repaired: true, + truncated: true, + rootAligned: true, + }; + } else if (chosen !== undefined) { + // Nothing salvageable, but we still know the output was truncated — + // never report a candidate scraped from a truncated document as complete. + chosen = { ...chosen, truncated: true }; + } + } if (chosen === undefined) { return null; } diff --git a/test/continuous-test-suite-coerce-truncation.ts b/test/continuous-test-suite-coerce-truncation.ts new file mode 100644 index 000000000..2e46ddaa5 --- /dev/null +++ b/test/continuous-test-suite-coerce-truncation.ts @@ -0,0 +1,184 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: truncated structured output (pure, no API). + * + * The `huge-output` cell of test:json-e2e caught the native Anthropic-direct + * path returning a STRING as `structuredData` where the schema required an + * object. The failure is the truncation path: when the response is cut at + * `max_tokens` the root brace never closes, so + * + * 1. the balanced-span scan walks past the unclosed root and matches a + * bracket pair that lives INSIDE a string value — `[step 1]` in a shell + * script becomes `["step 1"]`, a syntactically fine but semantically + * BOGUS array, reported with `truncated: false` (silently wrong), and + * 2. at some cut points nothing parsed at all, `coerceJsonToSchema` returned + * null, and the caller kept the raw text — a string. + * + * A prefix sweep over a realistic huge-output payload hit case 1 at ~22% of + * all cut points and case 2 at a handful more. This suite locks in the + * contract: whatever the cut point, a schema request yields a PLAIN OBJECT + * (partial when the model did not finish) and `truncated` is always set. + * + * Run: npx tsx test/continuous-test-suite-coerce-truncation.ts + */ +import { z } from "zod"; +import { defineSuite, assert, assertEqual } from "./helpers/harness.js"; +import { + coerceJsonToSchema, + schemaAccepts, +} from "../src/lib/utils/json/coerce.js"; + +const { test, runSuite } = defineSuite("Truncated structured-output recovery"); + +// The e2e `huge-output` schema: a large free-form `content` is where a +// max_tokens cut lands in practice. +const agentSchema = z.object({ + summary: z.string().min(1).max(4000), + attachment: z + .object({ + filename: z.string().min(1), + extension: z.string().min(1), + mimetype: z.string().min(1), + content: z.string().min(1), + }) + .nullable(), +}); + +// A shell script full of `[step N]` brackets and Windows backslash paths — +// exactly the payload the e2e huge-output prompt asks for, and exactly what +// makes an unclosed-root scan find bogus inner arrays. +const script = Array.from( + { length: 30 }, + (_, i) => + `echo "[step ${i + 1}] validating C:\\Users\\test\\app — grep -E \\"[0-9]\\d+\\""`, +).join("\n"); + +const FULL = JSON.stringify({ + summary: "a deployment probe script", + attachment: { + filename: "probe", + extension: "sh", + mimetype: "text/x-shellscript", + content: script, + }, +}); + +const isPlainObject = (v: unknown): boolean => + v !== null && typeof v === "object" && !Array.isArray(v); + +await test("every truncation point yields a plain object and reports truncated", () => { + // Structural diagnostics ONLY: a cut position cannot match the harness's + // isExpectedProviderError() predicate, but a recovered payload fragment can. + // Never interpolate a value into an assertion message here — see CLAUDE.md + // "Keep payloads out of assertion messages". + let hasNonObject = false; + let firstDegradedCut: number | undefined; + let hasUnflagged = false; + let firstUnflaggedCut: number | undefined; + for (let cut = 1; cut < FULL.length; cut++) { + const r = coerceJsonToSchema(FULL.slice(0, cut), agentSchema); + if (r === null || !isPlainObject(r?.structuredData)) { + hasNonObject = true; + firstDegradedCut ??= cut; + } + if (r !== null && !r.truncated) { + hasUnflagged = true; + firstUnflaggedCut ??= cut; + } + if (hasNonObject && hasUnflagged) { + break; + } + } + assertEqual( + hasNonObject, + false, + `each truncated prefix must recover a plain object${firstDegradedCut === undefined ? "" : ` (first degraded at cut ${firstDegradedCut})`}`, + ); + assertEqual( + hasUnflagged, + false, + `each truncated prefix must report truncation${firstUnflaggedCut === undefined ? "" : ` (first unflagged at cut ${firstUnflaggedCut})`}`, + ); +}); + +await test("a bracket pair inside a string value never wins over the partial root", () => { + // Cut just past `"content":"echo \"[step 1]` — the only BALANCED span in the + // text is the `[step 1]` living inside the (unterminated) string value. + const cut = FULL.slice(0, FULL.indexOf("[step 1]") + 30); + const r = coerceJsonToSchema(cut, agentSchema); + assert(isPlainObject(r?.structuredData), "recovered value is an object"); + const obj = r?.structuredData as Record; + assertEqual(obj.summary, "a deployment probe script", "root fields kept"); + assertEqual(r?.truncated, true, "truncation is surfaced"); +}); + +await test("salvages a partial object where jsonrepair alone fails", () => { + // Cut mid-value at `"mimetype":"text/` — jsonrepair cannot close this on its + // own and coerceJsonToSchema used to return null here, which left the caller + // holding the raw text. The structural back-off drops to the last completed + // field instead. + const cut = FULL.slice(0, FULL.indexOf("text/x-shellscript") + 5); + const r = coerceJsonToSchema(cut, agentSchema); + assert(isPlainObject(r?.structuredData), "partial object recovered"); + const att = (r?.structuredData as { attachment?: Record }) + .attachment; + assertEqual(att?.filename, "probe", "completed fields survive the salvage"); + assertEqual(att?.extension, "sh", "all completed fields survive"); + assertEqual(r?.truncated, true, "partial object is flagged truncated"); +}); + +await test("a complete document is never mislabelled truncated", () => { + const r = coerceJsonToSchema(FULL, agentSchema); + assertEqual(r?.truncated, false, "complete output not flagged truncated"); + assert( + agentSchema.safeParse(r?.structuredData).success, + "complete output satisfies the schema", + ); +}); + +await test("a root array truncated mid-element still recovers an array", () => { + const arrSchema = z.array(z.object({ id: z.number(), label: z.string() })); + const full = JSON.stringify([ + { id: 1, label: "alpha [1]" }, + { id: 2, label: "beta [2]" }, + { id: 3, label: "gamma [3]" }, + ]); + const r = coerceJsonToSchema(full.slice(0, full.length - 12), arrSchema); + assertEqual(Array.isArray(r?.structuredData), true, "array root preserved"); + assertEqual(r?.truncated, true, "truncation surfaced for array roots"); +}); + +// ── schemaAccepts: the gate that stops a raw string reaching structuredData ── + +await test("schemaAccepts rejects a raw string under an object schema", () => { + assertEqual( + schemaAccepts(agentSchema, "the model's raw text answer"), + false, + "an object schema must not accept a string", + ); + assertEqual( + schemaAccepts(agentSchema, { summary: "ok", attachment: null }), + true, + "a conforming object is accepted", + ); +}); + +await test("schemaAccepts keeps string-root schemas working", () => { + assertEqual( + schemaAccepts(z.string(), "hello"), + true, + "string schema + string", + ); + assertEqual(schemaAccepts(z.number(), 42), true, "number schema + number"); +}); + +await test("schemaAccepts is permissive when there is no validatable schema", () => { + assertEqual(schemaAccepts(undefined, "anything"), true, "no schema → accept"); + assertEqual( + schemaAccepts({ type: "object" } as unknown as z.ZodTypeAny, "anything"), + true, + "a schema without safeParse must not block a value", + ); +}); + +await runSuite(); diff --git a/test/continuous-test-suite-json-e2e.ts b/test/continuous-test-suite-json-e2e.ts index 9c4305f7c..86a001a87 100644 --- a/test/continuous-test-suite-json-e2e.ts +++ b/test/continuous-test-suite-json-e2e.ts @@ -531,4 +531,45 @@ await test("vertex:claude-sonnet-4-6 — forced truncation is observable, never ); }); +// The exact regression behind #1156 lived on the DIRECT Anthropic path: a +// max_tokens cut left `structuredData` as a STRING (the raw completion) where +// the schema required an object. Force the same cut here and assert the +// contract — structuredData is a plain object (never a string), and +// jsonTruncated === true tells a caller the object is partial. +await test("anthropic:claude-sonnet-4-6 — forced truncation yields a partial object, never a string", async () => { + let res: SchemaResult; + try { + res = (await nl.generate({ + input: { text: HUGE_OUTPUT_PROMPT }, + provider: "anthropic", + model: "claude-sonnet-4-6", + schema: agentSchema, + tools: pingTool, + disableTools: false, + temperature: 0, + maxTokens: 200, // force a cut-off mid-JSON (260-line script won't fit) + })) as SchemaResult; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (isInfraError(msg)) { + throw new Skip(`anthropic: ${msg.slice(0, 120)}`); + } + throw err; + } + JSON.parse(res.content); // must be valid JSON, truncated or not + const isObject = + res.structuredData !== undefined && + res.structuredData !== null && + typeof res.structuredData === "object" && + !Array.isArray(res.structuredData); + assert(isObject, "structuredData must be a plain object, never a string"); + assert( + res.jsonTruncated === true, + `truncation must be flagged jsonTruncated=true (jsonTruncated=${res.jsonTruncated}, finishReason=${res.finishReason})`, + ); + console.log( + ` · direct-Anthropic forced truncation: plain object + jsonTruncated=true`, + ); +}); + await runSuite();